diff --git a/Alignment/CommonAlignment/BuildFile.xml b/Alignment/CommonAlignment/BuildFile.xml index f8625ce106320..5880f7fdcc7db 100644 --- a/Alignment/CommonAlignment/BuildFile.xml +++ b/Alignment/CommonAlignment/BuildFile.xml @@ -1,6 +1,7 @@ + diff --git a/Alignment/CommonAlignment/interface/Alignable.h b/Alignment/CommonAlignment/interface/Alignable.h index f5cb581c1f694..a013c37c5337d 100644 --- a/Alignment/CommonAlignment/interface/Alignable.h +++ b/Alignment/CommonAlignment/interface/Alignable.h @@ -3,6 +3,7 @@ #include "Alignment/CommonAlignment/interface/AlignableSurface.h" #include "Alignment/CommonAlignment/interface/StructureType.h" +#include "Alignment/CommonAlignment/interface/Utilities.h" #include "DataFormats/DetId/interface/DetId.h" class AlignmentErrorsExtended; @@ -18,9 +19,6 @@ class SurfaceDeformation; * Any Alignable object can be moved and rotated. * Also an alignment uncertainty can be set. * - * $Date: 2011/09/19 11:42:35 $ - * $Revision: 1.36 $ - * (last update by $Author: mussgill $) */ class AlignmentParameters; @@ -39,6 +37,8 @@ class Alignable typedef align::Alignables Alignables; typedef align::StructureType StructureType; + enum class CompConstraintType { NONE, POSITION, POSITION_Z }; + /// Constructor from id and surface, setting also geomDetId /// (AlignableNavigator relies on the fact that only AlignableDet/DetUnit have geomDetId!) Alignable( align::ID, const AlignableSurface& ); @@ -50,6 +50,10 @@ class Alignable /// Destructor virtual ~Alignable(); + /// Updater using id and surface. + /// The given id has to match the current id. + void update(align::ID, const AlignableSurface&); + /// Set the AlignmentParameters void setAlignmentParameters( AlignmentParameters* dap ); @@ -184,6 +188,9 @@ class Alignable /// Return the ID of Alignable, i.e. DetId of 'first' component GeomDet(Unit). align::ID id() const { return theId; } + /// Return the alignable type of contraints wrt. its components + virtual CompConstraintType compConstraintType() const { return compConstraintType_; } + /// Recursive printout of alignable information virtual void dump() const = 0; @@ -203,9 +210,17 @@ class Alignable /// cache the current position, rotation and other parameters (e.g. surface deformations), also for possible components virtual void cacheTransformation(); + /// cache for the given run the current position, rotation and other + /// parameters (e.g. surface deformations), also for possible components + virtual void cacheTransformation(const align::RunNumber&); + /// restore the previously cached transformation, also for possible components virtual void restoreCachedTransformation(); + /// restore for the given run the previously cached transformation, also for + /// possible components + virtual void restoreCachedTransformation(const align::RunNumber&); + /// Return survey info const SurveyDet* survey() const { return theSurvey; } @@ -213,11 +228,12 @@ class Alignable void setSurvey( const SurveyDet* ); protected: + template + using Cache = std::map; void addDisplacement( const GlobalVector& displacement ); void addRotation( const RotationType& rotation ); - -protected: + virtual void updateMother(const GlobalVector& shift); DetId theDetId; // used to check if Alignable is associated to a GeomDet // ugly way to keep AlignableNavigator happy for now @@ -233,10 +249,17 @@ class Alignable GlobalVector theCachedDisplacement; RotationType theCachedRotation; + CompConstraintType compConstraintType_{CompConstraintType::NONE}; + Alignables theDeepComponents; // list of lowest daughters // contain itself if Alignable is a unit + Cache surfacesCache_; + Cache displacementsCache_; + Cache rotationsCache_; + private: + /// private default ctr. to enforce usage of the specialised ones Alignable() {}; diff --git a/Alignment/CommonAlignment/interface/AlignableBeamSpot.h b/Alignment/CommonAlignment/interface/AlignableBeamSpot.h index 9e794fe99ef15..876cf20c2a289 100644 --- a/Alignment/CommonAlignment/interface/AlignableBeamSpot.h +++ b/Alignment/CommonAlignment/interface/AlignableBeamSpot.h @@ -87,6 +87,9 @@ class AlignableBeamSpot : public Alignable void initialize(double x, double y, double z, double dxdz, double dydz); + /// reset beam spot to the uninitialized state + void reset(); + /// returns the DetId corresponding to the alignable beam spot. Also used /// by BeamSpotGeomDet and BeamSpotTransientTrackingRecHit static const DetId detId() { return DetId((DetId::Tracker< void update(T) = delete; + void setSurface( const AlignableSurface& s) { theSurface = s; } StructureType theStructureType; diff --git a/Alignment/CommonAlignment/interface/AlignableCompositeBuilder.h b/Alignment/CommonAlignment/interface/AlignableCompositeBuilder.h index 4e4b4be3f7f64..5b33cabad7cd2 100644 --- a/Alignment/CommonAlignment/interface/AlignableCompositeBuilder.h +++ b/Alignment/CommonAlignment/interface/AlignableCompositeBuilder.h @@ -39,7 +39,7 @@ class AlignableCompositeBuilder { /// - TPBHalfBarrel (with TPBLayer as children) /// - TPBBarrel (with TPBHalfBarrel as children) /// Returns the number of composite Alignables which were built. - unsigned int buildAll(AlignableMap&); + unsigned int buildAll(AlignableMap&, bool update = false); /// Return tracker alignable object ID provider derived from the tracker's geometry const AlignableObjectId& objectIdProvider() const { return alignableObjectId_; } @@ -49,7 +49,7 @@ class AlignableCompositeBuilder { /// Builds the components for a given level in the hierarchy. unsigned int buildLevel(unsigned int parentLevel, AlignableMap&, - std::ostringstream&); + std::ostringstream&, bool update = false); /// Calculates the theoretical max. number of components for a given level /// in the hierarchy. diff --git a/Alignment/CommonAlignment/interface/AlignableDet.h b/Alignment/CommonAlignment/interface/AlignableDet.h index 9ea5badca9286..796205fe1a32a 100644 --- a/Alignment/CommonAlignment/interface/AlignableDet.h +++ b/Alignment/CommonAlignment/interface/AlignableDet.h @@ -17,6 +17,10 @@ class AlignableDet: public AlignableComposite /// Destructor virtual ~AlignableDet(); + /// Updater from GeomDet + /// The given GeomDet id has to match the current id. + void update(const GeomDet* geomDet, bool updateComponents = true); + /// Set the AlignmentPositionError and, if (propagateDown), to all components virtual void setAlignmentPositionError(const AlignmentPositionError &ape, bool propagateDown); diff --git a/Alignment/CommonAlignment/interface/AlignableDetUnit.h b/Alignment/CommonAlignment/interface/AlignableDetUnit.h index b5194f7c76a4e..2faff6d3c10c6 100644 --- a/Alignment/CommonAlignment/interface/AlignableDetUnit.h +++ b/Alignment/CommonAlignment/interface/AlignableDetUnit.h @@ -21,6 +21,10 @@ class AlignableDetUnit : public Alignable /// Destructor virtual ~AlignableDetUnit(); + /// Updater from GeomDetUnit + /// The given GeomDetUnit id has to match the current id. + void update(const GeomDetUnit* geomDetUnit); + /// No components here => exception! virtual void addComponent( Alignable* ); @@ -76,9 +80,15 @@ class AlignableDetUnit : public Alignable /// cache the current position, rotation and other parameters (e.g. surface deformations) virtual void cacheTransformation(); + /// cache for the given run the current position, rotation and other parameters (e.g. surface deformations) + virtual void cacheTransformation(const align::RunNumber&); + /// restore the previously cached transformation virtual void restoreCachedTransformation(); + /// restore for the given run the previously cached transformation + virtual void restoreCachedTransformation(const align::RunNumber&); + /// alignment position error - for checking only, otherwise use alignmentErrors() above! const AlignmentPositionError* alignmentPositionError() const { return theAlignmentPositionError;} @@ -87,6 +97,7 @@ class AlignableDetUnit : public Alignable AlignmentPositionError* theAlignmentPositionError; SurfaceDeformation* theSurfaceDeformation; SurfaceDeformation* theCachedSurfaceDeformation; + Cache surfaceDeformationsCache_; }; #endif diff --git a/Alignment/CommonAlignment/interface/AlignableExtras.h b/Alignment/CommonAlignment/interface/AlignableExtras.h index 0d0460dbc5c3c..01b732fb870aa 100644 --- a/Alignment/CommonAlignment/interface/AlignableExtras.h +++ b/Alignment/CommonAlignment/interface/AlignableExtras.h @@ -47,6 +47,9 @@ class AlignableExtras void initializeBeamSpot(double x, double y, double z, double dxdz, double dydz); + /// Initialize the alignable beam spot with the given parameters + void resetBeamSpot(); + private: AlignableMap alignableLists_; //< kind of map of lists of alignables diff --git a/Alignment/CommonAlignment/interface/Utilities.h b/Alignment/CommonAlignment/interface/Utilities.h index 18a9a8e41dfef..e8583b3d68f6b 100644 --- a/Alignment/CommonAlignment/interface/Utilities.h +++ b/Alignment/CommonAlignment/interface/Utilities.h @@ -13,8 +13,9 @@ #include #include +#include "CondCore/CondDB/interface/Time.h" #include "CondFormats/Alignment/interface/Definitions.h" -#include "Alignment/CommonAlignment/interface/AlignableObjectId.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" class Alignable; class AlignmentParameters; @@ -33,6 +34,10 @@ namespace align typedef std::map, AlgebraicMatrix> Correlations; + using RunNumber = cond::RealTimeType::type; + using RunRange = std::pair; + using RunRanges = std::vector; + /// Convert rotation matrix to angles about x-, y-, z-axes (frame rotation). EulerAngles toAngles( const RotationType& @@ -70,6 +75,12 @@ namespace align void rectify( RotationType& ); + + + RunRanges makeNonOverlappingRunRanges(const edm::VParameterSet& runRanges, + const RunNumber& defaultRun); + RunRanges makeUniqueRunRanges(const edm::VParameterSet& runRanges, + const RunNumber& defaultRun); } #endif diff --git a/Alignment/CommonAlignment/python/tools/trackselectionRefitting.py b/Alignment/CommonAlignment/python/tools/trackselectionRefitting.py index 9301bdc54641e..0963e4de73602 100644 --- a/Alignment/CommonAlignment/python/tools/trackselectionRefitting.py +++ b/Alignment/CommonAlignment/python/tools/trackselectionRefitting.py @@ -153,7 +153,27 @@ def getSequence(process, collection, options["TrackHitFilter"]["Tracker"].update({ "minimumHits": 10, }) - pass + elif collection == "ALCARECOTkAlUpsilonMuMu": + options["TrackSelector"]["Alignment"].update({ + "ptMin": 3.0, + "etaMin": -2.4, + "etaMax": 2.4, + "nHitMin": 10, + "applyMultiplicityFilter": True, + "minMultiplicity": 2, + "maxMultiplicity": 2, + ("minHitsPerSubDet", "inPIXEL"): 1, + ("TwoBodyDecaySelector", "applyChargeFilter"): True, + ("TwoBodyDecaySelector", "charge"): 0, + ("TwoBodyDecaySelector", + "applyMassrangeFilter"): not openMassWindow, + ("TwoBodyDecaySelector", "minXMass"): 9.2, + ("TwoBodyDecaySelector", "maxXMass"): 9.7, + ("TwoBodyDecaySelector", "daughterMass"): 0.105 + }) + options["TrackHitFilter"]["Tracker"].update({ + "minimumHits": 10, + }) else: print "Unknown input track collection:", collection sys.exit(1) diff --git a/Alignment/CommonAlignment/src/Alignable.cc b/Alignment/CommonAlignment/src/Alignable.cc index 23e907d4878c7..6db26357466a0 100644 --- a/Alignment/CommonAlignment/src/Alignable.cc +++ b/Alignment/CommonAlignment/src/Alignable.cc @@ -46,6 +46,25 @@ Alignable::~Alignable() delete theSurvey; } +//__________________________________________________________________________________________________ +void Alignable::update(align::ID id, const AlignableSurface& surf) +{ + if (theId != id) { + throw cms::Exception("Alignment") + << "@SUB=Alignable::update\n" + << "Current alignable ID does not match ID of the update."; + } + const auto shift = surf.position() - theSurface.position(); + theSurface = surf; + + // reset displacement and rotations after update + theDisplacement = GlobalVector(); + theRotation = RotationType(); + + // recalculate containing composite's position + updateMother(shift); +} + //__________________________________________________________________________________________________ bool Alignable::firstCompsWithParams(Alignables ¶mComps) const { @@ -269,7 +288,7 @@ AlignmentSurfaceDeformations* Alignable::surfaceDeformations( void ) const return allSurfaceDeformations; } - + void Alignable::cacheTransformation() { // first treat itself @@ -286,6 +305,18 @@ void Alignable::cacheTransformation() } +void Alignable::cacheTransformation(const align::RunNumber& run) +{ + // first treat itself + surfacesCache_[run] = theSurface; + displacementsCache_[run] = theDisplacement; + rotationsCache_[run] = theRotation; + + // now treat components (a clean design would move that to AlignableComposite...) + const Alignables comps(this->components()); + for (auto& it: comps) it->cacheTransformation(run); +} + void Alignable::restoreCachedTransformation() { // first treat itself @@ -294,7 +325,7 @@ void Alignable::restoreCachedTransformation() theRotation = theCachedRotation; // now treat components (a clean design would move that to AlignableComposite...) - const Alignables comps(this->components()); + const auto comps = this->components(); for (auto it = comps.begin(); it != comps.end(); ++it) { (*it)->restoreCachedTransformation(); @@ -302,6 +333,25 @@ void Alignable::restoreCachedTransformation() } +void Alignable::restoreCachedTransformation(const align::RunNumber& run) +{ + if (surfacesCache_.find(run) == surfacesCache_.end()) { + throw cms::Exception("Alignment") + << "@SUB=Alignable::restoreCachedTransformation\n" + << "Trying to restore cached transformation for a run (" << run + << ") that has not been cached."; + } else { + // first treat itself + theSurface = surfacesCache_[run]; + theDisplacement = displacementsCache_[run]; + theRotation = rotationsCache_[run]; + + // now treat components (a clean design would move that to AlignableComposite...) + const auto comps = this->components(); + for (auto it: comps) it->restoreCachedTransformation(); + } +} + //__________________________________________________________________________________________________ void Alignable::setSurvey( const SurveyDet* survey ) { @@ -310,3 +360,26 @@ void Alignable::setSurvey( const SurveyDet* survey ) theSurvey = survey; } + +//______________________________________________________________________________ +void Alignable::updateMother(const GlobalVector& shift) { + + if (!theMother) return; + + const auto thisComps = this->deepComponents().size(); + const auto motherComps = theMother->deepComponents().size(); + const auto motherShift = shift * static_cast(thisComps) / motherComps; + + switch(theMother->compConstraintType()) { + case CompConstraintType::NONE: + break; + case CompConstraintType::POSITION_Z: + theMother->theSurface.move(GlobalVector(0,0, motherShift.z())); + theMother->updateMother(GlobalVector(0,0, motherShift.z())); + break; + case CompConstraintType::POSITION: + theMother->theSurface.move(motherShift); + theMother->updateMother(motherShift); + break; + } +} diff --git a/Alignment/CommonAlignment/src/AlignableBeamSpot.cc b/Alignment/CommonAlignment/src/AlignableBeamSpot.cc index 5a201d755465d..372c7c6ea89bd 100644 --- a/Alignment/CommonAlignment/src/AlignableBeamSpot.cc +++ b/Alignment/CommonAlignment/src/AlignableBeamSpot.cc @@ -152,6 +152,7 @@ AlignmentErrorsExtended* AlignableBeamSpot::alignmentErrors( void ) const return m_alignmentErrors; } +//______________________________________________________________________________ void AlignableBeamSpot::initialize(double x, double y, double z, double dxdz, double dydz) { @@ -177,3 +178,12 @@ void AlignableBeamSpot::initialize(double x, double y, double z, theInitializedFlag = true; } + +//______________________________________________________________________________ +void AlignableBeamSpot::reset() +{ + Alignable::update(this->id(), AlignableSurface()); + delete theAlignmentPositionError; + theAlignmentPositionError = nullptr; + theInitializedFlag = false; +} diff --git a/Alignment/CommonAlignment/src/AlignableComposite.cc b/Alignment/CommonAlignment/src/AlignableComposite.cc index 86380dc24cc5e..7b0c148b57c00 100644 --- a/Alignment/CommonAlignment/src/AlignableComposite.cc +++ b/Alignment/CommonAlignment/src/AlignableComposite.cc @@ -15,21 +15,52 @@ AlignableComposite::AlignableComposite( const GeomDet* geomDet ) : Alignable( geomDet->geographicalId().rawId(), geomDet->surface() ), theStructureType(align::AlignableDet) { + compConstraintType_ = Alignable::CompConstraintType::POSITION; } +//__________________________________________________________________________________________________ AlignableComposite::AlignableComposite(align::ID id, StructureType type, const RotationType& rot): Alignable(id, rot), theStructureType(type) { + compConstraintType_ = Alignable::CompConstraintType::POSITION; } +//__________________________________________________________________________________________________ AlignableComposite::~AlignableComposite() { for (unsigned int i = 0; i < theComponents.size(); ++i) delete theComponents[i]; } +//__________________________________________________________________________________________________ +void AlignableComposite::update(const GeomDet* geomDet) +{ + if (!geomDet) { + throw cms::Exception("Alignment") + << "@SUB=AlignableComposite::update\n" + << "Trying to update with GeomDet* pointing to 'nullptr'."; + } + + Alignable::update(geomDet->geographicalId().rawId(), geomDet->surface()); +} + +//__________________________________________________________________________________________________ +void AlignableComposite::update(align::ID id, + StructureType type, + const RotationType& rot) +{ + if (theStructureType != type) { + throw cms::Exception("Alignment") + << "@SUB=AlignableComposite::update\n" + << "Current alignable type does not match type of the update."; + } + // composite's position is already updated by components, i.e. it needs to be kept + Alignable::update(id, AlignableSurface{this->globalPosition(), rot}); +} + +//__________________________________________________________________________________________________ void AlignableComposite::addComponent(Alignable* ali) { const Alignables& newComps = ali->deepComponents(); diff --git a/Alignment/CommonAlignment/src/AlignableCompositeBuilder.cc b/Alignment/CommonAlignment/src/AlignableCompositeBuilder.cc index 4654512b91115..0bd8c43108480 100644 --- a/Alignment/CommonAlignment/src/AlignableCompositeBuilder.cc +++ b/Alignment/CommonAlignment/src/AlignableCompositeBuilder.cc @@ -40,7 +40,7 @@ ::clearAlignmentLevels() { //_____________________________________________________________________________ unsigned int AlignableCompositeBuilder -::buildAll(AlignableMap& alignableMap) +::buildAll(AlignableMap& alignableMap, bool update) { auto highestLevel = alignmentLevels_.back()->levelType; @@ -50,7 +50,7 @@ ::buildAll(AlignableMap& alignableMap) unsigned int numCompositeAlignables = 0; for (unsigned int level = 1; level < alignmentLevels_.size(); ++level) { - numCompositeAlignables += buildLevel(level, alignableMap, ss); + numCompositeAlignables += buildLevel(level, alignableMap, ss, update); } ss << "built " << numCompositeAlignables << " CompositeAlignables for " @@ -71,7 +71,8 @@ ::buildAll(AlignableMap& alignableMap) unsigned int AlignableCompositeBuilder ::buildLevel(unsigned int parentLevel, AlignableMap& alignableMap, - std::ostringstream& ss) + std::ostringstream& ss, + bool update) { unsigned int childLevel = parentLevel - 1; unsigned int maxNumParents = maxNumComponents(parentLevel); @@ -81,7 +82,7 @@ ::buildLevel(unsigned int parentLevel, auto& children = alignableMap.find(alignableObjectId_.idToString(childType)); auto& parents = alignableMap.get (alignableObjectId_.idToString(parentType)); - parents.reserve(maxNumParents); + if (!update) parents.reserve(maxNumParents); // This vector is used indicate if a parent already exists. It is initialized // with 'naked' Alignables-pointers; if the pointer is not naked (!= nullptr) @@ -95,7 +96,12 @@ ::buildLevel(unsigned int parentLevel, auto& parent = tmpParents[index]; // if parent was not built yet ... - if (!parent) { + if (!parent && !update) { + if (update) { + throw cms::Exception("LogicError") + << "@SUB=AlignableCompositeBuilder::buildLevel\n" + << "trying to update a non-existing AlignableComposite"; + } // ... build new composite Alignable with ID of child (obviously its the // first child of the Alignable) if (alignmentLevels_[parentLevel]->isFlat) { @@ -106,10 +112,24 @@ ::buildLevel(unsigned int parentLevel, align::RotationType()); } parents.push_back(parent); + } else if (update) { + if (alignmentLevels_[parentLevel]->isFlat) { + // needed to update rotation of flat composites + auto mother = dynamic_cast(child->mother()); + if (!mother) { + throw cms::Exception("LogicError") + << "@SUB=AlignableCompositeBuilder::buildLevel\n" + << "trying to update a flat composite that is not of type " + << "AlignableComposite"; + } + if (mother->id() == child->id()) { + mother->update(child->id(), parentType, child->globalRotation()); + } + } } - // in all cases add the child to the parent Alignable - parent->addComponent(child); + // in all cases (except updates) add the child to the parent Alignable + if (!update) parent->addComponent(child); } ss << " built " << parents.size() << " " diff --git a/Alignment/CommonAlignment/src/AlignableDet.cc b/Alignment/CommonAlignment/src/AlignableDet.cc index 5c0cfd76ac87a..365f7d39e66d9 100644 --- a/Alignment/CommonAlignment/src/AlignableDet.cc +++ b/Alignment/CommonAlignment/src/AlignableDet.cc @@ -17,6 +17,10 @@ AlignableDet::AlignableDet( const GeomDet* geomDet, bool addComponents ) : AlignableComposite( geomDet ), theAlignmentPositionError(0) { + // ensure that the surface is not constrained to the average position of the + // components: + compConstraintType_ = Alignable::CompConstraintType::NONE; + if (geomDet->alignmentPositionError()) { // false: do not propagate APE to (anyway not yet existing) daughters this->setAlignmentPositionError(*(geomDet->alignmentPositionError()), false); @@ -55,6 +59,62 @@ AlignableDet::~AlignableDet() } +//______________________________________________________________________________ +void AlignableDet::update(const GeomDet* geomDet, bool updateComponents) +{ + AlignableComposite::update(geomDet); + + if (geomDet->alignmentPositionError()) { + // false: do not propagate APE to daughters (done by their update functions) + this->setAlignmentPositionError(*(geomDet->alignmentPositionError()), false); + } + + if (updateComponents) { + if (geomDet->components().size() == 0 ) { // Is a DetUnit + throw cms::Exception("BadHierarchy") + << "[AlignableDet] GeomDet with DetId " + << geomDet->geographicalId().rawId() + << " has no components, use AlignableDetUnit.\n"; + } else { // Push back all components + const auto& geomDets = geomDet->components(); + for (const auto& idet: geomDets) { + auto unit = dynamic_cast(idet); + if (!unit) { + throw cms::Exception("BadHierarchy") + << "[AlignableDet] component not GeomDetUnit, call with " + << "updateComponents==false and build hierarchy yourself.\n"; + // -> e.g. AlignableDTChamber + } + + const auto components = this->components(); + auto comp = + std::find_if(components.begin(), components.end(), + [&unit](const auto& c) { + return c->id() == unit->geographicalId().rawId(); }); + + if (comp != components.end()) { + auto aliDetUnit = dynamic_cast(*comp); + if (aliDetUnit) { + aliDetUnit->update(unit); + } else { + throw cms::Exception("LogicError") + << "[AlignableDet::update] cast to 'AlignableDetUnit*' failed " + << "while it should not\n"; + } + } else { + throw cms::Exception("GeometryMismatch") + << "[AlignableDet::update] GeomDet with DetId " + << unit->geographicalId().rawId() + << " not found in current geometry.\n"; + } + } + } + // Ensure that the surface is not screwed up by update of components, it must stay the GeomDet's one: + theSurface = AlignableSurface(geomDet->surface()); + } // end updateComponents +} + + //__________________________________________________________________________________________________ void AlignableDet::setAlignmentPositionError(const AlignmentPositionError& ape, bool propagateDown) { diff --git a/Alignment/CommonAlignment/src/AlignableDetUnit.cc b/Alignment/CommonAlignment/src/AlignableDetUnit.cc index cfdbec8124fc9..aa34956aaa741 100644 --- a/Alignment/CommonAlignment/src/AlignableDetUnit.cc +++ b/Alignment/CommonAlignment/src/AlignableDetUnit.cc @@ -38,6 +38,29 @@ AlignableDetUnit::~AlignableDetUnit() delete theAlignmentPositionError; delete theSurfaceDeformation; delete theCachedSurfaceDeformation; + for (auto surface: surfaceDeformationsCache_) delete surface.second; +} + +//__________________________________________________________________________________________________ +void AlignableDetUnit::update(const GeomDetUnit *geomDetUnit) +{ + if (!geomDetUnit) { + throw cms::Exception("Alignment") + << "@SUB=AlignableDetUnit::update\n" + << "Trying to update with GeomDetUnit* pointing to 'nullptr'."; + } + + Alignable::update(geomDetUnit->geographicalId().rawId(), geomDetUnit->surface()); + + if (geomDetUnit->alignmentPositionError()) { // take over APE from geometry + // 2nd argument w/o effect: + this->setAlignmentPositionError(*(geomDetUnit->alignmentPositionError()), false); + } + + if (geomDetUnit->surfaceDeformation()) { // take over surface modification + // 2nd argument w/o effect: + this->setSurfaceDeformation(geomDetUnit->surfaceDeformation(), false); + } } //__________________________________________________________________________________________________ @@ -155,13 +178,22 @@ void AlignableDetUnit::addSurfaceDeformation(const SurfaceDeformation *deformati //__________________________________________________________________________________________________ void AlignableDetUnit::dump() const { + std::ostringstream parameters; + if (theSurfaceDeformation) { + parameters << " surface deformation parameters:"; + for (const auto& param: theSurfaceDeformation->parameters()) { + parameters << " " << param; + } + } else { + parameters << " no surface deformation parameters"; + } edm::LogInfo("AlignableDump") << " AlignableDetUnit has position = " << this->globalPosition() << ", orientation:" << std::endl << this->globalRotation() << std::endl << " total displacement and rotation: " << this->displacement() << std::endl - << this->rotation(); - + << this->rotation() << "\n" + << parameters.str(); } @@ -236,6 +268,24 @@ void AlignableDetUnit::cacheTransformation() theCachedSurfaceDeformation = theSurfaceDeformation->clone(); } +//__________________________________________________________________________________________________ +void AlignableDetUnit::cacheTransformation(const align::RunNumber& run) +{ + surfacesCache_[run] = theSurface; + displacementsCache_[run] = theDisplacement; + rotationsCache_[run] = theRotation; + + auto existingCache = surfaceDeformationsCache_.find(run); + if (existingCache != surfaceDeformationsCache_.end()) { + delete existingCache->second; + existingCache->second = 0; + } + + if (theSurfaceDeformation) { + surfaceDeformationsCache_[run] = theSurfaceDeformation->clone(); + } +} + //__________________________________________________________________________________________________ void AlignableDetUnit::restoreCachedTransformation() { @@ -252,3 +302,27 @@ void AlignableDetUnit::restoreCachedTransformation() this->setSurfaceDeformation(theCachedSurfaceDeformation, false); } } + +//__________________________________________________________________________________________________ +void AlignableDetUnit::restoreCachedTransformation(const align::RunNumber& run) +{ + if (surfacesCache_.find(run) == surfacesCache_.end()) { + throw cms::Exception("Alignment") + << "@SUB=Alignable::restoreCachedTransformation\n" + << "Trying to restore cached transformation for a run (" << run + << ") that has not been cached."; + } else { + theSurface = surfacesCache_[run]; + theDisplacement = displacementsCache_[run]; + theRotation = rotationsCache_[run]; + + if (theSurfaceDeformation) { + delete theSurfaceDeformation; + theSurfaceDeformation = 0; + } + + if (surfaceDeformationsCache_[run]) { + this->setSurfaceDeformation(surfaceDeformationsCache_[run], false); + } + } +} diff --git a/Alignment/CommonAlignment/src/AlignableExtras.cc b/Alignment/CommonAlignment/src/AlignableExtras.cc index fb1474830979b..ea4c6ee0aa655 100644 --- a/Alignment/CommonAlignment/src/AlignableExtras.cc +++ b/Alignment/CommonAlignment/src/AlignableExtras.cc @@ -86,6 +86,7 @@ AlignmentErrorsExtended* AlignableExtras::alignmentErrors( void ) const return m_alignmentErrors; } +//______________________________________________________________________________ void AlignableExtras::initializeBeamSpot(double x, double y, double z, double dxdz, double dydz) { @@ -98,3 +99,17 @@ void AlignableExtras::initializeBeamSpot(double x, double y, double z, << " AlignableBeamSpot not available. Cannot initialize!" << std::endl; } } + +//______________________________________________________________________________ +void AlignableExtras::resetBeamSpot() +{ + align::Alignables& alis = beamSpot(); + AlignableBeamSpot * aliBS = dynamic_cast(alis.back()); + if (aliBS) { + aliBS->reset(); + } else { + edm::LogWarning("AlignableExtras") + << "@SUB=AlignableExtras::resetBeamSpot" + << "AlignableBeamSpot not available. Cannot reset!" << std::endl; + } +} diff --git a/Alignment/CommonAlignment/src/Utilities.cc b/Alignment/CommonAlignment/src/Utilities.cc index 1d4a3f1f7f87d..5ee39b54dc005 100644 --- a/Alignment/CommonAlignment/src/Utilities.cc +++ b/Alignment/CommonAlignment/src/Utilities.cc @@ -1,10 +1,9 @@ // #include "Math/GenVector/Rotation3D.h" #include "FWCore/MessageLogger/interface/MessageLogger.h" +#include "FWCore/Utilities/interface/Parse.h" #include "Alignment/CommonAlignment/interface/Utilities.h" -#include "Alignment/TrackerAlignment/interface/AlignableTracker.h" -#include "Alignment/MuonAlignment/interface/AlignableMuon.h" align::EulerAngles align::toAngles(const RotationType& rot) @@ -213,3 +212,62 @@ void align::rectify(RotationType& rot) rot = toMatrix( toAngles(rot) ); // fast rectification but less precise } + + +align::RunRanges +align::makeNonOverlappingRunRanges(const edm::VParameterSet& runRanges, + const RunNumber& defaultRun) +{ + const auto beginValue = cond::timeTypeSpecs[cond::runnumber].beginValue; + const auto endValue = cond::timeTypeSpecs[cond::runnumber].endValue; + RunRanges uniqueRunRanges; + if (!runRanges.empty()) { + std::map uniqueFirstRunNumbers; + for (const auto& ipset: runRanges) { + const auto RunRangeStrings = + ipset.getParameter >("RunRanges"); + for (const auto& irange: RunRangeStrings) { + if (irange.find(':')==std::string::npos) { + long int temp{strtol(irange.c_str(), 0, 0)}; + auto first = (temp != -1) ? temp : beginValue; + uniqueFirstRunNumbers[first] = first; + } else { + edm::LogWarning("BadConfig") + << "@SUB=align::makeNonOverlappingRunRanges" + << "Config file contains old format for 'RunRangeSelection'. Only " + << "the start run number is used internally. The number of the last" + << " run is ignored and can besafely removed from the config file."; + auto tokens = edm::tokenize(irange, ":"); + long int temp{strtol(tokens[0].c_str(), 0, 0)}; + auto first = (temp != -1) ? temp : beginValue; + uniqueFirstRunNumbers[first] = first; + } + } + } + + for (const auto& iFirst: uniqueFirstRunNumbers) { + uniqueRunRanges.push_back(std::make_pair(iFirst.first, endValue)); + } + for (unsigned int i = 0;i ConstTrajTrackPairs; typedef std::vector Calibrations; -typedef cond::RealTimeType::type RunNumber; -typedef std::pair RunRange; -typedef std::vector RunRanges; - class AlignmentAlgorithmBase @@ -65,8 +61,8 @@ class AlignmentAlgorithmBase // in other files. typedef std::pair ConstTrajTrackPair; typedef std::vector< ConstTrajTrackPair > ConstTrajTrackPairCollection; - typedef cond::RealTimeType::type RunNumber; - typedef std::pair RunRange; + using RunNumber = align::RunNumber; + using RunRange = align::RunRange; /// define event information passed to algorithms class EventInfo { @@ -149,7 +145,7 @@ class AlignmentAlgorithmBase virtual void run( const edm::EventSetup &setup, const EventInfo &eventInfo) = 0; /// called at begin of run - virtual void beginRun(const edm::EventSetup &setup) {}; + virtual void beginRun(const edm::Run&, const edm::EventSetup&, bool changed) {}; /// called at end of run - order of arguments like in EDProducer etc. virtual void endRun(const EndRunInfo &runInfo, const edm::EventSetup &setup) {}; diff --git a/Alignment/CommonAlignmentAlgorithm/interface/AlignmentParameterStore.h b/Alignment/CommonAlignmentAlgorithm/interface/AlignmentParameterStore.h index d83f6856752a1..c8a8e8c732e2e 100644 --- a/Alignment/CommonAlignmentAlgorithm/interface/AlignmentParameterStore.h +++ b/Alignment/CommonAlignmentAlgorithm/interface/AlignmentParameterStore.h @@ -86,9 +86,15 @@ class AlignmentParameterStore /// cache the current position, rotation and other parameters void cacheTransformations(void); + /// cache for the given run the current position, rotation and other parameters + void cacheTransformations(const align::RunNumber&); + /// restore the previously cached position, rotation and other parameters void restoreCachedTransformations(void); + /// restore for the given run the previously cached position, rotation and other parameters + void restoreCachedTransformations(const align::RunNumber&); + /// acquire shifts/rotations from alignables of the store and copy into /// alignment parameters (local frame) void acquireRelativeParameters(void); diff --git a/Alignment/CommonAlignmentAlgorithm/src/AlignmentParameterStore.cc b/Alignment/CommonAlignmentAlgorithm/src/AlignmentParameterStore.cc index 11aca49a5b154..7866cd18b3fa5 100644 --- a/Alignment/CommonAlignmentAlgorithm/src/AlignmentParameterStore.cc +++ b/Alignment/CommonAlignmentAlgorithm/src/AlignmentParameterStore.cc @@ -351,6 +351,13 @@ void AlignmentParameterStore::cacheTransformations(void) } +//__________________________________________________________________________________________________ +void AlignmentParameterStore::cacheTransformations(const align::RunNumber& run) +{ + for (const auto& iali: theAlignables) iali->cacheTransformation(run); +} + + //__________________________________________________________________________________________________ void AlignmentParameterStore::restoreCachedTransformations(void) { @@ -359,6 +366,14 @@ void AlignmentParameterStore::restoreCachedTransformations(void) (*iali)->restoreCachedTransformation(); } + +//__________________________________________________________________________________________________ +void AlignmentParameterStore::restoreCachedTransformations(const align::RunNumber& run) +{ + for (const auto& iali: theAlignables) iali->restoreCachedTransformation(run); +} + + //__________________________________________________________________________________________________ void AlignmentParameterStore::acquireRelativeParameters(void) { @@ -686,6 +701,8 @@ ::hierarchyConstraints(const Alignable *ali, const align::Alignables &aliComps, const ParametersToParametersDerivatives p2pDerivs(**iComp, *ali); if (!p2pDerivs.isOK()) { + // std::cerr << (*iComp)->alignmentParameters()->type() << " " + // << ali->alignmentParameters()->type() << std::endl; throw cms::Exception("BadConfig") << "AlignmentParameterStore::hierarchyConstraints" << " Bad match of types of AlignmentParameters classes.\n"; diff --git a/Alignment/CommonAlignmentProducer/plugins/AlignmentProducer.cc b/Alignment/CommonAlignmentProducer/plugins/AlignmentProducer.cc index ebd6650ed4ed4..f6df335c88f53 100644 --- a/Alignment/CommonAlignmentProducer/plugins/AlignmentProducer.cc +++ b/Alignment/CommonAlignmentProducer/plugins/AlignmentProducer.cc @@ -24,8 +24,6 @@ #include "FWCore/Framework/interface/ESTransientHandle.h" #include "FWCore/Framework/interface/Run.h" -#include "FWCore/Utilities/interface/Parse.h" - // Conditions database #include "FWCore/ServiceRegistry/interface/Service.h" #include "CondCore/DBOutputService/interface/PoolDBOutputService.h" @@ -81,6 +79,9 @@ AlignmentProducer::AlignmentProducer(const edm::ParameterSet& iConfig) : theAlignmentAlgo(0), theAlignmentParameterStore(0), theAlignableExtras(0), theAlignableTracker(0), theAlignableMuon(0), globalPositions_(0), + uniqueRunRanges_ + (align::makeUniqueRunRanges(iConfig.getParameter("RunRangeSelection"), + cond::timeTypeSpecs[cond::runnumber].beginValue)), nevent_(0), theParameterSet(iConfig), theMaxLoops( iConfig.getUntrackedParameter("maxLoops") ), stNFixAlignables_(iConfig.getParameter("nFixAlignables") ), @@ -96,6 +97,7 @@ AlignmentProducer::AlignmentProducer(const edm::ParameterSet& iConfig) : doMuon_( iConfig.getUntrackedParameter("doMuon") ), useExtras_( iConfig.getUntrackedParameter("useExtras") ), useSurvey_( iConfig.getParameter("useSurvey") ), + enableAlignableUpdates_(iConfig.getParameter("enableAlignableUpdates")), tjTkAssociationMapTag_(iConfig.getParameter("tjTkAssociationMapTag")), beamSpotTag_(iConfig.getParameter("beamSpotTag")), tkLasBeamTag_(iConfig.getParameter("tkLasBeamTag")), @@ -113,15 +115,7 @@ AlignmentProducer::AlignmentProducer(const edm::ParameterSet& iConfig) : } // Create the alignment algorithm - edm::ParameterSet algoConfig = iConfig.getParameter( "algoConfig" ); - edm::VParameterSet iovSelection = iConfig.getParameter( "RunRangeSelection" ); - algoConfig.addUntrackedParameter( "RunRangeSelection", iovSelection ); - std::string algoName = algoConfig.getParameter( "algoName" ); - theAlignmentAlgo = AlignmentAlgorithmPluginFactory::get( )->create( algoName, algoConfig ); - - // Check if found - if ( !theAlignmentAlgo ) - throw cms::Exception("BadConfig") << "Couldn't find algorithm called " << algoName; + createAlignmentAlgorithm(iConfig); // Now create monitors: edm::ParameterSet monitorConfig = iConfig.getParameter( "monitorConfig" ); @@ -199,109 +193,8 @@ AlignmentProducer::produceCSC( const MuonGeometryRecord& iRecord ) void AlignmentProducer::beginOfJob( const edm::EventSetup& iSetup ) { edm::LogInfo("Alignment") << "@SUB=AlignmentProducer::beginOfJob"; + initAlignmentAlgorithm(iSetup); - //Retrieve tracker topology from geometry - edm::ESHandle tTopoHandle; - iSetup.get().get(tTopoHandle); - const TrackerTopology* const tTopo = tTopoHandle.product(); - - // Create the geometries from the ideal geometries (first time only) - this->createGeometries_( iSetup ); - - // Retrieve and apply alignments, if requested (requires DB setup) - if ( applyDbAlignment_ ) { - // we need GlobalPositionRcd - and have to keep track for later removal - // before writing again to DB... - edm::ESHandle globalPositionRcd; - iSetup.get().get(globalPositionRcd); - globalPositions_ = new Alignments(*globalPositionRcd); - - if ( doTracker_ ) { // apply to tracker - this->applyDB - (&(*theTracker), iSetup, - align::DetectorGlobalPosition(*globalPositions_, DetId(DetId::Tracker))); - this->applyDB(&(*theTracker), iSetup); - } - - if ( doMuon_ ) { // apply to tracker - this->applyDB - (&(*theMuonDT), iSetup, - align::DetectorGlobalPosition(*globalPositions_, DetId(DetId::Muon))); - this->applyDB - (&(*theMuonCSC), iSetup, - align::DetectorGlobalPosition(*globalPositions_, DetId(DetId::Muon))); - } - } - - // Create alignable tracker and muon - if (doTracker_) { - theAlignableTracker = new AlignableTracker( &(*theTracker), tTopo ); - } - - if (doMuon_) { - theAlignableMuon = new AlignableMuon( &(*theMuonDT), &(*theMuonCSC) ); - } - - if (useExtras_) { - theAlignableExtras = new AlignableExtras(); - } - - // Create alignment parameter builder - edm::LogInfo("Alignment") << "@SUB=AlignmentProducer::beginOfJob" - << "Creating AlignmentParameterBuilder"; - edm::ParameterSet aliParamBuildCfg = - theParameterSet.getParameter("ParameterBuilder"); - AlignmentParameterBuilder alignmentParameterBuilder(theAlignableTracker, - theAlignableMuon, - theAlignableExtras, - aliParamBuildCfg ); - // Fix alignables if requested - if (stNFixAlignables_>0) alignmentParameterBuilder.fixAlignables(stNFixAlignables_); - - // Get list of alignables - Alignables theAlignables = alignmentParameterBuilder.alignables(); - - edm::LogInfo("Alignment") << "Constructed theAlignables calling alignmentParameterBuilder.alignables()"; - edm::LogInfo("Alignment") << "@SUB=AlignmentProducer::beginOfJob" - << "got " << theAlignables.size() << " alignables"; - - // Create AlignmentParameterStore - edm::ParameterSet aliParamStoreCfg = - theParameterSet.getParameter("ParameterStore"); - theAlignmentParameterStore = new AlignmentParameterStore(theAlignables, aliParamStoreCfg); - edm::LogInfo("Alignment") << "@SUB=AlignmentProducer::beginOfJob" - << "AlignmentParameterStore created!"; - - // Apply misalignment scenario to alignable tracker and muon if requested - // WARNING: this assumes scenarioConfig can be passed to both muon and tracker - if (doMisalignmentScenario_ && (doTracker_ || doMuon_)) { - edm::LogInfo("Alignment") << "@SUB=AlignmentProducer::beginOfJob" - << "Applying misalignment scenario to " - << (doTracker_ ? "tracker" : "") - << (doMuon_ ? (doTracker_ ? " and muon" : "muon") : "."); - edm::ParameterSet scenarioConfig - = theParameterSet.getParameter( "MisalignmentScenario" ); - if (doTracker_) { - TrackerScenarioBuilder scenarioBuilder( theAlignableTracker ); - scenarioBuilder.applyScenario( scenarioConfig ); - } - if (doMuon_) { - MuonScenarioBuilder muonScenarioBuilder( theAlignableMuon ); - muonScenarioBuilder.applyScenario( scenarioConfig ); - } - } else { - edm::LogInfo("Alignment") << "@SUB=AlignmentProducer::beginOfJob" - << "NOT applying misalignment scenario!"; - } - - // Apply simple misalignment - const std::string sParSel(theParameterSet.getParameter("parameterSelectorSimple")); - this->simpleMisalignment_(theAlignables, sParSel, stRandomShift_, stRandomRotation_, true); - - // Initialize alignment algorithm and integrated calibration and pass the latter to algorithm - theAlignmentAlgo->initialize( iSetup, - theAlignableTracker, theAlignableMuon, theAlignableExtras, - theAlignmentParameterStore ); for (auto iCal = theCalibrations.begin(); iCal != theCalibrations.end(); ++iCal) { (*iCal)->beginOfJob(theAlignableTracker, theAlignableMuon, theAlignableExtras); } @@ -333,35 +226,23 @@ void AlignmentProducer::endOfJob() edm::LogError("Alignment") << "@SUB=AlignmentProducer::endOfJob" << "Did not process any " << "events in last loop, do not dare to store to DB."; } else { - - // Expand run ranges and make them unique - edm::VParameterSet runRangeSelectionVPSet(theParameterSet.getParameter("RunRangeSelection")); - RunRanges uniqueRunRanges(this->makeNonOverlappingRunRanges(runRangeSelectionVPSet)); - if (uniqueRunRanges.empty()) { // create dummy IOV - const RunRange runRange(cond::timeTypeSpecs[cond::runnumber].beginValue, - cond::timeTypeSpecs[cond::runnumber].endValue); - uniqueRunRanges.push_back(runRange); - } - std::vector beamSpotParameters; - for (RunRanges::const_iterator iRunRange = uniqueRunRanges.begin(); - iRunRange != uniqueRunRanges.end(); - ++iRunRange) { + for (const auto& iRunRange: uniqueRunRanges_) { - theAlignmentAlgo->setParametersForRunRange(*iRunRange); + theAlignmentAlgo->setParametersForRunRange(iRunRange); // Save alignments to database if (saveToDB_ || saveApeToDB_ || saveDeformationsToDB_) - this->writeForRunRange((*iRunRange).first); + this->writeForRunRange(iRunRange.first); // Deal with extra alignables, e.g. beam spot if (theAlignableExtras) { - Alignables &alis = theAlignableExtras->beamSpot(); - if (!alis.empty()) { - BeamSpotAlignmentParameters *beamSpotAliPars = dynamic_cast(alis[0]->alignmentParameters()); - beamSpotParameters.push_back(beamSpotAliPars->parameters()); - } + Alignables &alis = theAlignableExtras->beamSpot(); + if (!alis.empty()) { + BeamSpotAlignmentParameters *beamSpotAliPars = dynamic_cast(alis[0]->alignmentParameters()); + beamSpotParameters.push_back(beamSpotAliPars->parameters()); + } } } @@ -369,17 +250,17 @@ void AlignmentProducer::endOfJob() std::ostringstream bsOutput; std::vector::const_iterator itPar = beamSpotParameters.begin(); - for (RunRanges::const_iterator iRunRange = uniqueRunRanges.begin(); - iRunRange != uniqueRunRanges.end(); - ++iRunRange, ++itPar) { - bsOutput << "Run range: " << (*iRunRange).first << " - " << (*iRunRange).second << "\n"; - bsOutput << " Displacement: x=" << (*itPar)[0] << ", y=" << (*itPar)[1] << "\n"; - bsOutput << " Slope: dx/dz=" << (*itPar)[2] << ", dy/dz=" << (*itPar)[3] << "\n"; + for (auto iRunRange = uniqueRunRanges_.cbegin(); + iRunRange != uniqueRunRanges_.cend(); + ++iRunRange, ++itPar) { + bsOutput << "Run range: " << (*iRunRange).first << " - " << (*iRunRange).second << "\n"; + bsOutput << " Displacement: x=" << (*itPar)[0] << ", y=" << (*itPar)[1] << "\n"; + bsOutput << " Slope: dx/dz=" << (*itPar)[2] << ", dy/dz=" << (*itPar)[3] << "\n"; } edm::LogInfo("Alignment") << "@SUB=AlignmentProducer::endOfJob" - << "Parameters for alignable beamspot:\n" - << bsOutput.str(); + << "Parameters for alignable beamspot:\n" + << bsOutput.str(); } for (auto iCal = theCalibrations.begin(); iCal != theCalibrations.end(); ++iCal) { @@ -407,30 +288,7 @@ void AlignmentProducer::startingNewLoop(unsigned int iLoop ) for (std::vector::const_iterator monitor = theMonitors.begin(); monitor != theMonitors.end(); ++monitor) { (*monitor)->startingNewLoop(); } - - edm::LogInfo("Alignment") << "@SUB=AlignmentProducer::startingNewLoop" - << "Now physically apply alignments to geometry..."; - - - // Propagate changes to reconstruction geometry (from initialisation or iteration) - GeometryAligner aligner; - if ( doTracker_ ) { - std::auto_ptr alignments(theAlignableTracker->alignments()); - std::auto_ptr alignmentErrors(theAlignableTracker->alignmentErrors()); - aligner.applyAlignments( &(*theTracker),&(*alignments),&(*alignmentErrors), AlignTransform() ); // don't apply global a second time! - std::auto_ptr aliDeforms(theAlignableTracker->surfaceDeformations()); - aligner.attachSurfaceDeformations(&(*theTracker), &(*aliDeforms)); - - } - if ( doMuon_ ) { - std::auto_ptr dtAlignments( theAlignableMuon->dtAlignments()); - std::auto_ptr dtAlignmentErrorsExtended( theAlignableMuon->dtAlignmentErrorsExtended()); - std::auto_ptr cscAlignments( theAlignableMuon->cscAlignments()); - std::auto_ptr cscAlignmentErrorsExtended( theAlignableMuon->cscAlignmentErrorsExtended()); - - aligner.applyAlignments( &(*theMuonDT), &(*dtAlignments), &(*dtAlignmentErrorsExtended), AlignTransform() ); // don't apply global a second time! - aligner.applyAlignments( &(*theMuonCSC), &(*cscAlignments), &(*cscAlignmentErrorsExtended), AlignTransform() ); // nope! - } + applyAlignmentsToGeometry(); } @@ -536,7 +394,14 @@ AlignmentProducer::duringLoop( const edm::Event& event, // ---------------------------------------------------------------------------- void AlignmentProducer::beginRun(const edm::Run &run, const edm::EventSetup &setup) { - theAlignmentAlgo->beginRun(setup); // do not forward edm::Run... + const auto update = setupChanged(setup) && enableAlignableUpdates_; + if (update) { + edm::LogInfo("Alignment") << "@SUB=AlignmentProducer::beginRun" + << "EventSetup-Record changed."; + initAlignmentAlgorithm(setup, true); + } + + theAlignmentAlgo->beginRun(run, setup, update); } // ---------------------------------------------------------------------------- @@ -572,10 +437,124 @@ void AlignmentProducer::endLuminosityBlock(const edm::LuminosityBlock &lumiBlock theAlignmentAlgo->endLuminosityBlock(setup); // do not forward edm::LuminosityBlock } -// ---------------------------------------------------------------------------- -void AlignmentProducer::simpleMisalignment_(const Alignables &alivec, const std::string &selection, - float shift, float rot, bool local) +//------------------------------------------------------------------------------ +void +AlignmentProducer::createAlignmentAlgorithm(const edm::ParameterSet& config) +{ + auto algoConfig = config.getParameter("algoConfig"); + auto iovSelection = config.getParameter("RunRangeSelection"); + algoConfig.addUntrackedParameter("RunRangeSelection", + iovSelection); + algoConfig.addUntrackedParameter("firstIOV", + uniqueRunRanges_.front().first); + algoConfig.addUntrackedParameter("enableAlignableUpdates", + enableAlignableUpdates_); + + const auto algoName = algoConfig.getParameter("algoName"); + theAlignmentAlgo = + AlignmentAlgorithmPluginFactory::get()->create(algoName, algoConfig); + + if (!theAlignmentAlgo) { + throw cms::Exception("BadConfig") + << "Couldn't find the called alignment algorithm: " << algoName; + } +} + + +//------------------------------------------------------------------------------ +bool +AlignmentProducer::setupChanged(const edm::EventSetup& setup) +{ + bool changed{false}; + + if (watchIdealGeometryRcd.check(setup)) { + changed = true; + } + + if (watchGlobalPositionRcd.check(setup)) { + changed = true; + } + + if (doTracker_) { + if (watchTrackerAlRcd.check(setup)) { + changed = true; + } + + if (watchTrackerAlErrorExtRcd.check(setup)) { + changed = true; + } + + if (watchTrackerSurDeRcd.check(setup)) { + changed = true; + } + } + + if (doMuon_) { + if (watchDTAlRcd.check(setup)) { + changed = true; + } + + if (watchDTAlErrExtRcd.check(setup)) { + changed = true; + } + + if (watchCSCAlRcd.check(setup)) { + changed = true; + } + + if (watchCSCAlErrExtRcd.check(setup)) { + changed = true; + } + } + + /* TODO: ExtraAlignables: Which record(s) to check? + * + if (useExtras_) {} + */ + + return changed; +} + + +//------------------------------------------------------------------------------ +void +AlignmentProducer::initAlignmentAlgorithm(const edm::EventSetup& setup, + bool update) +{ + // Retrieve tracker topology from geometry + edm::ESHandle tTopoHandle; + setup.get().get(tTopoHandle); + const TrackerTopology* const tTopo = tTopoHandle.product(); + + // Create the geometries from the ideal geometries + createGeometries(setup, tTopo); + + applyAlignmentsToDB(setup); + createAlignables(tTopo, update); + buildParameterStore(); + applyMisalignment(); + + // Initialize alignment algorithm and integrated calibration and pass the + // latter to algorithm + edm::LogInfo("Alignment") + << "@SUB=AlignmentProducer::initAlignmentAlgorithm" + << "Initializing alignment algorithm."; + theAlignmentAlgo->initialize(setup, + theAlignableTracker, + theAlignableMuon, + theAlignableExtras, + theAlignmentParameterStore); + + applyAlignmentsToGeometry(); +} + + +// ---------------------------------------------------------------------------- +void +AlignmentProducer::simpleMisalignment(const Alignables &alivec, + const std::string &selection, + float shift, float rot, bool local) { std::ostringstream output; // collecting output @@ -642,12 +621,11 @@ void AlignmentProducer::simpleMisalignment_(const Alignables &alivec, const std: } -//__________________________________________________________________________________________________ -void AlignmentProducer::createGeometries_( const edm::EventSetup& iSetup ) +//------------------------------------------------------------------------------ +void +AlignmentProducer::createGeometries(const edm::EventSetup& iSetup, + const TrackerTopology* tTopo) { - edm::ESTransientHandle cpv; - iSetup.get().get( cpv ); - if (doTracker_) { edm::ESHandle geometricDet; iSetup.get().get( geometricDet ); @@ -655,15 +633,15 @@ void AlignmentProducer::createGeometries_( const edm::EventSetup& iSetup ) edm::ESHandle ptp; iSetup.get().get( ptp ); - edm::ESHandle tTopoHandle; - iSetup.get().get(tTopoHandle); - const TrackerTopology* const tTopo = tTopoHandle.product(); - TrackerGeomBuilderFromGeometricDet trackerBuilder; - theTracker = std::shared_ptr( trackerBuilder.build(&(*geometricDet), *ptp, tTopo )); + + theTracker = std::shared_ptr + (trackerBuilder.build(&(*geometricDet), *ptp, tTopo)); } if (doMuon_) { + edm::ESTransientHandle cpv; + iSetup.get().get(cpv); edm::ESHandle mdc; iSetup.get().get(mdc); DTGeometryBuilderFromDDD DTGeometryBuilder; @@ -675,6 +653,195 @@ void AlignmentProducer::createGeometries_( const edm::EventSetup& iSetup ) } } + +//------------------------------------------------------------------------------ +void +AlignmentProducer::applyAlignmentsToDB(const edm::EventSetup& setup) +{ + // Retrieve and apply alignments, if requested (requires z setup) + if (applyDbAlignment_) { + // we need GlobalPositionRcd - and have to keep track for later removal + // before writing again to DB... + + edm::ESHandle globalAlignments; + setup.get().get(globalAlignments); + globalPositions_ = new Alignments(*globalAlignments); + + if (doTracker_) { + applyDB + (&(*theTracker), setup, + align::DetectorGlobalPosition(*globalPositions_, DetId(DetId::Tracker)) + ); + + applyDB(&(*theTracker), setup); + } + + if (doMuon_) { + applyDB + (&(*theMuonDT), setup, + align::DetectorGlobalPosition(*globalPositions_, DetId(DetId::Muon)) + ); + + applyDB + (&(*theMuonCSC), setup, + align::DetectorGlobalPosition(*globalPositions_, DetId(DetId::Muon)) + ); + } + } +} + + + +//------------------------------------------------------------------------------ +void +AlignmentProducer::createAlignables(const TrackerTopology* tTopo, bool update) +{ + if (doTracker_) { + if (update) { + theAlignableTracker->update(&(*theTracker), tTopo); + } else { + theAlignableTracker = new AlignableTracker(&(*theTracker), tTopo); + } + } + + if (doMuon_) { + if (update) { + theAlignableMuon->update(&(*theMuonDT), &(*theMuonCSC)); + } else { + theAlignableMuon = new AlignableMuon(&(*theMuonDT), &(*theMuonCSC)); + } + } + + if (useExtras_) { + if (update) { + // FIXME: Requires further code changes to track beam spot condition changes + } else { + theAlignableExtras = new AlignableExtras(); + } + } +} + + +//------------------------------------------------------------------------------ +void +AlignmentProducer::buildParameterStore() +{ + // Create alignment parameter builder + edm::LogInfo("Alignment") << "@SUB=AlignmentProducer::buildParameterStore" + << "Creating AlignmentParameterBuilder"; + + auto alParamBuildCfg = + theParameterSet.getParameter("ParameterBuilder"); + auto alParamStoreCfg = + theParameterSet.getParameter("ParameterStore"); + + AlignmentParameterBuilder alignmentParameterBuilder(theAlignableTracker, + theAlignableMuon, + theAlignableExtras, + alParamBuildCfg); + + // Fix alignables if requested + if (stNFixAlignables_ > 0) { + alignmentParameterBuilder.fixAlignables(stNFixAlignables_); + } + + // Get list of alignables + auto theAlignables = alignmentParameterBuilder.alignables(); + edm::LogInfo("Alignment") << "@SUB=AlignmentProducer::buildParameterStore" + << "got " << theAlignables.size() << " alignables"; + + // Create AlignmentParameterStore + theAlignmentParameterStore = + new AlignmentParameterStore(theAlignables, alParamStoreCfg); + edm::LogInfo("Alignment") << "@SUB=AlignmentProducer::buildParameterStore" + << "AlignmentParameterStore created!"; +} + +//------------------------------------------------------------------------------ +void +AlignmentProducer::applyMisalignment() +{ + // Apply misalignment scenario to alignable tracker and muon if requested + // WARNING: this assumes scenarioConfig can be passed to both muon and tracker + + if (doMisalignmentScenario_ && (doTracker_ || doMuon_)) { + edm::LogInfo("Alignment") << "@SUB=AlignmentProducer::applyMisalignment" + << "Applying misalignment scenario to " + << (doTracker_ ? "tracker" : "") + << (doMuon_ ? (doTracker_ ? " and muon" : "muon") : "."); + auto scenarioConfig = + theParameterSet.getParameter("MisalignmentScenario"); + + if (doTracker_) { + TrackerScenarioBuilder scenarioBuilder(theAlignableTracker); + scenarioBuilder.applyScenario(scenarioConfig); + } + if (doMuon_) { + MuonScenarioBuilder muonScenarioBuilder(theAlignableMuon); + muonScenarioBuilder.applyScenario(scenarioConfig); + } + + } else { + edm::LogInfo("Alignment") << "@SUB=AlignmentProducer::applyMisalignment" + << "NOT applying misalignment scenario!"; + } + + // Apply simple misalignment + const auto sParSel = + theParameterSet.getParameter("parameterSelectorSimple"); + simpleMisalignment(theAlignmentParameterStore->alignables(), sParSel, + stRandomShift_, stRandomRotation_, true); +} + + + +//------------------------------------------------------------------------------ +void +AlignmentProducer::applyAlignmentsToGeometry() +{ + edm::LogInfo("Alignment") + << "@SUB=AlignmentProducer::applyAlignmentsToGeometry" + << "Now physically apply alignments to geometry..."; + + // Propagate changes to reconstruction geometry (from initialisation or iteration) + GeometryAligner aligner; + + if (doTracker_) { + std::unique_ptr alignments(theAlignableTracker->alignments()); + std::unique_ptr + alignmentErrExt(theAlignableTracker->alignmentErrors()); + std::unique_ptr + aliDeforms(theAlignableTracker->surfaceDeformations()); + + aligner.applyAlignments(&(*theTracker), alignments.get(), + alignmentErrExt.get(), AlignTransform()); + aligner.attachSurfaceDeformations(&(*theTracker), aliDeforms.get()); + } + + if (doMuon_) { + std::unique_ptr dtAlignments(theAlignableMuon->dtAlignments()); + std::unique_ptr cscAlignments(theAlignableMuon->cscAlignments()); + + std::unique_ptr + dtAlignmentErrExt (theAlignableMuon->dtAlignmentErrorsExtended()); + std::unique_ptr + cscAlignmentErrExt(theAlignableMuon->cscAlignmentErrorsExtended()); + + aligner.applyAlignments(&(*theMuonDT), &(*dtAlignments), + &(*dtAlignmentErrExt), AlignTransform()); + aligner.applyAlignments(&(*theMuonCSC), &(*cscAlignments), + &(*cscAlignmentErrExt), AlignTransform()); + } +} + + void AlignmentProducer::addSurveyInfo_(Alignable* ali) { const std::vector& comp = ali->components(); @@ -966,70 +1133,5 @@ void AlignmentProducer::writeDB(AlignmentSurfaceDeformations *alignmentSurfaceDe } } -AlignmentProducer::RunRanges -AlignmentProducer::makeNonOverlappingRunRanges(const edm::VParameterSet& RunRangeSelectionVPSet) -{ - static bool oldRunRangeSelectionWarning = false; - - const RunNumber beginValue = cond::timeTypeSpecs[cond::runnumber].beginValue; - const RunNumber endValue = cond::timeTypeSpecs[cond::runnumber].endValue; - - RunRanges uniqueRunRanges; - if (!RunRangeSelectionVPSet.empty()) { - - std::map uniqueFirstRunNumbers; - - for (std::vector::const_iterator ipset = RunRangeSelectionVPSet.begin(); - ipset != RunRangeSelectionVPSet.end(); - ++ipset) { - const std::vector RunRangeStrings = (*ipset).getParameter >("RunRanges"); - for (std::vector::const_iterator irange = RunRangeStrings.begin(); - irange != RunRangeStrings.end(); - ++irange) { - - if ((*irange).find(':')==std::string::npos) { - - RunNumber first = beginValue; - long int temp = strtol((*irange).c_str(), 0, 0); - if (temp!=-1) first = temp; - uniqueFirstRunNumbers[first] = first; - - } else { - - if (!oldRunRangeSelectionWarning) { - edm::LogWarning("BadConfig") << "@SUB=AlignmentProducer::makeNonOverlappingRunRanges" - << "Config file contains old format for 'RunRangeSelection'. Only the start run\n" - << "number is used internally. The number of the last run is ignored and can be\n" - << "safely removed from the config file.\n"; - oldRunRangeSelectionWarning = true; - } - - std::vector tokens = edm::tokenize(*irange, ":"); - long int temp; - RunNumber first = beginValue; - temp = strtol(tokens[0].c_str(), 0, 0); - if (temp!=-1) first = temp; - uniqueFirstRunNumbers[first] = first; - } - } - } - - for (std::map::iterator iFirst = uniqueFirstRunNumbers.begin(); - iFirst!=uniqueFirstRunNumbers.end(); - ++iFirst) { - uniqueRunRanges.push_back(std::pair((*iFirst).first, endValue)); - } - for (unsigned int i = 0;i(beginValue, endValue)); - - } - - return uniqueRunRanges; -} DEFINE_FWK_LOOPER( AlignmentProducer ); diff --git a/Alignment/CommonAlignmentProducer/plugins/AlignmentProducer.h b/Alignment/CommonAlignmentProducer/plugins/AlignmentProducer.h index 3ff8354a4d276..e60194f0e5782 100644 --- a/Alignment/CommonAlignmentProducer/plugins/AlignmentProducer.h +++ b/Alignment/CommonAlignmentProducer/plugins/AlignmentProducer.h @@ -25,6 +25,7 @@ #include "Geometry/Records/interface/TrackerDigiGeometryRecord.h" // Alignment +#include "Alignment/CommonAlignment/interface/Utilities.h" #include "Alignment/CommonAlignmentAlgorithm/interface/AlignmentAlgorithmBase.h" #include "Alignment/CommonAlignmentMonitor/interface/AlignmentMonitorBase.h" #include "FWCore/ParameterSet/interface/ParameterSet.h" @@ -57,13 +58,13 @@ class AlignmentProducer : public edm::ESProducerLooper { public: - typedef std::vector Alignables; + using Alignables = align::Alignables; typedef std::pair ConstTrajTrackPair; typedef std::vector ConstTrajTrackPairCollection; - typedef AlignmentAlgorithmBase::RunNumber RunNumber; - typedef AlignmentAlgorithmBase::RunRange RunRange; - typedef std::vector RunRanges; + using RunNumber = align::RunNumber; + using RunRange = align::RunRange; + using RunRanges = align::RunRanges; /// Constructor AlignmentProducer( const edm::ParameterSet& iConfig ); @@ -108,12 +109,40 @@ class AlignmentProducer : public edm::ESProducerLooper // private member functions + /// Creates the choosen alignment algorithm (specified in config-file) + void createAlignmentAlgorithm(const edm::ParameterSet&); + + /// Checks if one of the EventSetup-Records has changed + bool setupChanged(const edm::EventSetup&); + + /// Creates Geometry and Alignables of the Tracker and initializes the + /// AlignmentAlgorithm @theAlignmentAlgo + void initAlignmentAlgorithm(const edm::EventSetup&, bool update = false); + + /// Applies Alignments from Database (GlobalPositionRcd) to Geometry + /// @theTracker + void applyAlignmentsToDB(const edm::EventSetup&); + + /// Creates Alignables @theAlignableTracker from the previously loaded + /// Geometry @theTracker + void createAlignables(const TrackerTopology*, bool update = false); + + /// Creates the @theAlignmentParameterStore, which manages all Alignables + void buildParameterStore(); + + /// Applies misalignment scenario to @theAlignableTracker + void applyMisalignment(); + /// Apply random shifts and rotations to selected alignables, according to configuration - void simpleMisalignment_(const Alignables &alivec, const std::string &selection, + void simpleMisalignment(const Alignables &alivec, const std::string &selection, float shift, float rot, bool local); /// Create tracker and muon geometries - void createGeometries_( const edm::EventSetup& ); + void createGeometries(const edm::EventSetup&, const TrackerTopology*); + + /// Applies Alignments, AlignmentErrors and SurfaceDeformations to + /// @theTracker + void applyAlignmentsToGeometry(); /// Apply DB constants belonging to (Err)Rcd to geometry, /// taking into account 'globalPosition' correction. @@ -147,7 +176,6 @@ class AlignmentProducer : public edm::ESProducerLooper /// read in survey records void readInSurveyRcds( const edm::EventSetup& ); - RunRanges makeNonOverlappingRunRanges(const edm::VParameterSet& RunRangeSelectionVPSet); // private data members @@ -170,6 +198,7 @@ class AlignmentProducer : public edm::ESProducerLooper /// GlobalPositions that might be read from DB, NULL otherwise const Alignments *globalPositions_; + const RunRanges uniqueRunRanges_; int nevent_; edm::ParameterSet theParameterSet; @@ -184,6 +213,7 @@ class AlignmentProducer : public edm::ESProducerLooper const bool saveToDB_, saveApeToDB_,saveDeformationsToDB_; const bool doTracker_,doMuon_,useExtras_; const bool useSurvey_; // true to read survey info from DB + const bool enableAlignableUpdates_; // event input tags const edm::InputTag tjTkAssociationMapTag_; // map with tracks/trajectories @@ -192,6 +222,19 @@ class AlignmentProducer : public edm::ESProducerLooper const edm::InputTag clusterValueMapTag_; // ValueMap containing associtaion cluster - flag // ESWatcher + + edm::ESWatcher watchIdealGeometryRcd; + edm::ESWatcher watchGlobalPositionRcd; + + edm::ESWatcher watchTrackerAlRcd; + edm::ESWatcher watchTrackerAlErrorExtRcd; + edm::ESWatcher watchTrackerSurDeRcd; + + edm::ESWatcher watchDTAlRcd; + edm::ESWatcher watchDTAlErrExtRcd; + edm::ESWatcher watchCSCAlRcd; + edm::ESWatcher watchCSCAlErrExtRcd; + edm::ESWatcher watchTkSurveyRcd_; edm::ESWatcher watchTkSurveyErrRcd_; edm::ESWatcher watchDTSurveyRcd_; diff --git a/Alignment/CommonAlignmentProducer/plugins/PCLTrackerAlProducer.cc b/Alignment/CommonAlignmentProducer/plugins/PCLTrackerAlProducer.cc index a9b3460f37e22..cd05e80e11b42 100644 --- a/Alignment/CommonAlignmentProducer/plugins/PCLTrackerAlProducer.cc +++ b/Alignment/CommonAlignmentProducer/plugins/PCLTrackerAlProducer.cc @@ -20,7 +20,6 @@ #include "FWCore/Framework/interface/Run.h" #include "FWCore/MessageLogger/interface/MessageLogger.h" -#include "FWCore/Utilities/interface/Parse.h" #include "FWCore/ServiceRegistry/interface/Service.h" #include "FWCore/Utilities/interface/EDGetToken.h" #include "CondCore/DBOutputService/interface/PoolDBOutputService.h" @@ -82,6 +81,7 @@ ::PCLTrackerAlProducer(const edm::ParameterSet& config) : doMuon_ (config.getUntrackedParameter("doMuon") ), useExtras_ (config.getUntrackedParameter("useExtras")), useSurvey_ (config.getParameter ("useSurvey")), + enableAlignableUpdates_ (config.getParameter ("enableAlignableUpdates")), /* Event input tags */ tjTkAssociationMapTag_ (config.getParameter("tjTkAssociationMapTag")), @@ -176,14 +176,14 @@ ::endJob() void PCLTrackerAlProducer ::beginRun(const edm::Run& run, const edm::EventSetup& setup) { - if (setupChanged(setup)) { + const bool changed{setupChanged(setup)}; + if (changed) { edm::LogInfo("Alignment") << "@SUB=PCLTrackerAlProducer::beginRun" << "EventSetup-Record changed."; initAlignmentAlgorithm(setup); } - // Do not forward edm::Run - theAlignmentAlgo->beginRun(setup); + theAlignmentAlgo->beginRun(run, setup, changed); if (setupChanged(setup)) { initAlignmentAlgorithm(setup); @@ -322,6 +322,11 @@ ::createAlignmentAlgorithm(const edm::ParameterSet& config) edm::VParameterSet iovSelection = config.getParameter("RunRangeSelection"); algoConfig.addUntrackedParameter("RunRangeSelection", iovSelection); + // provide required parameters while keeping current functionality for PCL: + algoConfig.addUntrackedParameter("firstIOV", 1); + algoConfig.addUntrackedParameter("enableAlignableUpdates", + enableAlignableUpdates_); + std::string algoName = algoConfig.getParameter("algoName"); theAlignmentAlgo = AlignmentAlgorithmPluginFactory::get()->create(algoName, algoConfig); @@ -1004,15 +1009,10 @@ ::storeAlignmentsToDB() << "do not dare to store to DB."; } else { // Expand run ranges and make them unique - edm::VParameterSet runRangeSelectionVPSet(theParameterSet.getParameter("RunRangeSelection")); - RunRanges uniqueRunRanges(makeNonOverlappingRunRanges(runRangeSelectionVPSet)); - - // create dummy IOV - if (uniqueRunRanges.empty()) { - const RunRange runRange(cond::timeTypeSpecs[cond::runnumber].beginValue, - cond::timeTypeSpecs[cond::runnumber].endValue); - uniqueRunRanges.push_back(runRange); - } + const auto runRangeSelectionVPSet = + theParameterSet.getParameter("RunRangeSelection"); + align::RunRanges uniqueRunRanges + (align::makeUniqueRunRanges(runRangeSelectionVPSet, theFirstRun)); std::vector beamSpotParameters; @@ -1056,72 +1056,6 @@ ::storeAlignmentsToDB() } } -//_____________________________________________________________________________ -RunRanges PCLTrackerAlProducer -::makeNonOverlappingRunRanges(const edm::VParameterSet& RunRangeSelectionVPSet) -{ - static bool oldRunRangeSelectionWarning = false; - - const RunNumber beginValue = cond::timeTypeSpecs[cond::runnumber].beginValue; - const RunNumber endValue = cond::timeTypeSpecs[cond::runnumber].endValue; - - RunRanges uniqueRunRanges; - if (!RunRangeSelectionVPSet.empty()) { - - std::map uniqueFirstRunNumbers; - - for (auto ipset = RunRangeSelectionVPSet.begin(); - ipset != RunRangeSelectionVPSet.end(); - ++ipset) { - const std::vector RunRangeStrings = (*ipset).getParameter >("RunRanges"); - - for (auto irange = RunRangeStrings.begin(); - irange != RunRangeStrings.end(); - ++irange) { - - if ((*irange).find(':') == std::string::npos) { - - RunNumber first = beginValue; - long int temp = strtol((*irange).c_str(), 0, 0); - if (temp!=-1) first = temp; - uniqueFirstRunNumbers[first] = first; - - } else { - if (!oldRunRangeSelectionWarning) { - edm::LogWarning("BadConfig") << "@SUB=PCLTrackerAlProducer::makeNonOverlappingRunRanges" - << "Config file contains old format for 'RunRangeSelection'. Only the start run\n" - << "number is used internally. The number of the last run is ignored and can be\n" - << "safely removed from the config file.\n"; - oldRunRangeSelectionWarning = true; - } - - std::vector tokens = edm::tokenize(*irange, ":"); - long int temp; - RunNumber first = beginValue; - temp = strtol(tokens[0].c_str(), 0, 0); - if (temp!=-1) first = temp; - uniqueFirstRunNumbers[first] = first; - } - } - } - - for (auto iFirst = uniqueFirstRunNumbers.begin(); - iFirst != uniqueFirstRunNumbers.end(); - ++iFirst) { - uniqueRunRanges.push_back(std::pair((*iFirst).first, endValue)); - } - - for (size_t i = 0; i < uniqueRunRanges.size()-1; ++i) { - uniqueRunRanges[i].second = uniqueRunRanges[i+1].first - 1; - } - - } else { - uniqueRunRanges.push_back(std::pair(theFirstRun, endValue)); - } - - return uniqueRunRanges; -} - //_____________________________________________________________________________ void PCLTrackerAlProducer ::writeForRunRange(cond::Time_t time) diff --git a/Alignment/CommonAlignmentProducer/plugins/PCLTrackerAlProducer.h b/Alignment/CommonAlignmentProducer/plugins/PCLTrackerAlProducer.h index 37b1faa40c8eb..a46e558b5db67 100644 --- a/Alignment/CommonAlignmentProducer/plugins/PCLTrackerAlProducer.h +++ b/Alignment/CommonAlignmentProducer/plugins/PCLTrackerAlProducer.h @@ -44,6 +44,7 @@ #include "Alignment/CommonAlignment/interface/Alignable.h" #include "Alignment/CommonAlignment/interface/AlignableExtras.h" +#include "Alignment/CommonAlignment/interface/Utilities.h" #include "Alignment/TrackerAlignment/interface/AlignableTracker.h" #include "Alignment/MuonAlignment/interface/AlignableMuon.h" @@ -190,9 +191,6 @@ class PCLTrackerAlProducer : public edm::EDAnalyzer { /// Writes Alignments (i.e. Records) to database-file void storeAlignmentsToDB(); - /// Makes unique RunRanges (specified in config-file) - RunRanges makeNonOverlappingRunRanges(const edm::VParameterSet&); - /// Writes Alignments and AlignmentErrors for all sub detectors and the /// given run number void writeForRunRange(cond::Time_t); @@ -247,6 +245,7 @@ class PCLTrackerAlProducer : public edm::EDAnalyzer { const bool saveToDB_, saveApeToDB_, saveDeformationsToDB_; const bool doTracker_, doMuon_, useExtras_; const bool useSurvey_; + const bool enableAlignableUpdates_; /// Map with tracks/trajectories const edm::InputTag tjTkAssociationMapTag_; diff --git a/Alignment/CommonAlignmentProducer/python/AlignmentProducer_cff.py b/Alignment/CommonAlignmentProducer/python/AlignmentProducer_cff.py index 08cc7a2dfc9ac..12aacf2914579 100644 --- a/Alignment/CommonAlignmentProducer/python/AlignmentProducer_cff.py +++ b/Alignment/CommonAlignmentProducer/python/AlignmentProducer_cff.py @@ -70,7 +70,10 @@ # Save alignment to DB: true requires configuration of PoolDBOutputService - saveToDB = cms.bool(False), # save alignment? - saveApeToDB = cms.bool(False), # save APE? - saveDeformationsToDB = cms.bool(False) # save surface deformations (bows, etc.)? + saveToDB = cms.bool(False), # save alignment? + saveApeToDB = cms.bool(False), # save APE? + saveDeformationsToDB = cms.bool(False), # save surface deformations (bows, etc.)? + + # update alignables if triggered by corresponding input IOV boundary + enableAlignableUpdates = cms.bool(False), ) diff --git a/Alignment/CommonAlignmentProducer/python/TrackerAlignmentProducerForPCL_cff.py b/Alignment/CommonAlignmentProducer/python/TrackerAlignmentProducerForPCL_cff.py index 71be339ed5f8c..13ad93062c6b4 100644 --- a/Alignment/CommonAlignmentProducer/python/TrackerAlignmentProducerForPCL_cff.py +++ b/Alignment/CommonAlignmentProducer/python/TrackerAlignmentProducerForPCL_cff.py @@ -74,7 +74,10 @@ #trackerGeometryConstants = cms.PSet(trackerGeometryConstants_cfi.trackerGeometryConstants), # Save alignment to DB: true requires configuration of PoolDBOutputService - saveToDB = cms.bool(False), # save alignment? - saveApeToDB = cms.bool(False), # save APE? + saveToDB = cms.bool(False), # save alignment? + saveApeToDB = cms.bool(False), # save APE? saveDeformationsToDB = cms.bool(False), # save surface deformations (bows, etc.)? + + # update alignables if triggered by corresponding input IOV boundary + enableAlignableUpdates = cms.bool(False), ) diff --git a/Alignment/MillePedeAlignmentAlgorithm/interface/PedeLabelerBase.h b/Alignment/MillePedeAlignmentAlgorithm/interface/PedeLabelerBase.h index 44725a02b1fb1..9187c834f1da0 100644 --- a/Alignment/MillePedeAlignmentAlgorithm/interface/PedeLabelerBase.h +++ b/Alignment/MillePedeAlignmentAlgorithm/interface/PedeLabelerBase.h @@ -15,6 +15,7 @@ #include "FWCore/ParameterSet/interface/ParameterSet.h" #include "Alignment/CommonAlignment/interface/AlignableObjectId.h" +#include "Alignment/CommonAlignment/interface/Utilities.h" #include "Alignment/CommonAlignmentAlgorithm/interface/AlignmentAlgorithmBase.h" #include "TrackingTools/TrajectoryState/interface/TrajectoryStateOnSurface.h" @@ -34,9 +35,9 @@ class PedeLabelerBase { public: - typedef AlignmentAlgorithmBase::RunNumber RunNumber; - typedef AlignmentAlgorithmBase::RunRange RunRange; - typedef std::vector RunRanges; + using RunNumber = align::RunNumber; + using RunRange = align::RunRange; + using RunRanges = align::RunRanges; class TopLevelAlignables { diff --git a/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeAlignmentAlgorithm.cc b/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeAlignmentAlgorithm.cc index c956b64da6db2..bccd1cc3f55a2 100644 --- a/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeAlignmentAlgorithm.cc +++ b/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeAlignmentAlgorithm.cc @@ -12,6 +12,7 @@ #include "FWCore/Framework/interface/ESHandle.h" #include "FWCore/Framework/interface/EventSetup.h" +#include "FWCore/Framework/interface/Run.h" #include "FWCore/MessageLogger/interface/MessageLogger.h" #include "TrackingTools/PatternTools/interface/Trajectory.h" @@ -42,6 +43,7 @@ #include "Alignment/MuonAlignment/interface/AlignableMuon.h" #include "Alignment/CommonAlignment/interface/AlignableExtras.h" +#include "CondFormats/AlignmentRecord/interface/GlobalPositionRcd.h" #include "CondFormats/AlignmentRecord/interface/TrackerAlignmentRcd.h" #include "CondFormats/AlignmentRecord/interface/TrackerAlignmentErrorExtendedRcd.h" #include "CondFormats/AlignmentRecord/interface/TrackerSurfaceDeformationRcd.h" @@ -95,10 +97,19 @@ MillePedeAlignmentAlgorithm::MillePedeAlignmentAlgorithm(const edm::ParameterSet theAlignables(), theMinNumHits(cfg.getParameter("minNumHits")), theMaximalCor2D(cfg.getParameter("max2Dcorrelation")), + firstIOV_(cfg.getUntrackedParameter("firstIOV")), + ignoreFirstIOVCheck_(cfg.getUntrackedParameter("ignoreFirstIOVCheck")), + enableAlignableUpdates_(cfg.getUntrackedParameter("enableAlignableUpdates")), theLastWrittenIov(0), theGblDoubleBinary(cfg.getParameter("doubleBinary")), runAtPCL_(cfg.getParameter("runAtPCL")), - ignoreHitsWithoutGlobalDerivatives_(cfg.getParameter("ignoreHitsWithoutGlobalDerivatives")) + ignoreHitsWithoutGlobalDerivatives_(cfg.getParameter("ignoreHitsWithoutGlobalDerivatives")), + skipGlobalPositionRcdCheck_(cfg.getParameter("skipGlobalPositionRcdCheck")), + uniqueRunRanges_ + (align::makeUniqueRunRanges(cfg.getUntrackedParameter("RunRangeSelection"), + cond::timeTypeSpecs[cond::runnumber].beginValue)), + enforceSingleIOVInput_(!(enableAlignableUpdates_ && areIOVsSpecified())), + lastProcessedRun_(cond::timeTypeSpecs[cond::runnumber].beginValue) { if (!theDir.empty() && theDir.find_last_of('/') != theDir.size()-1) theDir += '/';// may need '/' edm::LogInfo("Alignment") << "@SUB=MillePedeAlignmentAlgorithm" << "Start in mode '" @@ -128,47 +139,53 @@ void MillePedeAlignmentAlgorithm::initialize(const edm::EventSetup &setup, << "Running with AlignabeMuon not yet tested."; } - // temporary fix to avoid corrupted database output - if (!runAtPCL_) { - const auto MIN_VAL{cond::timeTypeSpecs[cond::runnumber].beginValue}; - const auto MAX_VAL{cond::timeTypeSpecs[cond::runnumber].endValue}; + if (!runAtPCL_ && enforceSingleIOVInput_) { + const auto MIN_VAL = cond::timeTypeSpecs[cond::runnumber].beginValue; + const auto MAX_VAL = cond::timeTypeSpecs[cond::runnumber].endValue; const auto& iov_alignments = setup.get().validityInterval(); const auto& iov_surfaces = setup.get().validityInterval(); const auto& iov_errors = setup.get().validityInterval(); + + std::ostringstream message; + bool throwException{false}; if (iov_alignments.first().eventID().run() != MIN_VAL || - iov_alignments.last().eventID().run() != MAX_VAL) { - throw cms::Exception("DatabaseError") - << "@SUB=MillePedeAlignmentAlgorithm::initialize" - << "\nTrying to apply " << setup.get().key().name() - << " with multiple IOVs in tag.\n" - << "Validity range is " - << iov_alignments.first().eventID().run() << " - " - << iov_alignments.last().eventID().run(); + iov_alignments.last().eventID().run() != MAX_VAL) { + message + << "\nTrying to apply " << setup.get().key().name() + << " with multiple IOVs in tag without specifying 'RunRangeSelection'.\n" + << "Validity range is " + << iov_alignments.first().eventID().run() << " - " + << iov_alignments.last().eventID().run() << "\n"; + throwException = true; } if (iov_surfaces.first().eventID().run() != MIN_VAL || - iov_surfaces.last().eventID().run() != MAX_VAL) { - throw cms::Exception("DatabaseError") - << "@SUB=MillePedeAlignmentAlgorithm::initialize" - << "\nTrying to apply " - << setup.get().key().name() - << " with multiple IOVs in tag.\n" - << "Validity range is " - << iov_surfaces.first().eventID().run() << " - " - << iov_surfaces.last().eventID().run(); + iov_surfaces.last().eventID().run() != MAX_VAL) { + message + << "\nTrying to apply " + << setup.get().key().name() + << " with multiple IOVs in tag without specifying 'RunRangeSelection'.\n" + << "Validity range is " + << iov_surfaces.first().eventID().run() << " - " + << iov_surfaces.last().eventID().run() << "\n"; + throwException = true; } if (iov_errors.first().eventID().run() != MIN_VAL || - iov_errors.last().eventID().run() != MAX_VAL) { - throw cms::Exception("DatabaseError") - << "@SUB=MillePedeAlignmentAlgorithm::initialize" - << "\nTrying to apply " - << setup.get().key().name() - << " with multiple IOVs in tag.\n" - << "Validity range is " - << iov_errors.first().eventID().run() << " - " - << iov_errors.last().eventID().run(); + iov_errors.last().eventID().run() != MAX_VAL) { + message + << "\nTrying to apply " + << setup.get().key().name() + << " with multiple IOVs in tag without specifying 'RunRangeSelection'.\n" + << "Validity range is " + << iov_errors.first().eventID().run() << " - " + << iov_errors.last().eventID().run() << "\n"; + throwException = true; + } + if (throwException) { + throw cms::Exception("DatabaseError") + << "@SUB=MillePedeAlignmentAlgorithm::initialize" << message.str(); } } @@ -328,7 +345,11 @@ bool MillePedeAlignmentAlgorithm::setParametersForRunRange(const RunRange &runra { if (this->isMode(myPedeReadBit)) { // restore initial positions, rotations and deformations - theAlignmentParameterStore->restoreCachedTransformations(); + if (enableAlignableUpdates_) { + theAlignmentParameterStore->restoreCachedTransformations(runrange.first); + } else { + theAlignmentParameterStore->restoreCachedTransformations(); + } // Needed to shut up later warning from checkAliParams: theAlignmentParameterStore->resetParameters(); @@ -347,6 +368,7 @@ bool MillePedeAlignmentAlgorithm::setParametersForRunRange(const RunRange &runra return true; } + // Call at end of job --------------------------------------------------------- //____________________________________________________ void MillePedeAlignmentAlgorithm::terminate(const edm::EventSetup& iSetup) @@ -376,6 +398,26 @@ void MillePedeAlignmentAlgorithm::terminate() // cache all positions, rotations and deformations theAlignmentParameterStore->cacheTransformations(); + if (this->isMode(myPedeReadBit) && enableAlignableUpdates_) { + if (lastProcessedRun_ < uniqueRunRanges_.back().first) { + throw cms::Exception("BadConfig") + << "@SUB=MillePedeAlignmentAlgorithm::terminate\n" + << "Last IOV of 'RunRangeSelection' has not been processed. " + << "Please reconfigure your source to process the runs at least up to " + << uniqueRunRanges_.back().first << "."; + } + auto lastCachedRun = uniqueRunRanges_.front().first; + for (const auto& runRange: uniqueRunRanges_) { + const auto run = runRange.first; + if (std::find(cachedRuns_.begin(), cachedRuns_.end(), run) == cachedRuns_.end()) { + theAlignmentParameterStore->restoreCachedTransformations(lastCachedRun); + theAlignmentParameterStore->cacheTransformations(run); + } else { + lastCachedRun = run; + } + } + theAlignmentParameterStore->restoreCachedTransformations(); + } const std::string masterSteer(thePedeSteer->buildMasterSteer(files));// do only if myPedeSteerBit? if (this->isMode(myPedeRunBit)) { @@ -582,6 +624,96 @@ MillePedeAlignmentAlgorithm::addHitCount(const std::vector return nHitY; } + +void MillePedeAlignmentAlgorithm::beginRun(const edm::Run& run, + const edm::EventSetup& setup, + bool changed) { + if (run.run() < firstIOV_ && !ignoreFirstIOVCheck_) { + throw cms::Exception("Alignment") + << "@SUB=MillePedeAlignmentAlgorithm::beginRun\n" + << "Using data (run = " << run.run() + << ") prior to the first defined IOV (" << firstIOV_ << ")."; + } + + lastProcessedRun_ = run.run(); + + if (changed && enableAlignableUpdates_) { + const auto runNumber = run.run(); + auto firstRun = cond::timeTypeSpecs[cond::runnumber].beginValue; + for (auto runRange = uniqueRunRanges_.crbegin(); + runRange != uniqueRunRanges_.crend(); ++runRange) { + if (runNumber >= runRange->first) { + firstRun = runRange->first; + break; + } + } + if (std::find(cachedRuns_.begin(), cachedRuns_.end(), firstRun) + != cachedRuns_.end()) { + const auto& geometryRcd = setup.get(); + const auto& globalPosRcd = setup.get(); + const auto& alignmentRcd = setup.get(); + const auto& surfaceRcd = setup.get(); + const auto& errorRcd = setup.get(); + + std::ostringstream message; + bool throwException{false}; + message + << "Trying to cache tracker alignment payloads for a run (" << runNumber + << ") in an IOV (" << firstRun << ") that was already cached.\n" + << "The following records in your input database tag have an IOV " + << "boundary that does not match your IOV definition:\n"; + if (geometryRcd.validityInterval().first().eventID().run() > firstRun) { + message + << " - IdealGeometryRecord '" << geometryRcd.key().name() + << "' (since " + << geometryRcd.validityInterval().first().eventID().run() << ")\n"; + throwException = true; + } + if (globalPosRcd.validityInterval().first().eventID().run() > firstRun) { + message + << " - GlobalPositionRecord '" << globalPosRcd.key().name() + << "' (since " + << globalPosRcd.validityInterval().first().eventID().run() << ")"; + if (skipGlobalPositionRcdCheck_) { + message << " --> ignored\n"; + } else { + message << "\n"; + throwException = true; + } + } + if (alignmentRcd.validityInterval().first().eventID().run() > firstRun) { + message + << " - TrackerAlignmentRcd '" << alignmentRcd.key().name() + << "' (since " + << alignmentRcd.validityInterval().first().eventID().run() << ")\n"; + throwException = true; + } + if (surfaceRcd.validityInterval().first().eventID().run() > firstRun) { + message + << " - TrackerSurfaceDeformationRcd '" << surfaceRcd.key().name() + << "' (since " + << surfaceRcd.validityInterval().first().eventID().run() << ")\n"; + throwException = true; + } + if (errorRcd.validityInterval().first().eventID().run() > firstRun) { + message + << " - TrackerAlignmentErrorExtendedRcd '" << errorRcd.key().name() + << "' (since " + << errorRcd.validityInterval().first().eventID().run() << ")\n"; + throwException = true; + } + + if (throwException) { + throw cms::Exception("Alignment") + << "@SUB=MillePedeAlignmentAlgorithm::beginRun\n" << message.str(); + } + } else { + cachedRuns_.push_back(firstRun); + theAlignmentParameterStore->cacheTransformations(firstRun); + } + } +} + //____________________________________________________ void MillePedeAlignmentAlgorithm::endRun(const EventInfo &eventInfo, const EndRunInfo &runInfo, const edm::EventSetup &setup) @@ -1557,3 +1689,16 @@ void MillePedeAlignmentAlgorithm::addPxbSurvey(const edm::ParameterSet &pxbSurve outfile.close(); } + +bool MillePedeAlignmentAlgorithm::areIOVsSpecified() const { + const auto runRangeSelection = + theConfig.getUntrackedParameter("RunRangeSelection"); + + if (runRangeSelection.empty()) return false; + + const auto runRanges = + align::makeNonOverlappingRunRanges(runRangeSelection, + cond::timeTypeSpecs[cond::runnumber].beginValue); + + return !(runRanges.empty()); +} diff --git a/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeAlignmentAlgorithm.h b/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeAlignmentAlgorithm.h index 45d2b580840cd..88f3282c5e590 100644 --- a/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeAlignmentAlgorithm.h +++ b/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeAlignmentAlgorithm.h @@ -86,6 +86,11 @@ class MillePedeAlignmentAlgorithm : public AlignmentAlgorithmBase /// Run the algorithm on trajectories and tracks virtual void run(const edm::EventSetup &setup, const EventInfo &eventInfo) override; + /// called at begin of run + virtual void beginRun(const edm::Run& run, + const edm::EventSetup& setup, + bool changed) override; + // TODO: This method does NOT match endRun() in base class! Nobody is // calling this? /// Run on run products, e.g. TkLAS @@ -230,6 +235,9 @@ class MillePedeAlignmentAlgorithm : public AlignmentAlgorithmBase /// add measurement data from PXB survey void addPxbSurvey(const edm::ParameterSet &pxbSurveyCfg); + // + bool areIOVsSpecified() const; + //-------------------------------------------------------- // Data members //-------------------------------------------------------- @@ -257,6 +265,9 @@ class MillePedeAlignmentAlgorithm : public AlignmentAlgorithmBase unsigned int theMinNumHits; double theMaximalCor2D; /// maximal correlation allowed for 2D hit in TID/TEC. /// If larger, the 2D measurement gets diagonalized!!! + const align::RunNumber firstIOV_; + const bool ignoreFirstIOVCheck_; + const bool enableAlignableUpdates_; int theLastWrittenIov; // keeping track for output trees... std::vector theFloatBufferX; std::vector theFloatBufferY; @@ -268,6 +279,12 @@ class MillePedeAlignmentAlgorithm : public AlignmentAlgorithmBase const bool runAtPCL_; const bool ignoreHitsWithoutGlobalDerivatives_; + const bool skipGlobalPositionRcdCheck_; + + const align::RunRanges uniqueRunRanges_; + const bool enforceSingleIOVInput_; + std::vector cachedRuns_; + align::RunNumber lastProcessedRun_; }; DEFINE_EDM_PLUGIN(AlignmentAlgorithmPluginFactory, diff --git a/Alignment/MillePedeAlignmentAlgorithm/python/MillePedeAlignmentAlgorithm_cfi.py b/Alignment/MillePedeAlignmentAlgorithm/python/MillePedeAlignmentAlgorithm_cfi.py index 5f00655450b3b..e047d481d1995 100644 --- a/Alignment/MillePedeAlignmentAlgorithm/python/MillePedeAlignmentAlgorithm_cfi.py +++ b/Alignment/MillePedeAlignmentAlgorithm/python/MillePedeAlignmentAlgorithm_cfi.py @@ -23,10 +23,15 @@ monitorFile = cms.untracked.string('millePedeMonitor.root'), ## if empty: no monitoring... runAtPCL = cms.bool(False), # at the PCL the mille binaries are reset at lumi-section boundaries + ignoreFirstIOVCheck = cms.untracked.bool(False), # flag to ignore check if data is prior to first IOV ignoreHitsWithoutGlobalDerivatives = cms.bool(False), # - if all alignables and calibration for a # hit are set to '0', the hit is ignored # - has only an effect with non-GBL # material-effects description + skipGlobalPositionRcdCheck = cms.bool(False), # since the change of the GlobalPositionRcd is + # mostly driven by changes of the muon system + # it is often safe to ignore this change for + # tracker alignment # PSet that allows to configure the pede labeler, i.e. select the actual # labeler plugin to use and parameters for the selected plugin diff --git a/Alignment/MillePedeAlignmentAlgorithm/python/alignmentsetup/ConfigureAlignmentProducer.py b/Alignment/MillePedeAlignmentAlgorithm/python/alignmentsetup/ConfigureAlignmentProducer.py index ca0043eb5abc9..63fb027b87f84 100644 --- a/Alignment/MillePedeAlignmentAlgorithm/python/alignmentsetup/ConfigureAlignmentProducer.py +++ b/Alignment/MillePedeAlignmentAlgorithm/python/alignmentsetup/ConfigureAlignmentProducer.py @@ -15,11 +15,31 @@ def setConfiguration(process, collection, mode, monitorFile, binaryFile, # What tracks are used to construct the reference trajectories? process.AlignmentProducer.tjTkAssociationMapTag = "FinalTrackRefitter" + # enable proper handling of multi-IOV input + process.AlignmentProducer.enableAlignableUpdates = True + # Configure the algorithm process.AlignmentProducer.algoConfig = cms.PSet( process.MillePedeAlignmentAlgorithm) process.AlignmentProducer.algoConfig.mode = mode process.AlignmentProducer.algoConfig.mergeBinaryFiles = cms.vstring() + process.AlignmentProducer.algoConfig.skipGlobalPositionRcdCheck = True + + # default pede options: + process.AlignmentProducer.algoConfig.pedeSteerer.method = "sparseMINRES-QLP 3 0.8" + process.AlignmentProducer.algoConfig.minNumHits = 8 + process.AlignmentProducer.algoConfig.pedeSteerer.options = [ + "entries 50 10 2", + "outlierdownweighting 3", + "dwfractioncut 0.1", + "compress", + "threads 10 10", + "matiter 1", + "printcounts 2", + "chisqcut 30. 6.", + "bandwidth 6 1", + "monitorresiduals", + ] if mode == "mille": process.AlignmentProducer.algoConfig.binaryFile = binaryFile @@ -43,6 +63,15 @@ def setConfiguration(process, collection, mode, monitorFile, binaryFile, process.AlignmentProducer.algoConfig.TrajectoryFactory.MaterialEffects = "LocalGBL" # to account for multiple scattering in these layers process.AlignmentProducer.algoConfig.TrajectoryFactory.UseInvalidHits = True + elif collection == "ALCARECOTkAlUpsilonMuMu": + process.AlignmentProducer.algoConfig.TrajectoryFactory = cms.PSet( + process.TwoBodyDecayTrajectoryFactory + ) + process.AlignmentProducer.algoConfig.TrajectoryFactory.ParticleProperties.PrimaryMass = 9.4502 + process.AlignmentProducer.algoConfig.TrajectoryFactory.ParticleProperties.PrimaryWidth = 0.0644 + process.AlignmentProducer.algoConfig.TrajectoryFactory.MaterialEffects = "LocalGBL" + # to account for multiple scattering in these layers + process.AlignmentProducer.algoConfig.TrajectoryFactory.UseInvalidHits = True elif collection == "ALCARECOTkAlCosmicsCTF0T" and cosmicsZeroTesla: process.AlignmentProducer.algoConfig.TrajectoryFactory = cms.PSet( process.BrokenLinesBzeroTrajectoryFactory diff --git a/Alignment/MillePedeAlignmentAlgorithm/python/alignmentsetup/PedeSetup.py b/Alignment/MillePedeAlignmentAlgorithm/python/alignmentsetup/PedeSetup.py index 0e222cb6ebd6e..b214ff7a6f5a7 100644 --- a/Alignment/MillePedeAlignmentAlgorithm/python/alignmentsetup/PedeSetup.py +++ b/Alignment/MillePedeAlignmentAlgorithm/python/alignmentsetup/PedeSetup.py @@ -1,13 +1,15 @@ import FWCore.ParameterSet.Config as cms +import Alignment.MillePedeAlignmentAlgorithm.mpslib.tools as mps_tools -def setup(process, binary_files, tree_files): +def setup(process, binary_files, tree_files, run_start_geometry): """Pede-specific setup. Arguments: - `process`: cms.Process object - `binary_files`: list of binary files to be read by pede - `tree_files`: list of ROOT files created in the mille step + - `run_start_geometry`: run ID to pick the start geometry """ # write alignments, APEs, and surface deformations to DB by default @@ -57,7 +59,14 @@ def setup(process, binary_files, tree_files): # Set a new source and path. # -------------------------------------------------------------------------- - process.maxEvents = cms.untracked.PSet(input = cms.untracked.int32(1)) - process.source = cms.Source("EmptySource") + iovs = mps_tools.make_unique_runranges(process.AlignmentProducer) + number_of_events = iovs[-1] - iovs[0] + 1 + + process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(number_of_events)) + process.source = cms.Source( + "EmptySource", + firstRun = cms.untracked.uint32(run_start_geometry), + numberEventsInRun = cms.untracked.uint32(1)) process.dump = cms.EDAnalyzer("EventContentAnalyzer") process.p = cms.Path(process.dump) diff --git a/Alignment/MillePedeAlignmentAlgorithm/python/alignmentsetup/helper.py b/Alignment/MillePedeAlignmentAlgorithm/python/alignmentsetup/helper.py index dcca2ea3a70ed..af456cc973c24 100644 --- a/Alignment/MillePedeAlignmentAlgorithm/python/alignmentsetup/helper.py +++ b/Alignment/MillePedeAlignmentAlgorithm/python/alignmentsetup/helper.py @@ -19,3 +19,23 @@ def checked_out_MPS(): if e.args != (2, 'No such file or directory'): raise return checked_out, git_initialized + + +def set_pede_option(process, option): + """Utility function to set or override pede `option` defined in `process`. + + Arguments: + - `process`: cms.Process object + - `option`: option string + """ + + existing_options = process.AlignmentProducer.algoConfig.pedeSteerer.options + + exists = False + for i in xrange(len(existing_options)): + if existing_options[i].split()[0] == option.split()[0]: + existing_options[i] = option.strip() + exists = True + break + + if not exists: existing_options.append(option.strip()) diff --git a/Alignment/MillePedeAlignmentAlgorithm/python/mpslib/Mpslibclass.py b/Alignment/MillePedeAlignmentAlgorithm/python/mpslib/Mpslibclass.py index aad6a5bfaad8a..9fade8be2a0da 100755 --- a/Alignment/MillePedeAlignmentAlgorithm/python/mpslib/Mpslibclass.py +++ b/Alignment/MillePedeAlignmentAlgorithm/python/mpslib/Mpslibclass.py @@ -99,18 +99,18 @@ def read_db(self, db_file_name = __default_db_file_name): line = line.rstrip('\n') #removes the pesky \n from line parts = line.split(":") #read each line and split into parts list self.JOBNUMBER.append(int(parts[0])) - self.JOBDIR.append(parts[1]) + self.JOBDIR.append(parts[1].strip()) self.JOBID.append(int(parts[2])) - self.JOBSTATUS.append(parts[3]) + self.JOBSTATUS.append(parts[3].strip()) self.JOBNTRY.append(int(parts[4])) self.JOBRUNTIME.append(int(parts[5])) #int float? self.JOBNEVT.append(int(parts[6])) - self.JOBHOST.append(parts[7]) + self.JOBHOST.append(parts[7].strip()) self.JOBINCR.append(int(parts[8])) - self.JOBREMARK.append(parts[9]) - self.JOBSP1.append(parts[10]) - self.JOBSP2.append(parts[11]) - self.JOBSP3.append(parts[12]) + self.JOBREMARK.append(parts[9].strip()) + self.JOBSP1.append(parts[10].strip()) + self.JOBSP2.append(parts[11].strip()) + self.JOBSP3.append(parts[12].strip()) #count number of jobs if not self.JOBDIR[self.nJobs].startswith("jobm"): diff --git a/Alignment/MillePedeAlignmentAlgorithm/python/mpslib/tools.py b/Alignment/MillePedeAlignmentAlgorithm/python/mpslib/tools.py index 03fe177398c4f..8f0461840c96c 100644 --- a/Alignment/MillePedeAlignmentAlgorithm/python/mpslib/tools.py +++ b/Alignment/MillePedeAlignmentAlgorithm/python/mpslib/tools.py @@ -1,39 +1,31 @@ import os import sys +import importlib +import sqlalchemy import subprocess -import CondCore.Utilities.CondDBFW.shell as shell +import CondCore.Utilities.conddblib as conddb -def create_single_iov_db(global_tag, run_number, output_db): +def create_single_iov_db(inputs, run_number, output_db): """Create an sqlite file with single-IOV tags for alignment payloads. Arguments: - - `global_tag`: global tag from which to extract the payloads + - `inputs`: dictionary with input needed for payload extraction - `run_number`: run for which the IOVs are selected - `output_db`: name of the output sqlite file """ - con = shell.connect() - tags = con.global_tag_map(global_tag_name = global_tag, - record = ["TrackerAlignmentRcd", - "TrackerSurfaceDeformationRcd", - "TrackerAlignmentErrorExtendedRcd"]) - con.close_session() - - tags = {item["record"]: {"name": item["tag_name"]} - for item in tags.as_dicts()} - - for record,tag in tags.iteritems(): - iovs = con.tag(name = tag["name"]).iovs().as_dicts() + # find the IOV containing `run_number` + for record,tag in inputs.iteritems(): run_is_covered = False - for iov in reversed(iovs): - if iov["since"] <= run_number: - tag["since"] = str(iov["since"]) + for iov in reversed(tag["iovs"]): + if iov <= run_number: + tag["since"] = str(iov) run_is_covered = True break if not run_is_covered: msg = ("Run number {0:d} is not covered in '{1:s}' ({2:s}) from" - " '{3:s}'.".format(run_number, tag["name"], record, + " '{3:s}'.".format(run_number, tag["tag"], record, global_tag)) print msg print "Aborting..." @@ -41,18 +33,19 @@ def create_single_iov_db(global_tag, run_number, output_db): result = {} if os.path.exists(output_db): os.remove(output_db) - for record,tag in tags.iteritems(): + for record,tag in inputs.iteritems(): result[record] = {"connect": "sqlite_file:"+output_db, - "tag": "_".join([tag["name"], tag["since"]])} + "tag": "_".join([tag["tag"], tag["since"]])} cmd = ("conddb_import", "-f", "frontier://PromptProd/cms_conditions", "-c", result[record]["connect"], - "-i", tag["name"], + "-i", tag["tag"], "-t", result[record]["tag"], "-b", str(run_number), "-e", str(run_number)) run_checked(cmd) - run_checked(["sqlite3", output_db, "update iov set since=1"]) + if len(inputs) > 0: + run_checked(["sqlite3", output_db, "update iov set since=1"]) return result @@ -71,3 +64,94 @@ def run_checked(cmd): print "Problem in running the following command:" print " ".join(e.cmd) sys.exit(1) + + +def get_process_object(cfg): + """Returns cms.Process object defined in `cfg`. + + Arguments: + - `cfg`: path to CMSSW config file + """ + + sys.path.append(os.path.dirname(cfg)) # add location to python path + cache_stdout = sys.stdout + sys.stdout = open(os.devnull, "w") # suppress unwanted output + __configuration = \ + importlib.import_module(os.path.splitext(os.path.basename(cfg))[0]) + sys.stdout = cache_stdout + sys.path.pop() # clean up python path again + try: + os.remove(cfg+"c") # try to remove temporary .pyc file + except OSError as e: + if e.args == (2, "No such file or directory"): pass + else: raise + + return __configuration.process + + +def make_unique_runranges(ali_producer): + """Derive unique run ranges from AlignmentProducer PSet. + + Arguments: + - `ali_producer`: cms.PSet containing AlignmentProducer configuration + """ + + if (hasattr(ali_producer, "RunRangeSelection") and + len(ali_producer.RunRangeSelection) > 0): + iovs = set([int(iov) + for sel in ali_producer.RunRangeSelection + for iov in sel.RunRanges]) + if len(iovs) == 0: return [1] # single IOV starting from run 1 + return sorted(iovs) + else: + return [1] # single IOV starting from run 1 + + +def get_tags(global_tag, records): + """Get tags for `records` contained in `global_tag`. + + Arguments: + - `global_tag`: global tag of interest + - `records`: database records of interest + """ + + if len(records) == 0: return {} # avoid useless DB query + + # setting up the DB session + con = conddb.connect(url = conddb.make_url()) + session = con.session() + GlobalTagMap = session.get_dbtype(conddb.GlobalTagMap) + + # query tag names for records of interest contained in `global_tag` + tags = session.query(GlobalTagMap.record, GlobalTagMap.tag_name).\ + filter(GlobalTagMap.global_tag_name == global_tag, + GlobalTagMap.record.in_(records)).all() + + # closing the DB session + session.close() + + return {item[0]: {"tag": item[1], "connect": "pro"} for item in tags} + + +def get_iovs(db, tag): + """Retrieve the list of IOVs from `db` for `tag`. + + Arguments: + - `db`: database connection string + - `tag`: tag of database record + """ + + db = db.replace("sqlite_file:", "").replace("sqlite:", "") + + con = conddb.connect(url = conddb.make_url(db)) + session = con.session() + IOV = session.get_dbtype(conddb.IOV) + + iovs = set(session.query(IOV.since).filter(IOV.tag_name == tag).all()) + if len(iovs) == 0: + print "No IOVs found for tag '"+tag+"' in database '"+db+"'." + sys.exit(1) + + session.close() + + return sorted([int(item[0]) for item in iovs]) diff --git a/Alignment/MillePedeAlignmentAlgorithm/python/mpsvalidate/.gitignore b/Alignment/MillePedeAlignmentAlgorithm/python/mpsvalidate/.gitignore new file mode 100644 index 0000000000000..3acbf37c9056e --- /dev/null +++ b/Alignment/MillePedeAlignmentAlgorithm/python/mpsvalidate/.gitignore @@ -0,0 +1 @@ +TrackerTree.root diff --git a/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_alisetup.py b/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_alisetup.py index a2e2e8dbcbdb3..e3196fa37d863 100755 --- a/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_alisetup.py +++ b/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_alisetup.py @@ -6,13 +6,12 @@ ## Then calls mps_setup.pl for all datasets. ## ## Usage: -## mps_alisetup.py [-h] [-v] myconfig.ini +## mps_alisetup.py [-h] [-v] [-w] myconfig.ini ## import argparse import os import re -import importlib import subprocess import ConfigParser import sys @@ -76,35 +75,26 @@ def get_weight_configs(config): return configs -def create_input_db(cfg, run_number): +def create_input_db(cms_process, run_number): """ Create sqlite file with single-IOV tags and use it to override the GT. If the GT is already customized by the user, the customization has higher priority. Returns a snippet to be appended to the configuration file Arguments: - - `cfg`: path to python configuration + - `cms_process`: cms.Process object - `run_number`: run from which to extract the alignment payloads """ - sys.path.append(os.path.dirname(cfg)) - cache_stdout = sys.stdout - sys.stdout = open(os.devnull, "w") # suppress unwanted output - __configuration = \ - importlib.import_module(os.path.splitext(os.path.basename(cfg))[0]) - sys.stdout = cache_stdout - run_number = int(run_number) if not run_number > 0: print "'FirstRunForStartGeometry' must be positive, but is", run_number sys.exit(1) - global_tag = __configuration.process.GlobalTag.globaltag.value() input_db_name = os.path.abspath("alignment_input.db") - tags = mps_tools.create_single_iov_db(global_tag, run_number, input_db_name) - - for condition in __configuration.process.GlobalTag.toGet.value(): - if condition.record.value() in tags: del tags[condition.record.value()] + tags = mps_tools.create_single_iov_db( + check_iov_definition(cms_process, run_number), + run_number, input_db_name) result = "" for record,tag in tags.iteritems(): @@ -117,9 +107,115 @@ def create_input_db(cfg, run_number): " record = \""+record+"\",\n" " tag = \""+tag["tag"]+"\")\n") - os.remove(cfg+"c") return result + +def check_iov_definition(cms_process, first_run): + """ + Check consistency of input alignment payloads and IOV definition. + Returns a dictionary with the information needed to override possibly + problematic input taken from the global tag. + + Arguments: + - `cms_process`: cms.Process object containing the CMSSW configuration + - `first_run`: first run for start geometry + """ + + print "Checking consistency of IOV definition..." + iovs = mps_tools.make_unique_runranges(cms_process.AlignmentProducer) + + if first_run != iovs[0]: # simple consistency check + print "Value of 'FirstRunForStartGeometry' has to match first defined", + print "output IOV:", + print first_run, "!=", iovs[0] + sys.exit(1) + + + inputs = { + "TrackerAlignmentRcd": None, + "TrackerSurfaceDeformationRcd": None, + "TrackerAlignmentErrorExtendedRcd": None, + } + + for condition in cms_process.GlobalTag.toGet.value(): + if condition.record.value() in inputs: + inputs[condition.record.value()] = { + "tag": condition.tag.value(), + "connect": ("pro" + if not condition.hasParameter("connect") + else condition.connect.value()) + } + + inputs_from_gt = [record for record in inputs if inputs[record] is None] + inputs.update(mps_tools.get_tags(cms_process.GlobalTag.globaltag.value(), + inputs_from_gt)) + + for inp in inputs.itervalues(): + inp["iovs"] = mps_tools.get_iovs(inp["connect"], inp["tag"]) + + # check consistency of input with output + problematic_gt_inputs = {} + input_indices = {key: len(value["iovs"]) -1 + for key,value in inputs.iteritems()} + for iov in reversed(iovs): + for inp in inputs: + if inp in problematic_gt_inputs: continue + if input_indices[inp] < 0: + print "First output IOV boundary at run", iov, + print "is before the first input IOV boundary at", + print inputs[inp]["iovs"][0], "for '"+inp+"'." + print "Please check your run range selection." + sys.exit(1) + input_iov = inputs[inp]["iovs"][input_indices[inp]] + if iov < input_iov: + if inp in inputs_from_gt: + problematic_gt_inputs[inp] = inputs[inp] + print "Found problematic input taken from global tag." + print "Input IOV boundary at run",input_iov, + print "for '"+inp+"' is within output IOV starting with", + print "run", str(iov)+"." + print "Deriving an alignment with coarse IOV granularity", + print "starting from finer granularity leads to wrong", + print "results." + print "A single IOV input using the IOV of", + print "'FirstRunForStartGeometry' ("+str(first_run)+") is", + print "automatically created and used." + continue + print "Found input IOV boundary at run",input_iov, + print "for '"+inp+"' which is within output IOV starting with", + print "run", str(iov)+"." + print "Deriving an alignment with coarse IOV granularity", + print "starting from finer granularity leads to wrong results." + print "Please check your run range selection." + sys.exit(1) + elif iov == input_iov: + input_indices[inp] -= 1 + + # check consistency of 'TrackerAlignmentRcd' with other inputs + input_indices = {key: len(value["iovs"]) -1 + for key,value in inputs.iteritems() + if (key != "TrackerAlignmentRcd") + and (inp not in problematic_gt_inputs)} + for iov in reversed(inputs["TrackerAlignmentRcd"]["iovs"]): + for inp in input_indices: + input_iov = inputs[inp]["iovs"][input_indices[inp]] + if iov < input_iov: + print "Found input IOV boundary at run",input_iov, + print "for '"+inp+"' which is within 'TrackerAlignmentRcd'", + print "IOV starting with run", str(iov)+"." + print "Deriving an alignment with inconsistent IOV boundaries", + print "leads to wrong results." + print "Please check your input IOVs." + sys.exit(1) + elif iov == input_iov: + input_indices[inp] -= 1 + + print "IOV consistency check successful." + print "-"*60 + + return problematic_gt_inputs + + # ------------------------------------------------------------------------------ # set up argument parser and config parser @@ -246,6 +342,21 @@ def create_input_db(cfg, run_number): print "When using -w, a default configTemplate is needed to build a merge-config." raise SystemExit + try: + first_run = config.get("general", "FirstRunForStartGeometry") + except ConfigParser.NoOptionError: + print "Missing mandatory option 'FirstRunForStartGeometry' in [general] section." + raise SystemExit + + for section in config.sections(): + if section.startswith("dataset:"): + try: + collection = config.get(section, "collection") + break + except ConfigParser.NoOptionError: + print "Missing mandatory option 'collection' in section ["+section+"]." + raise SystemExit + try: with open(configTemplate,"r") as f: tmpFile = f.read() @@ -256,10 +367,20 @@ def create_input_db(cfg, run_number): tmpFile = re.sub('setupGlobaltag\s*\=\s*[\"\'](.*?)[\"\']', 'setupGlobaltag = \"'+globalTag+'\"', tmpFile) + tmpFile = re.sub('setupCollection\s*\=\s*[\"\'](.*?)[\"\']', + 'setupCollection = \"'+collection+'\"', + tmpFile) + tmpFile = re.sub(re.compile("setupRunStartGeometry\s*\=\s*.*$", re.M), + "setupRunStartGeometry = "+first_run, + tmpFile) thisCfgTemplate = "tmp.py" - with open(thisCfgTemplate, "w") as f: - f.write(tmpFile) + with open(thisCfgTemplate, "w") as f: f.write(tmpFile) + + cms_process = mps_tools.get_process_object(thisCfgTemplate) + + overrideGT = create_input_db(cms_process, first_run) + with open(thisCfgTemplate, "a") as f: f.write(overrideGT) for setting in pedesettings: print @@ -303,6 +424,16 @@ def create_input_db(cfg, run_number): # remove temporary file os.system("rm "+thisCfgTemplate) + + if overrideGT.strip() != "": + print "="*60 + msg = ("Overriding global tag with single-IOV tags extracted from '{}' " + "for run number '{}'.".format(generalOptions["globaltag"], + first_run)) + print msg + print "-"*60 + print overrideGT + sys.exit() @@ -410,6 +541,9 @@ def create_input_db(cfg, run_number): tmpFile = re.sub('setupGlobaltag\s*\=\s*[\"\'](.*?)[\"\']', 'setupGlobaltag = \"'+datasetOptions['globaltag']+'\"', tmpFile) + tmpFile = re.sub(re.compile("setupRunStartGeometry\s*\=\s*.*$", re.M), + "setupRunStartGeometry = "+ + generalOptions["FirstRunForStartGeometry"], tmpFile) tmpFile = re.sub('setupCollection\s*\=\s*[\"\'](.*?)[\"\']', 'setupCollection = \"'+datasetOptions['collection']+'\"', tmpFile) @@ -441,7 +575,8 @@ def create_input_db(cfg, run_number): append = '' firstDataset = False configTemplate = tmpFile - overrideGT = create_input_db(thisCfgTemplate, + cms_process = mps_tools.get_process_object(thisCfgTemplate) + overrideGT = create_input_db(cms_process, generalOptions["FirstRunForStartGeometry"]) with open(thisCfgTemplate, "a") as f: f.write(overrideGT) diff --git a/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_check.py b/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_check.py index e925ec09f59d2..6c6bdfd971a46 100755 --- a/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_check.py +++ b/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_check.py @@ -4,7 +4,7 @@ # -> check STDOUT files # -> check cmsRun.out # -> check alignment.log -# -> check if millebinaries are on eos +# -> check if millebinaries are on eos # -> check pede.dump # -> check millepede.log # -> check millepede.end @@ -28,395 +28,405 @@ # loop over FETCH jobs for i in xrange(len(lib.JOBID)): - # FIXME use bools? - batchSuccess = 0 - batchExited = 0 - finished = 0 - endofjob = 0 - eofile = 1 # do not deal with timel yet - timel = 0 - killed = 0 - ioprob = 0 - fw8001 = 0 - tooManyTracks = 0 - segviol = 0 - rfioerr = 0 - quota = 0 - nEvent = 0 - cputime = -1 - pedeAbend = 0 - pedeLogErr = 0 - pedeLogWrn = 0 - exceptionCaught = 0 - timeout = 0 - cfgerr = 0 - emptyDatErr = 0 - emptyDatOnFarm = 0 - cmdNotFound = 0 - insuffPriv = 0 - quotaspace = 0 - - pedeLogErrStr = "" - pedeLogWrnStr = "" - remark = "" - - disabled = ""; - if 'DISABLED' in lib.JOBSTATUS[i]: - disabled = 'DISABLED' - - if 'FETCH' in lib.JOBSTATUS[i]: - - # open the STDOUT file - stdOut = 'jobData/'+lib.JOBDIR[i]+'/STDOUT' - # unzip the STDOUT file if necessary - if os.access(stdOut+'.gz', os.R_OK): - os.system('gunzip '+stdOut+'.gz') - - STDFILE = open(stdOut,'r') - # scan records in input file. - # use regular expression to search. re.compile needed for options re.M and re.I - # re.M=re.MULTILINE enables matching of newline char - # re.I=re.IGNORECASE makes matching case-insensitive. - for line in STDFILE: - if re.search(re.compile('Unable to access quota space',re.M|re.I), line): - quotaspace = 1 - if re.search(re.compile('Unable to get quota space',re.M|re.I), line): - quotaspace = 1 - if re.search(re.compile('Disk quota exceeded',re.M|re.I), line): - quotaspace = 1 - if re.search(re.compile('CERN report: Job Killed',re.M), line): - killed = 1 - if re.search(re.compile('Job finished',re.M), line): - finished = 1 - if re.search(re.compile('connection timed out',re.M), line): - timeout = 1 - if re.search(re.compile('ConfigFileReadError',re.M), line): - cfgerr = 1 - if re.search(re.compile('0 bytes transferred',re.M), line): - emptyDatOnFarm = 1 - if re.search(re.compile('command not found',re.M), line): - cmdNotFound = 1 - # AP 26.11.2009 Insufficient privileges to rfcp files - if re.search(re.compile('stage_put: Insufficient user privileges',re.M), line): - insuffPriv = 1 - # AP 05.11.2015 Extract cpu-time. - # STDOUT doesn't contain NCU anymore. Now KSI2K and HS06 seconds are displayed. - # The ncuFactor is calculated from few samples by comparing KSI2K seconds with - # CPU time from email. - match = re.search(re.compile('This process used .+?(\d+) KSI2K seconds',re.M|re.I), line) - if match: - cpuFactor = 2.125 - cputime = int(round(int(match.group(1))/cpuFactor)) # match.group(1) is the matched digit - STDFILE.close() - - # gzip it afterwards: - print 'gzip -f '+stdOut - os.system('gzip -f '+stdOut) - - # GF: This file is not produced (anymore...) -> check for existence and read-access added - eazeLog = 'jobData/'+lib.JOBDIR[i]+'/cmsRun.out' - if os.access(eazeLog, os.R_OK): - # open the input file - INFILE = open(eazeLog,'r') - # scan records in input file - for line in INFILE: - # check if end of file has been reached - if re.search(re.compile('\',re.M), line): - eofile = 1 - if re.search(re.compile('Time limit reached\.',re.M), line): - timel = 1 - if re.search(re.compile('gives I\/O problem',re.M), line): - ioprob = 1 - if re.search(re.compile('FrameworkError ExitStatus=[\'\"]8001[\'\"]',re.M), line): - fw8001 = 1 - if re.search(re.compile('too many tracks',re.M), line): - tooManyTracks = 1 - if re.search(re.compile('segmentation violation',re.M), line): - segviol = 1 - if re.search(re.compile('failed RFIO error',re.M), line): - rfioerr = 1 - if re.search(re.compile('Request exceeds quota',re.M), line): - quota = 1 - INFILE.close() - - # if there is an alignment.log[.gz] file, check it as well - eazeLog = 'jobData/'+lib.JOBDIR[i]+'/alignment.log' - logZipped = 'no' - # unzip the logfile if necessary - if os.access(eazeLog+'.gz', os.R_OK): - os.system('gunzip '+eazeLog+'.gz') - logZipped = 'true' - - if os.access(eazeLog, os.R_OK): # access to alignment.log - # open the input file - INFILE = open(eazeLog,'r') - # scan records in input file - for line in INFILE: - # check if end of file has been reached - if re.search(re.compile('\',re.M), line): - eofile = 1 - if re.search(re.compile('EAZE\. Time limit reached\.',re.M), line): - timel = 1 - if re.search(re.compile('GAF gives I\/O problem',re.M), line): - ioprob = 1 - if re.search(re.compile('FrameworkError ExitStatus=[\'\"]8001[\'\"]',re.M), line): - fw8001 = 1 - if re.search(re.compile('too many tracks',re.M), line): - tooManyTracks = 1 - if re.search(re.compile('segmentation violation',re.M), line): - segviol = 1 - if re.search(re.compile('failed RFIO error',re.M), line): - rfioerr = 1 - if re.search(re.compile('Request exceeds quota',re.M), line): - quota = 1 - # check for newer (e.g. CMSSW_5_1_X) and older CMSSW: - if re.search(re.compile('Fatal Exception',re.M), line): - exceptionCaught = 1 - if re.search(re.compile('Exception caught in cmsRun',re.M), line): - exceptionCaught = 1 - # AP 07.09.2009 - Check that the job got to a normal end - if re.search(re.compile('AlignmentProducer::endOfJob\(\)',re.M), line): - endofjob = 1 - if re.search(re.compile('FwkReport -i main_input:sourc',re.M), line): - array = line.split() - nEvent = int(array[5]) - if nEvent==0 and re.search(re.compile('FwkReport -i PostSource',re.M), line): - array = line.split() - nEvent = int(array[5]) - # AP 31.07.2009 - To read number of events in CMSSW_3_2_2_patch2 - if nEvent==0 and re.search(re.compile('FwkReport -i AfterSource',re.M), line): - array = line.split() - nEvent = int(array[5]) - INFILE.close() - - if logZipped == 'true': - os.system('gzip '+eazeLog) - - else: # no access to alignment.log - print 'mps_check.py cannot find',eazeLog,'to test' - # AP 07.09.2009 - The following check cannot be done: set to 1 to avoid fake error type - endofjob = 1 - - # for mille jobs checks that milleBinary file is not empty - if i0): - emptyDatErr = 1 - - # merge jobs: additional checks for merging job - else: - # if there is a pede.dump file check it as well - eazeLog = 'jobData/'+lib.JOBDIR[i]+'/pede.dump' - if os.access(eazeLog+'.gz', os.R_OK): - # unzip - but clean before and save to tmp - os.system('rm -f /tmp/pede.dump') - os.system('gunzip -c '+eazeLog+'.gz > /tmp/pede.dump') - eazeLog = '/tmp/pede.dump' - if os.access(eazeLog, os.R_OK): - INFILE = open(eazeLog,'r') # open pede.dump - - # scan records in INFILE - pedeAbend = 1 - usedPedeMem = 0. - for line in INFILE: - # check if pede has reached its normal end - if re.search(re.compile('Millepede II.* ending',re.M), line): - pedeAbend = 0 - # extract memory usage - match = re.search(re.compile('Peak dynamic memory allocation: (.+) GB',re.I), line) - if match: - mem = match.group(1) - mem = re.sub('\s', '', mem) - # if mem is a float - if re.search(re.compile('^\d+\.\d+$',re.M), mem): - usedPedeMem = float(mem) - else: - print 'mps_check.py: Found Pede peak memory allocation but extracted number is not a float:',mem - - # check memory usage - # no point in asking if lib.pedeMem is defined. Initialized as lib.pedeMem=-1 - if lib.pedeMem > 0 and usedPedeMem > 0.: - memoryratio = usedPedeMem /(lib.pedeMem/1024.) - # print a warning if more than approx. 4 GB have been - # requested of which less than 75% are used by Pede - if lib.pedeMem > 4000 and memoryratio < 75. : - print 'Warning:',round(lib.pedeMem / 1024.,2),'GB of memory for Pede requested, but only',round(memoryratio,1),'\% of it has been used! Consider to request less memory in order to save resources.' - - # clean up /tmp/pede.dump if needed - if eazeLog == '/tmp/pede.dump': - os.system('rm /tmp/pede.dump') - - # pede.dump not found or no read-access - else: - print 'mps_check.py cannot find',eazeLog,'to test' - - # if there is a millepede.log file, check it as well - eazeLog = 'jobData/'+lib.JOBDIR[i]+'/millepede.log' - logZipped = 'no' - if os.access(eazeLog+'.gz', os.R_OK): - os.system('gunzip '+eazeLog+'.gz') - logZipped = 'true' - - if os.access(eazeLog, os.R_OK): - # open log file - INFILE = open(eazeLog,'r') - # scan records in input file - for line in INFILE: - # Cheks for Pede Errors - if re.search(re.compile('step no descending',re.M), line): - pedeLogErr = 1 - pedeLogErrStr += line - if re.search(re.compile('Constraint equation discrepancies:',re.M), line): - pedeLogErr = 1 - pedeLogErrStr += line - # AP 07.09.2009 - Checks for Pede Warnings: - if re.search(re.compile('insufficient constraint equations',re.M), line): - pedeLogWrn = 1 - pedeLogWrnStr += line - INFILE.close() - - if logZipped == 'true': - os.system('gzip '+eazeLog) - else: - print 'mps_check.py cannot find',eazeLog,'to test' - - - # check millepede.end -- added F. Meier 03.03.2015 - eazeLog = 'jobData/'+lib.JOBDIR[i]+'/millepede.end' - logZipped = 'no' - if os.access(eazeLog+'.gz', os.R_OK): - os.system('gunzip'+eazeLog+'.gz') - logZipped = 'true' - - if os.access(eazeLog, os.R_OK): - # open log file - INFILE = open(eazeLog,'r') - # scan records in input file - for line in INFILE: - # Checks for the output code. 0 is OK, 1 is WARN, anything else is FAIL - # searches the line for a number with or without a sign - match = re.search(re.compile('([-+]?\d+)',re.M), line) - if match: - if int(match.group(1)) == 1: - pedeLogWrn = 1 - pedeLogWrnStr += line - elif int(match.group(1)) != 0: - pedeLogErr = 1 - pedeLogErrStr += line - INFILE.close() - if logZipped == 'true': - os.system('gzip '+eazeLog) - else: - print 'mps_check.py cannot find',eazeLog,'to test' - - # end of merge job checks - # evaluate Errors: - farmhost = ' ' - - okStatus = 'OK' - if not eofile == 1: - print lib.JOBDIR[i],lib.JOBID[i],'did not reach end of file' - okStatus = 'ABEND' - if quotaspace == 1: - print lib.JOBDIR[i],lib.JOBID[i],'had quota space problem' - okStatus = 'FAIL' - remark = 'eos quota space problem' - if ioprob == 1: - print lib.JOBDIR[i],lib.JOBID[i],'had I/O problem' - okStatus = 'FAIL' - if fw8001 == 1: - print lib.JOBDIR[i],lib.JOBID[i],'had Framework error 8001 problem' - remark = 'fwk error 8001' - okStatus = 'FAIL' - if timeout == 1: - print lib.JOBDIR[i],lib.JOBID[i],'had connection timed out problem' - remark = 'connection timed out' - if cfgerr == 1: - print lib.JOBDIR[i],lib.JOBID[i],'had config file error' - remark = 'cfg file error' - okStatus = 'FAIL' - if killed == 1: - print lib.JOBDIR[i],lib.JOBID[i],'Job Killed (probably time exceeded)' - remark = "killed"; - okStatus = "FAIL" - if timel == 1: - print lib.JOBDIR[i],lib.JOBID[i],'ran into time limit' - okStatus = 'TIMEL' - if tooManyTracks == 1: - print lib.JOBDIR[i],lib.JOBID[i],'too many tracks' - if segviol == 1: - print lib.JOBDIR[i],lib.JOBID[i],'SEGVIOL encountered' - remark = 'seg viol' - okStatus = 'FAIL' - if rfioerr == 1: - print lib.JOBDIR[i],lib.JOBID[i],'RFIO error encountered' - remark = 'rfio error' - okStatus = 'FAIL' - if quota == 1: - print lib.JOBDIR[i],lib.JOBID[i],'Request exceeds quota' - if exceptionCaught == 1: - print lib.JOBDIR[i],lib.JOBID[i],'Exception caught in cmsrun' - remark = 'Exception caught' - okStatus = 'FAIL' - if emptyDatErr == 1: - print 'milleBinary???.dat file not found or empty' - remark = 'empty milleBinary' - if emptyDatOnFarm > 0: - print '...but already empty on farm so OK (or check job',i+1,'yourself...)' - else: - okStatus = 'FAIL' - if cmdNotFound == 1: - print lib.JOBDIR[i],lib.JOBID[i],'Command not found' - remark = 'cmd not found' - okStatus = 'FAIL' - if insuffPriv == 1: - print lib.JOBDIR[i],lib.JOBID[i],'Insufficient privileges to rfcp files' - remark = 'Could not rfcp files' - okStatus = 'FAIL' - if pedeAbend == 1: - print lib.JOBDIR[i],lib.JOBID[i],'Pede did not end normally' - remark = 'pede failed' - okStatus = 'FAIL' - if pedeLogErr == 1: - print lib.JOBDIR[i],lib.JOBID[i],'Problems in running Pede:' - print pedeLogErrStr - remark = 'pede error' - okStatus = 'FAIL' - if pedeLogWrn == 1: - # AP 07.09.2009 - Reports Pede Warnings (but do _not_ set job status to FAIL) - print lib.JOBDIR[i],lib.JOBID[i],'Warnings in running Pede:' - print pedeLogWrnStr - remark = 'pede warnings' - okStatus = 'WARN' - if endofjob != 1: - print lib.JOBDIR[i],lib.JOBID[i],'Job not ended' - remark = 'job not ended' - okStatus = 'FAIL' - - # print warning line to stdout - if okStatus != "OK": - print lib.JOBDIR[i],lib.JOBID[i],' -------- ',okStatus - - # update number of events - lib.JOBNEVT[i] = nEvent - # udate Jobstatus - lib.JOBSTATUS[i] = disabled+okStatus - # update cputime - print cputime - lib.JOBRUNTIME[i] = cputime - # update remark - lib.JOBREMARK[i] = remark - # update host - ##lib.JOBHOST[i] = farmhost + # FIXME use bools? + batchSuccess = 0 + batchExited = 0 + finished = 0 + endofjob = 0 + eofile = 1 # do not deal with timel yet + timel = 0 + killed = 0 + ioprob = 0 + fw8001 = 0 + tooManyTracks = 0 + segviol = 0 + rfioerr = 0 + quota = 0 + nEvent = 0 + cputime = -1 + pedeAbend = 0 + pedeLogErr = 0 + pedeLogWrn = 0 + exceptionCaught = 0 + timeout = 0 + cfgerr = 0 + emptyDatErr = 0 + emptyDatOnFarm = 0 + cmdNotFound = 0 + insuffPriv = 0 + quotaspace = 0 + + pedeLogErrStr = "" + pedeLogWrnStr = "" + remark = "" + + disabled = ""; + if 'DISABLED' in lib.JOBSTATUS[i]: + disabled = 'DISABLED' + + if 'FETCH' in lib.JOBSTATUS[i]: + + # open the STDOUT file + stdOut = 'jobData/'+lib.JOBDIR[i]+'/STDOUT' + # unzip the STDOUT file if necessary + if os.access(stdOut+'.gz', os.R_OK): + os.system('gunzip '+stdOut+'.gz') + + try: + with open(stdOut, "r") as STDFILE: + # scan records in input file. + # use regular expression to search. re.compile needed for options re.M and re.I + # re.M=re.MULTILINE enables matching of newline char + # re.I=re.IGNORECASE makes matching case-insensitive. + for line in STDFILE: + if re.search(re.compile('Unable to access quota space',re.M|re.I), line): + quotaspace = 1 + if re.search(re.compile('Unable to get quota space',re.M|re.I), line): + quotaspace = 1 + if re.search(re.compile('Disk quota exceeded',re.M|re.I), line): + quotaspace = 1 + if re.search(re.compile('CERN report: Job Killed',re.M), line): + killed = 1 + if re.search(re.compile('Job finished',re.M), line): + finished = 1 + if re.search(re.compile('connection timed out',re.M), line): + timeout = 1 + if re.search(re.compile('ConfigFileReadError',re.M), line): + cfgerr = 1 + if re.search(re.compile('0 bytes transferred',re.M), line): + emptyDatOnFarm = 1 + if re.search(re.compile('command not found',re.M), line): + cmdNotFound = 1 + # AP 26.11.2009 Insufficient privileges to rfcp files + if re.search(re.compile('stage_put: Insufficient user privileges',re.M), line): + insuffPriv = 1 + # AP 05.11.2015 Extract cpu-time. + # STDOUT doesn't contain NCU anymore. Now KSI2K and HS06 seconds are displayed. + # The ncuFactor is calculated from few samples by comparing KSI2K seconds with + # CPU time from email. + match = re.search(re.compile('This process used .+?(\d+) KSI2K seconds',re.M|re.I), line) + if match: + cpuFactor = 2.125 + cputime = int(round(int(match.group(1))/cpuFactor)) # match.group(1) is the matched digit + + # gzip it afterwards: + print 'gzip -f '+stdOut + os.system('gzip -f '+stdOut) + except IOError as e: + if e.args == (2, "No such file or directory"): + print "mps_check.py cannot find", stdOut, "to test" + else: + raise + + # GF: This file is not produced (anymore...) -> check for existence and read-access added + eazeLog = 'jobData/'+lib.JOBDIR[i]+'/cmsRun.out' + if os.access(eazeLog, os.R_OK): + # open the input file + with open(eazeLog, "r") as INFILE: + # scan records in input file + for line in INFILE: + # check if end of file has been reached + if re.search(re.compile('\',re.M), line): + eofile = 1 + if re.search(re.compile('Time limit reached\.',re.M), line): + timel = 1 + if re.search(re.compile('gives I\/O problem',re.M), line): + ioprob = 1 + if re.search(re.compile('FrameworkError ExitStatus=[\'\"]8001[\'\"]',re.M), line): + fw8001 = 1 + if re.search(re.compile('too many tracks',re.M), line): + tooManyTracks = 1 + if re.search(re.compile('segmentation violation',re.M), line): + segviol = 1 + if re.search(re.compile('failed RFIO error',re.M), line): + rfioerr = 1 + if re.search(re.compile('Request exceeds quota',re.M), line): + quota = 1 + + # if there is an alignment.log[.gz] file, check it as well + eazeLog = 'jobData/'+lib.JOBDIR[i]+'/alignment.log' + logZipped = 'no' + # unzip the logfile if necessary + if os.access(eazeLog+'.gz', os.R_OK): + os.system('gunzip '+eazeLog+'.gz') + logZipped = 'true' + + if os.access(eazeLog, os.R_OK): # access to alignment.log + # open the input file + with open(eazeLog,'r') as INFILE: + # scan records in input file + for line in INFILE: + # check if end of file has been reached + if re.search(re.compile('\',re.M), line): + eofile = 1 + if re.search(re.compile('EAZE\. Time limit reached\.',re.M), line): + timel = 1 + if re.search(re.compile('GAF gives I\/O problem',re.M), line): + ioprob = 1 + if re.search(re.compile('FrameworkError ExitStatus=[\'\"]8001[\'\"]',re.M), line): + fw8001 = 1 + if re.search(re.compile('too many tracks',re.M), line): + tooManyTracks = 1 + if re.search(re.compile('segmentation violation',re.M), line): + segviol = 1 + if re.search(re.compile('failed RFIO error',re.M), line): + rfioerr = 1 + if re.search(re.compile('Request exceeds quota',re.M), line): + quota = 1 + # check for newer (e.g. CMSSW_5_1_X) and older CMSSW: + if re.search(re.compile('Fatal Exception',re.M), line): + exceptionCaught = 1 + if re.search(re.compile('Exception caught in cmsRun',re.M), line): + exceptionCaught = 1 + # AP 07.09.2009 - Check that the job got to a normal end + if re.search(re.compile('AlignmentProducer::endOfJob\(\)',re.M), line): + endofjob = 1 + if re.search(re.compile('FwkReport -i main_input:sourc',re.M), line): + array = line.split() + nEvent = int(array[5]) + if nEvent==0 and re.search(re.compile('FwkReport -i PostSource',re.M), line): + array = line.split() + nEvent = int(array[5]) + # AP 31.07.2009 - To read number of events in CMSSW_3_2_2_patch2 + if nEvent==0 and re.search(re.compile('FwkReport -i AfterSource',re.M), line): + array = line.split() + nEvent = int(array[5]) + + if logZipped == 'true': + os.system('gzip '+eazeLog) + + else: # no access to alignment.log + print 'mps_check.py cannot find',eazeLog,'to test' + # AP 07.09.2009 - The following check cannot be done: set to 1 to avoid fake error type + endofjob = 1 + + # for mille jobs checks that milleBinary file is not empty + if i0): + emptyDatErr = 1 + + # merge jobs: additional checks for merging job + else: + # if there is a pede.dump file check it as well + eazeLog = 'jobData/'+lib.JOBDIR[i]+'/pede.dump' + if os.access(eazeLog+'.gz', os.R_OK): + # unzip - but clean before and save to tmp + os.system('rm -f /tmp/pede.dump') + os.system('gunzip -c '+eazeLog+'.gz > /tmp/pede.dump') + eazeLog = '/tmp/pede.dump' + if os.access(eazeLog, os.R_OK): + with open(eazeLog, "r") as INFILE: # open pede.dump + # scan records in INFILE + pedeAbend = 1 + usedPedeMem = 0. + for line in INFILE: + # check if pede has reached its normal end + if re.search(re.compile('Millepede II.* ending',re.M), line): + pedeAbend = 0 + # extract memory usage + match = re.search(re.compile('Peak dynamic memory allocation: (.+) GB',re.I), line) + if match: + mem = match.group(1) + mem = re.sub('\s', '', mem) + # if mem is a float + if re.search(re.compile('^\d+\.\d+$',re.M), mem): + usedPedeMem = float(mem) + else: + print 'mps_check.py: Found Pede peak memory allocation but extracted number is not a float:',mem + + # check memory usage + # no point in asking if lib.pedeMem is defined. Initialized as lib.pedeMem=-1 + if lib.pedeMem > 0 and usedPedeMem > 0.: + memoryratio = usedPedeMem /(lib.pedeMem/1024.) + # print a warning if more than approx. 4 GB have been + # requested of which less than 75% are used by Pede + if lib.pedeMem > 4000 and memoryratio < 0.75 : + msg = ("Warning: {0:.2f} GB of memory for Pede " + "requested, but only {1:.1f}% of it has been " + "used! Consider to request less memory in order " + "to save resources.").format(lib.pedeMem/1024.0, + memoryratio*100) + print msg + else: + msg = ("Info: Used {0:.1f}% of {1:.2f} GB of memory " + "which has been requested for Pede.") + print msg.format(memoryratio*100, lib.pedeMem/1024.0) + + + # clean up /tmp/pede.dump if needed + if eazeLog == '/tmp/pede.dump': + os.system('rm /tmp/pede.dump') + + # pede.dump not found or no read-access + else: + print 'mps_check.py cannot find',eazeLog,'to test' + + # if there is a millepede.log file, check it as well + eazeLog = 'jobData/'+lib.JOBDIR[i]+'/millepede.log' + logZipped = 'no' + if os.access(eazeLog+'.gz', os.R_OK): + os.system('gunzip '+eazeLog+'.gz') + logZipped = 'true' + + if os.access(eazeLog, os.R_OK): + # open log file + with open(eazeLog, "r") as INFILE: + # scan records in input file + for line in INFILE: + # Checks for Pede Errors + if re.search(re.compile('step no descending',re.M), line): + pedeLogErr = 1 + pedeLogErrStr += line + if re.search(re.compile('Constraint equation discrepancies:',re.M), line): + pedeLogErr = 1 + pedeLogErrStr += line + # AP 07.09.2009 - Checks for Pede Warnings: + if re.search(re.compile('insufficient constraint equations',re.M), line): + pedeLogWrn = 1 + pedeLogWrnStr += line + + if logZipped == 'true': + os.system('gzip '+eazeLog) + else: + print 'mps_check.py cannot find',eazeLog,'to test' + + + # check millepede.end -- added F. Meier 03.03.2015 + eazeLog = 'jobData/'+lib.JOBDIR[i]+'/millepede.end' + logZipped = 'no' + if os.access(eazeLog+'.gz', os.R_OK): + os.system('gunzip'+eazeLog+'.gz') + logZipped = 'true' + + if os.access(eazeLog, os.R_OK): + # open log file + with open(eazeLog, "r") as INFILE: + # scan records in input file + for line in INFILE: + # Checks for the output code. 0 is OK, 1 is WARN, anything else is FAIL + # searches the line for a number with or without a sign + match = re.search(re.compile('([-+]?\d+)',re.M), line) + if match: + if int(match.group(1)) == 1: + pedeLogWrn = 1 + pedeLogWrnStr += line + elif int(match.group(1)) != 0: + pedeLogErr = 1 + pedeLogErrStr += line + if logZipped == 'true': + os.system('gzip '+eazeLog) + else: + print 'mps_check.py cannot find',eazeLog,'to test' + + # end of merge job checks + # evaluate Errors: + farmhost = ' ' + + okStatus = 'OK' + if not eofile == 1: + print lib.JOBDIR[i],lib.JOBID[i],'did not reach end of file' + okStatus = 'ABEND' + if quotaspace == 1: + print lib.JOBDIR[i],lib.JOBID[i],'had quota space problem' + okStatus = 'FAIL' + remark = 'eos quota space problem' + if ioprob == 1: + print lib.JOBDIR[i],lib.JOBID[i],'had I/O problem' + okStatus = 'FAIL' + if fw8001 == 1: + print lib.JOBDIR[i],lib.JOBID[i],'had Framework error 8001 problem' + remark = 'fwk error 8001' + okStatus = 'FAIL' + if timeout == 1: + print lib.JOBDIR[i],lib.JOBID[i],'had connection timed out problem' + remark = 'connection timed out' + if cfgerr == 1: + print lib.JOBDIR[i],lib.JOBID[i],'had config file error' + remark = 'cfg file error' + okStatus = 'FAIL' + if killed == 1: + print lib.JOBDIR[i],lib.JOBID[i],'Job Killed (probably time exceeded)' + remark = "killed"; + okStatus = "FAIL" + if timel == 1: + print lib.JOBDIR[i],lib.JOBID[i],'ran into time limit' + okStatus = 'TIMEL' + if tooManyTracks == 1: + print lib.JOBDIR[i],lib.JOBID[i],'too many tracks' + if segviol == 1: + print lib.JOBDIR[i],lib.JOBID[i],'SEGVIOL encountered' + remark = 'seg viol' + okStatus = 'FAIL' + if rfioerr == 1: + print lib.JOBDIR[i],lib.JOBID[i],'RFIO error encountered' + remark = 'rfio error' + okStatus = 'FAIL' + if quota == 1: + print lib.JOBDIR[i],lib.JOBID[i],'Request exceeds quota' + if exceptionCaught == 1: + print lib.JOBDIR[i],lib.JOBID[i],'Exception caught in cmsrun' + remark = 'Exception caught' + okStatus = 'FAIL' + if emptyDatErr == 1: + print 'milleBinary???.dat file not found or empty' + remark = 'empty milleBinary' + if emptyDatOnFarm > 0: + print '...but already empty on farm so OK (or check job',i+1,'yourself...)' + else: + okStatus = 'FAIL' + if cmdNotFound == 1: + print lib.JOBDIR[i],lib.JOBID[i],'Command not found' + remark = 'cmd not found' + okStatus = 'FAIL' + if insuffPriv == 1: + print lib.JOBDIR[i],lib.JOBID[i],'Insufficient privileges to rfcp files' + remark = 'Could not rfcp files' + okStatus = 'FAIL' + if pedeAbend == 1: + print lib.JOBDIR[i],lib.JOBID[i],'Pede did not end normally' + remark = 'pede failed' + okStatus = 'FAIL' + if pedeLogErr == 1: + print lib.JOBDIR[i],lib.JOBID[i],'Problems in running Pede:' + print pedeLogErrStr + remark = 'pede error' + okStatus = 'FAIL' + if pedeLogWrn == 1: + # AP 07.09.2009 - Reports Pede Warnings (but do _not_ set job status to FAIL) + print lib.JOBDIR[i],lib.JOBID[i],'Warnings in running Pede:' + print pedeLogWrnStr + remark = 'pede warnings' + okStatus = 'WARN' + if endofjob != 1: + print lib.JOBDIR[i],lib.JOBID[i],'Job not ended' + remark = 'job not ended' + okStatus = 'FAIL' + + # print warning line to stdout + if okStatus != "OK": + print lib.JOBDIR[i],lib.JOBID[i],' -------- ',okStatus + + # update number of events + lib.JOBNEVT[i] = nEvent + # udate Jobstatus + lib.JOBSTATUS[i] = disabled+okStatus + # update cputime + print cputime + lib.JOBRUNTIME[i] = cputime + # update remark + lib.JOBREMARK[i] = remark + # update host + ##lib.JOBHOST[i] = farmhost lib.write_db() diff --git a/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_create_file_lists.py b/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_create_file_lists.py index d58d1ca5423d6..34563f73d155f 100755 --- a/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_create_file_lists.py +++ b/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_create_file_lists.py @@ -12,6 +12,7 @@ import cPickle import difflib import argparse +import functools import subprocess import multiprocessing import Utilities.General.cmssw_das_client as cmssw_das_client @@ -45,14 +46,15 @@ def __init__(self, argv): - `args`: command line arguments """ + self._parser = self._define_parser() + self._args = self._parser.parse_args(argv) + if not check_proxy(): print_msg( "Please create proxy via 'voms-proxy-init -voms cms -rfc'.") sys.exit(1) self._dataset_regex = re.compile(r"^/([^/]+)/([^/]+)/([^/]+)$") - parser = self._define_parser() - self._args = parser.parse_args(argv) self._validate_input() self._datasets = sorted([dataset for pattern in self._args.datasets @@ -159,24 +161,29 @@ def _define_parser(self): def _validate_input(self): """Validate command line arguments.""" - if self._args.events: - if self._args.tracks or self._args.rate: - msg = ("-n/--events-for-alignment must not be used with " - "--tracks-for-alignment or --track-rate") - parser.error(msg) - print_msg("Requested {0:d} events for alignment." - .format(self._args.events)) - else: - if not (self._args.tracks and self._args.rate): + if self._args.events is None: + if (self._args.tracks is None) and (self._args.rate is None): + msg = ("either -n/--events-for-alignment or both of " + "--tracks-for-alignment and --track-rate are required") + self._parser.error(msg) + if (((self._args.tracks is not None) and (self._args.rate is None)) or + ((self._args.rate is not None)and (self._args.tracks is None))): msg = ("--tracks-for-alignment and --track-rate must be used " "together") - parser.error(msg) + self._parser.error(msg) self._args.events = int(math.ceil(self._args.tracks / self._args.rate)) print_msg("Requested {0:d} tracks with {1:.2f} tracks/event " "-> {2:d} events for alignment." .format(self._args.tracks, self._args.rate, self._args.events)) + else: + if (self._args.tracks is not None) or (self._args.rate is not None): + msg = ("-n/--events-for-alignment must not be used with " + "--tracks-for-alignment or --track-rate") + self._parser.error(msg) + print_msg("Requested {0:d} events for alignment." + .format(self._args.events)) for dataset in self._args.datasets: if not re.match(self._dataset_regex, dataset): @@ -273,19 +280,9 @@ def _request_dataset_information(self): (self._events_in_dataset, self._files, self._file_info) = self._cache.get() + if self._args.random: random.shuffle(self._files) return - self._events_in_dataset = 0 - self._files = [] - for dataset in self._datasets: - print_msg("Requesting information for dataset '{0:s}'." - .format(dataset)) - self._events_in_dataset += get_events_per_dataset(dataset) - self._files.extend(get_files(dataset)) - if self._args.random: random.shuffle(self._files) - - result = print_msg("Counting events in {0:d} dataset files. This may " - "take several minutes...".format(len(self._files))) # workaround to deal with KeyboardInterrupts in the worker processes: # - ignore interrupt signals in workers (see initializer) # - use a timeout of size sys.maxint to avoid a bug in multiprocessing @@ -296,12 +293,28 @@ def _request_dataset_information(self): pool = multiprocessing.Pool( processes = number_of_processes, initializer = lambda: signal.signal(signal.SIGINT, signal.SIG_IGN)) - count = pool.map_async(get_events_per_file, self._files).get(sys.maxsize) - self._file_info = dict(zip(self._files, count)) + + print_msg("Requesting information for the following dataset(s):") + for d in self._datasets: print_msg("\t"+d) + print_msg("This may take a while...") + + result = pool.map_async(get_events_per_dataset, self._datasets).get(sys.maxint) + self._events_in_dataset = sum(result) + + get_file_info = functools.partial(_get_properties, + properties = ["name", "nevents"], + filters = ["nevents > 0"], + entity = "dataset", + sub_entity = "file") + result = pool.map_async(get_file_info, self._datasets).get(sys.maxint) + self._file_info = {} + for item in result: self._file_info.update(dict(item)) + self._files = sorted(self._file_info.keys()) # write information to cache self._cache.set(self._events_in_dataset, self._files, self._file_info) self._cache.dump() + if self._args.random: random.shuffle(self._files) def _create_file_lists(self): @@ -424,60 +437,66 @@ def _print_eventcounts(self): def _write_file_lists(self): """Write file lists to disk.""" - self._create_alignment_file_list(self._formatted_dataset+".txt", - self._files_alignment) + self._create_dataset_txt(self._formatted_dataset, self._files_alignment) + self._create_dataset_cff( + "_".join(["Alignment", self._formatted_dataset]), + self._files_alignment) - self._create_validation_dataset("_".join(["Dataset", - self._formatted_dataset, - "cff.py"]), - self._files_validation) + self._create_dataset_cff( + "_".join(["Validation", self._formatted_dataset]), + self._files_validation) for iov in sorted(self._iovs): iov_str = "since{0:d}".format(iov) - self._create_alignment_file_list( - "_".join([self._formatted_dataset, iov_str])+".txt", + self._create_dataset_txt( + "_".join([self._formatted_dataset, iov_str]), self._iov_info_alignment[iov]["files"]) + self._create_dataset_cff( + "_".join(["Alignment", self._formatted_dataset, iov_str]), + self._iov_info_validation[iov]["files"]) if (self._iov_info_validation[iov]["events"] < self._args.minimum_events_validation): continue - self._create_validation_dataset( - "_".join(["Dataset", self._formatted_dataset, iov_str, "cff.py"]), + self._create_dataset_cff( + "_".join(["Validation", self._formatted_dataset, iov_str]), self._iov_info_validation[iov]["files"]) for run in sorted(self._run_info): if (self._run_info[run]["events"] < self._args.minimum_events_validation): continue - self._create_validation_dataset( - "_".join(["Dataset", self._formatted_dataset, str(run), "cff.py"]), + self._create_dataset_cff( + "_".join(["Validation", self._formatted_dataset, str(run)]), self._run_info[run]["files"]) - def _create_alignment_file_list(self, name, file_list): + def _create_dataset_txt(self, name, file_list): """Write alignment file list to disk. Arguments: - `name`: name of the file list - - `file_list`: list of files to written to `name` + - `file_list`: list of files to write to `name` """ - print_msg("Creating MillePede file list: "+name) + name += ".txt" + print_msg("Creating datset file list: "+name) with open(os.path.join(self._formatted_dataset, name), "w") as f: f.write("\n".join(file_list)) - def _create_validation_dataset(self, name, file_list): + def _create_dataset_cff(self, name, file_list): """ - Create configuration fragment to define a dataset for validation. + Create configuration fragment to define a dataset. Arguments: - `name`: name of the configuration fragment - - `file_list`: list of files to written to `name` + - `file_list`: list of files to write to `name` """ - print_msg("Creating validation dataset configuration fragment: "+name) + name = "_".join(["Dataset",name, "cff.py"]) + print_msg("Creating dataset configuration fragment: "+name) file_list_str = "" for sub_list in get_chunks(file_list, 255): @@ -605,19 +624,38 @@ def empty(self): ################################################################################ -def das_client(query): +def das_client(query, check_key = None): """ Submit `query` to DAS client and handle possible errors. Further treatment of the output might be necessary. Arguments: - `query`: DAS query + - `check_key`: optional key to be checked for; retriggers query if needed """ - for _ in xrange(3): # maximum of 3 tries + + error = True + for i in xrange(5): # maximum of 5 tries das_data = cmssw_das_client.get_data(query, limit = 0) - if das_data["status"] != "error": break + + if das_data["status"] == "ok": + if das_data["nresults"] == 0 or check_key is None: + error = False + break + + result_count = 0 + for d in find_key(das_data["data"], check_key): + result_count += len(d) + if result_count == 0: + das_data["status"] = "error" + das_data["reason"] = ("DAS did not return required data.") + continue + else: + error = False + break + if das_data["status"] == "error": - print_msg("DAS query '{}' failed 3 times. " + print_msg("DAS query '{}' failed 5 times. " "The last time for the the following reason:".format(query)) print das_data["reason"] sys.exit(1) @@ -678,8 +716,9 @@ def get_files(dataset_name): - `dataset_name`: name of the dataset """ - data = das_client("file dataset={0:s} | grep file.name, file.nevents > 0" - .format(dataset_name)) + data = das_client(("file dataset={0:s} system=dbs3 | "+ + "grep file.name, file.nevents > 0").format(dataset_name), + "file") return [find_key(f["file"], "name") for f in data] @@ -690,8 +729,8 @@ def get_datasets(dataset_pattern): - `dataset_pattern`: pattern of dataset names """ - data = das_client("dataset dataset={0:s} | grep dataset.name" - .format(dataset_pattern)) + data = das_client("dataset dataset={0:s} system=dbs3 | grep dataset.name" + .format(dataset_pattern), "dataset") return [find_key(f["dataset"], "name") for f in data] @@ -723,10 +762,35 @@ def _get_events(entity, name): - `name`: name of entity """ - data = das_client("{0:s}={1:s} | grep {0:s}.nevents".format(entity, name)) + data = das_client("{0:s}={1:s} system=dbs3 | grep {0:s}.nevents" + .format(entity, name), entity) return int(find_key(find_key(data, entity), "nevents")) +def _get_properties(name, entity, properties, filters, sub_entity = None): + """Retrieve `properties` from `entity` called `name`. + + Arguments: + - `name`: name of entity + - `entity`: type of entity + - `properties`: list of property names + - `filters`: list of filters on properties + - `sub_entity`: type of entity from which to extract the properties; + defaults to `entity` + """ + + if sub_entity is None: sub_entity = entity + props = ["{0:s}.{1:s}".format(sub_entity,prop.split()[0]) + for prop in properties] + conditions = ["{0:s}.{1:s}".format(sub_entity, filt) + for filt in filters] + + data = das_client("{0:s} {1:s}={2:s} system=dbs3 | grep {3:s}" + .format(sub_entity, entity, name, + ", ".join(props+conditions)), sub_entity) + return [[find_key(f[sub_entity], prop) for prop in properties] for f in data] + + def get_chunks(long_list, chunk_size): """ Generates list of sub-lists of `long_list` with a maximum size of diff --git a/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_fetch.py b/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_fetch.py index 6fe68461ff6a8..cd8334abdc0a8 100755 --- a/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_fetch.py +++ b/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_fetch.py @@ -2,10 +2,10 @@ # # # Fetch jobs that have DONE status -# This step is mainly foreseen in case job result files need +# This step is mainly foreseen in case job result files need # to be copied from a spool area. # On LSF batch, the job output is already in our directories, -# hence this function does hardly anything except for calling +# hence this function does hardly anything except for calling # mps_check.py. import Alignment.MillePedeAlignmentAlgorithm.mpslib.Mpslibclass as mpslib @@ -19,24 +19,25 @@ # loop over DONE jobs for i in xrange(len(lib.JOBID)): - if 'DONE' in lib.JOBSTATUS[i]: - # move the LSF output to /jobData/ - theJobDir = 'jobData/'+lib.JOBDIR[i] - theBatchDirectory = 'LSFJOB\_%d' % lib.JOBID[i] - - command = 'mv %s/* %s/' % (theBatchDirectory, theJobDir) - os.system(command) - command = 'rmdir '+theBatchDirectory - os.system(command) - - # update the status - if 'DISABLED' in lib.JOBSTATUS[i]: - lib.JOBSTATUS[i] = 'DISABLEDFETCH' - else: - lib.JOBSTATUS[i] = 'FETCH' - + # check also "FETCH" to recover from possibly failed runs of 'mps_fetch.py' + if lib.JOBSTATUS[i] in ("DONE", "FETCH", "DISABLEDFETCH"): + # move the LSF output to /jobData/ + theJobDir = 'jobData/'+lib.JOBDIR[i] + theBatchDirectory = 'LSFJOB\_%d' % lib.JOBID[i] + + command = 'mv %s/* %s/' % (theBatchDirectory, theJobDir) + os.system(command) + command = 'rmdir '+theBatchDirectory + os.system(command) + + # update the status + if 'DISABLED' in lib.JOBSTATUS[i]: + lib.JOBSTATUS[i] = 'DISABLEDFETCH' + else: + lib.JOBSTATUS[i] = 'FETCH' + lib.write_db() # call mps_check os.system('mps_check.py') - + diff --git a/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_fire.py b/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_fire.py index e4d3f567a68f4..238312c839c7a 100755 --- a/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_fire.py +++ b/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_fire.py @@ -15,10 +15,31 @@ import Alignment.MillePedeAlignmentAlgorithm.mpslib.Mpslibclass as mpslib import os import sys +import shutil import subprocess import re import argparse +def forward_proxy(rundir): + """Forward proxy to location visible from the batch system. + + Arguments: + - `rundir`: directory for storing the forwarded proxy + """ + + # check first if proxy is set + try: + subprocess.check_call(["voms-proxy-info", "--exists"]) + except subprocess.CalledProcessError: + print "Please initialize your proxy before submitting." + sys.exit(1) + + local_proxy = subprocess.check_output(["voms-proxy-info", "--path"]).strip() + shutil.copyfile(local_proxy, os.path.join(rundir,".user_proxy")) + + + + parser = argparse.ArgumentParser( description="Submit jobs that are setup in local mps database to batch system.", ) @@ -34,6 +55,9 @@ action="store_true", help=("force the submission of the Pede job in case some "+ "Mille jobs are not in the OK state")) +parser.add_argument("-p", "--forward-proxy", dest="forwardProxy", default=False, + action="store_true", + help="forward VOMS proxy to batch system") args = parser.parse_args(sys.argv[1:]) @@ -75,6 +99,9 @@ for i in xrange(lib.nJobs): if lib.JOBSTATUS[i] == 'SETUP': if nSub < args.maxJobs: + if args.forwardProxy: + forward_proxy(os.path.join(theJobData,lib.JOBDIR[i])) + # submit a new job with 'bsub -J ...' and check output # for some reasons LSF wants script with full path submission = 'bsub -J %s %s %s/%s/theScript.sh' % \ @@ -148,11 +175,9 @@ os.system('cp -p '+scriptPath+' '+backupScriptPath) # get the name of merge cfg file -> either the.py or alignment_merge.py - command = 'cat '+backupScriptPath+' | grep cmsRun | grep "\.py" | head -1 | awk \'{gsub("^.*cmsRun ","");print $1}\'' + command = 'cat '+backupScriptPath+' | grep CONFIG_FILE | head -1 | awk -F"/" \'{print $NF}\'' mergeCfg = subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True) - command = 'basename '+mergeCfg - mergeCfg = subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True) - mergeCfg = mergeCfg.replace('\n','') + mergeCfg = mergeCfg.strip() # make a backup copy of the cfg backupCfgPath = Path+'/%s.bak' % mergeCfg @@ -175,7 +200,7 @@ os.system('cp -pf '+backupScriptPath+' '+scriptPath) # get the name of merge cfg file - command = 'cat '+scriptPath+' | grep cmsRun | grep "\.py" | head -1 | awk \'{gsub("^.*cmsRun ","");print $1}\'' + command = "cat "+scriptPath+" | grep '^\s*CONFIG_FILE' | awk -F'=' '{print $2}'" mergeCfg = subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True) command = 'basename '+mergeCfg mergeCfg = subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True) @@ -192,6 +217,7 @@ # submit merge job nMerge = i-lib.nJobs # 'index' of this merge job curJobName = 'm'+str(nMerge)+'_'+theJobName + if args.forwardProxy: forward_proxy(os.path.dirname(scriptPath)) submission = 'bsub -J %s %s %s' % (curJobName,resources,scriptPath) result = subprocess.check_output(submission, stderr=subprocess.STDOUT, shell=True) print ' '+result, diff --git a/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_parse_pedechi2hist.py b/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_parse_pedechi2hist.py index d7c631274b83a..e65953266da66 100755 --- a/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_parse_pedechi2hist.py +++ b/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_parse_pedechi2hist.py @@ -20,6 +20,7 @@ import re import argparse +import Alignment.MillePedeAlignmentAlgorithm.mpslib.tools as mps_tools import Alignment.MillePedeAlignmentAlgorithm.mpslib.Mpslibclass as mpslib @@ -96,14 +97,9 @@ def get_used_binaries(cfg, no_binary_check): - `no_binary_check`: if 'True' a check for file existence is skipped """ - sys.path.append(os.path.dirname(cfg)) + cms_process = mps_tools.get_process_object(cfg) - cache_stdout = sys.stdout - sys.stdout = open(os.devnull, "w") # suppress unwanted output - from alignment_merge import process.AlignmentProducer.algoConfig - sys.stdout = cache_stdout - - binaries = algoConfig.mergeBinaryFiles + binaries = cms_process.AlignmentProducer.algoConfig.mergeBinaryFiles if no_binary_check: used_binaries = binaries else: @@ -113,7 +109,6 @@ def get_used_binaries(cfg, no_binary_check): used_binaries = [int(re.sub(r"milleBinary(\d+)\.dat", r"\1", b)) for b in used_binaries] - del sys.path[-1] # remove the temporary search path extension return used_binaries diff --git a/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_prepare_input_db.py b/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_prepare_input_db.py index 888d0b10f29c2..9a7e1bb3594f2 100755 --- a/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_prepare_input_db.py +++ b/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_prepare_input_db.py @@ -28,9 +28,13 @@ def main(argv = None): help="name of the output file (default: '%(default)s')") args = parser.parse_args(argv) - mps_tools.create_single_iov_db(args.global_tag, - args.run_number, - args.output_db) + inputs = mps_tools.get_tags(args.global_tag, + ["TrackerAlignmentRcd", + "TrackerSurfaceDeformationRcd", + "TrackerAlignmentErrorExtendedRcd"]) + for inp in inputs.itervalues(): + inp["iovs"] = mps_tools.get_iovs(inp["connect"], inp["tag"]) + mps_tools.create_single_iov_db(inputs, args.run_number, args.output_db) ################################################################################ diff --git a/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_scriptm.pl b/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_scriptm.pl index c7d572ce789dc..4c336f46a2cd3 100755 --- a/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_scriptm.pl +++ b/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_scriptm.pl @@ -99,6 +99,12 @@ BEGIN } $nn = ($body =~ s/RUNDIR=(.+)$/RUNDIR=$runDir/m); +# replace the cfg name +$nn = ($body =~ s/CONFIG_FILE=(.*)$/CONFIG_FILE=\$RUNDIR\/$cfgName/m); +if ($nn <1) { + print "Warning: mps_script matches cfg: $nn\n"; +} + #replace CMSSW_RELEASE_AREA with evironment variable $body =~ s/cd\s+CMSSW_RELEASE_AREA/cd $ENV{'CMSSW_BASE'}/g; @@ -117,19 +123,6 @@ BEGIN $nn = ($body =~ s/MSSDIRPOOL=(.*)$/MSSDIRPOOL=/m); } -# replace the cfg name -$nn = ($body =~ m/cmsRun +([A-Z,a-z,0-9\-\.])/g); - -# $nn = ($body =~ m/cmsRun +(.+)/g); -if ($nn <1) { - print "Warning: mps_script matches cfg: $nn\n"; -} -# $nn = ($body =~ s/cmsRun\s([A-Za-z0-9]+?\.cfg)/cmsRun $cfgName/g); -# $nn = ($body =~ s/cmsRun +(.+)/cmsRun $cfgName/g); -$nn = ($body =~ s/cmsRun +[a-zA-Z_0-9\-]+\.cfg/cmsRun \$RUNDIR\/$cfgName/g); - -$nn = $body =~ s/mps_parse_pedechi2hist\.pl\s(.+)\s\w*?\.cfg/mps_parse_pedechi2hist\.pl $1 \$RUNDIR\/$cfgName/gi; - # now we have to expand lines that contain the ISN directive @LINES = split "\n",$body; diff --git a/Alignment/MillePedeAlignmentAlgorithm/templates/mps_runMille_template.sh b/Alignment/MillePedeAlignmentAlgorithm/templates/mps_runMille_template.sh index 60e64db2bcb5f..f87a48ecf6c79 100644 --- a/Alignment/MillePedeAlignmentAlgorithm/templates/mps_runMille_template.sh +++ b/Alignment/MillePedeAlignmentAlgorithm/templates/mps_runMille_template.sh @@ -26,6 +26,7 @@ clean_up () { #LSF signals according to http://batch.web.cern.ch/batch/lsf-return-codes.html trap clean_up HUP INT TERM SEGV USR2 XCPU XFSZ IO +export X509_USER_PROXY=${RUNDIR}/.user_proxy # The batch job directory (will vanish after job end): BATCH_DIR=$(pwd) diff --git a/Alignment/MillePedeAlignmentAlgorithm/templates/mps_runPede_rfcp_template.sh b/Alignment/MillePedeAlignmentAlgorithm/templates/mps_runPede_rfcp_template.sh index edc35f09986e8..08bf3cc239e6f 100644 --- a/Alignment/MillePedeAlignmentAlgorithm/templates/mps_runPede_rfcp_template.sh +++ b/Alignment/MillePedeAlignmentAlgorithm/templates/mps_runPede_rfcp_template.sh @@ -18,6 +18,9 @@ cd - RUNDIR=$HOME/scratch0/some/path MSSDIR=/castor/cern.ch/user/u/username/another/path MSSDIRPOOL= +CONFIG_FILE= + +export X509_USER_PROXY=${RUNDIR}/.user_proxy #get list of treefiles TREEFILELIST= @@ -123,7 +126,7 @@ echo Running directory changed to $(pwd). echo "\nDirectory content before running cmsRun:" ls -lh # Execute. The cfg file name will be overwritten by MPS -time cmsRun the.cfg +time cmsRun $CONFIG_FILE # clean up what has been staged in (to avoid copy mistakes...) rm treeFileISN.root @@ -158,7 +161,7 @@ else # If nothing checked out, take from release: cp $CMSSW_RELEASE_BASE/src/Alignment/MillePedeAlignmentAlgorithm/macros/createChi2ndfplot.C . fi -mps_parse_pedechi2hist.py -d $RUNDIR/../../mps.db --his millepede.his -c the.cfg +mps_parse_pedechi2hist.py -d $RUNDIR/../../mps.db --his millepede.his -c $CONFIG_FILE if [ -f chi2pedehis.txt ]; then root -l -x -b -q 'createChi2ndfplot.C+("chi2pedehis.txt")' fi @@ -201,7 +204,7 @@ cp -p *.db $RUNDIR cp -p *.end $RUNDIR # copy aligment_merge.py for mps_validate.py -cp -p the.cfg alignment_merge.py +cp -p $CONFIG_FILE alignment_merge.py # run mps_validate.py campaign=`basename $MSSDIR` mps_validate.py -m $campaign -p ./ diff --git a/Alignment/MillePedeAlignmentAlgorithm/templates/universalConfigTemplate.py b/Alignment/MillePedeAlignmentAlgorithm/templates/universalConfigTemplate.py index 1af0bd11a023c..2f2e35b512222 100644 --- a/Alignment/MillePedeAlignmentAlgorithm/templates/universalConfigTemplate.py +++ b/Alignment/MillePedeAlignmentAlgorithm/templates/universalConfigTemplate.py @@ -47,6 +47,7 @@ setupCosmicsZeroTesla = False setupPrimaryWidth = -1.0 setupJson = "placeholder_json" +setupRunStartGeometry = -1 ################################################################################ # Variables edited by MPS (mps_setup and mps_merge). Be careful. @@ -130,25 +131,17 @@ ## insert Pedesettings ## ######################### -# # typical pede settings are listed below; +# # reasonable pede settings are already defined in +# # 'confAliProducer.setConfiguration' above # # if you want obtain alignment errors, use "inversion 3 0.8" as # # process.AlignmentProducer.algoConfig.pedeSteerer.method and set # # process.AlignmentProducer.saveApeToDB = True +# # a list of possible options is documented here: +# # http://www.desy.de/~kleinwrt/MP2/doc/html/option_page.html#sec-cmd +# # you can change pede settings as follows: # -# process.AlignmentProducer.algoConfig.pedeSteerer.method = "sparseMINRES-QLP 3 0.8" -# process.AlignmentProducer.algoConfig.pedeSteerer.options = [ -# "entries 50 10 2", -# "outlierdownweighting 3", -# "dwfractioncut 0.1", -# "compress", -# "threads 10", -# "matiter 1", -# "printcounts 2", -# "chisqcut 30. 6.", -# "bandwidth 6 1", -# "monitorresiduals", -# ] -# process.AlignmentProducer.algoConfig.minNumHits = 8 +# import Alignment.MillePedeAlignmentAlgorithm.alignmentsetup.helper as helper +# helper.set_pede_option(process, "entries 50 10 2") ################################################################################ @@ -174,4 +167,5 @@ import Alignment.MillePedeAlignmentAlgorithm.alignmentsetup.PedeSetup as pede pede.setup(process, binary_files = merge_binary_files, - tree_files = merge_tree_files) + tree_files = merge_tree_files, + run_start_geometry = setupRunStartGeometry) diff --git a/Alignment/MillePedeAlignmentAlgorithm/test/BuildFile.xml b/Alignment/MillePedeAlignmentAlgorithm/test/BuildFile.xml new file mode 100644 index 0000000000000..8df6d02ad4bd5 --- /dev/null +++ b/Alignment/MillePedeAlignmentAlgorithm/test/BuildFile.xml @@ -0,0 +1 @@ + diff --git a/Alignment/MuonAlignment/interface/AlignableCSCChamber.h b/Alignment/MuonAlignment/interface/AlignableCSCChamber.h index 5779a3d0b82eb..bcc863969e43b 100644 --- a/Alignment/MuonAlignment/interface/AlignableCSCChamber.h +++ b/Alignment/MuonAlignment/interface/AlignableCSCChamber.h @@ -28,6 +28,9 @@ class AlignableCSCChamber: public AlignableDet { /// Constructor AlignableCSCChamber(const GeomDet *geomDet); + + /// Updater + void update(const GeomDet* geomDet); }; #endif // ALIGNABLE_CSC_CHAMBER_H diff --git a/Alignment/MuonAlignment/interface/AlignableMuon.h b/Alignment/MuonAlignment/interface/AlignableMuon.h index 36897bd8b502c..32a407dbd1a3a 100644 --- a/Alignment/MuonAlignment/interface/AlignableMuon.h +++ b/Alignment/MuonAlignment/interface/AlignableMuon.h @@ -45,6 +45,9 @@ class AlignableMuon: public AlignableComposite /// Destructor ~AlignableMuon(); + /// Updater using DTGeometry and CSCGeometry. + /// The given geometries have to match the current ones. + void update(const DTGeometry* , const CSCGeometry*); /// Return all components virtual align::Alignables components() const { return theMuonComponents; } @@ -52,7 +55,7 @@ class AlignableMuon: public AlignableComposite /// Alignable tracker has no mother virtual Alignable* mother() { return 0; } - // Methods to return specific of components + /// Methods to return specific of components align::Alignables DTLayers(); align::Alignables DTSuperLayers(); align::Alignables DTChambers(); @@ -65,16 +68,16 @@ class AlignableMuon: public AlignableComposite align::Alignables CSCRings(); align::Alignables CSCEndcaps(); - // Get DT alignments sorted by DetId + /// Get DT alignments sorted by DetId Alignments* dtAlignments(); - // Get DT alignment errors sorted by DetId + /// Get DT alignment errors sorted by DetId AlignmentErrorsExtended* dtAlignmentErrorsExtended(); - // Get CSC alignments sorted by DetId + /// Get CSC alignments sorted by DetId Alignments* cscAlignments(); - // Get CSC alignment errors sorted by DetId + /// Get CSC alignment errors sorted by DetId AlignmentErrorsExtended* cscAlignmentErrorsExtended(); @@ -83,30 +86,30 @@ class AlignableMuon: public AlignableComposite private: - // Get the position (centered at 0 by default) + /// Get the position (centered at 0 by default) PositionType computePosition(); - // Get the global orientation (no rotation by default) + /// Get the global orientation (no rotation by default) RotationType computeOrientation(); - // Get the Surface + /// Get the Surface AlignableSurface computeSurface(); - // Get alignments sorted by DetId + /// Get alignments sorted by DetId Alignments* alignments() const; - // Get alignment errors sorted by DetId + /// Get alignment errors sorted by DetId AlignmentErrorsExtended* alignmentErrors() const; - // Sub-structure builders + // Sub-structure builders - // Build muon barrel - void buildDTBarrel( const DTGeometry* ); + /// Build muon barrel + void buildDTBarrel(const DTGeometry*, bool update = false); - // Build muon end caps - void buildCSCEndcap( const CSCGeometry* ); + /// Build muon end caps + void buildCSCEndcap(const CSCGeometry*, bool update = false); /// Set mothers recursively void recursiveSetMothers( Alignable* alignable ); @@ -114,8 +117,7 @@ class AlignableMuon: public AlignableComposite /// alignable object ID provider const AlignableObjectId alignableObjectId_; - // Containers of separate components - + /// Containers of separate components std::vector theDTChambers; std::vector theDTStations; std::vector theDTWheels; diff --git a/Alignment/MuonAlignment/src/AlignableCSCChamber.cc b/Alignment/MuonAlignment/src/AlignableCSCChamber.cc index 24bd90459906c..3edee3e9f1753 100644 --- a/Alignment/MuonAlignment/src/AlignableCSCChamber.cc +++ b/Alignment/MuonAlignment/src/AlignableCSCChamber.cc @@ -7,11 +7,22 @@ #include "Alignment/MuonAlignment/interface/AlignableCSCChamber.h" -AlignableCSCChamber::AlignableCSCChamber(const GeomDet *geomDet): AlignableDet(geomDet) +AlignableCSCChamber::AlignableCSCChamber(const GeomDet* geomDet) : + AlignableDet(geomDet) { - theStructureType = align::AlignableCSCChamber; - // DO NOT let the chamber position become an average of the layers - this->theSurface = geomDet->surface(); + theStructureType = align::AlignableCSCChamber; + // DO NOT let the chamber position become an average of the layers + // FIXME: is this redundant? + theSurface = geomDet->surface(); + compConstraintType_ = Alignable::CompConstraintType::NONE; +} + +void AlignableCSCChamber::update(const GeomDet* geomDet) +{ + AlignableDet::update(geomDet); + // DO NOT let the chamber position become an average of the layers + // FIXME: is this redundant? + theSurface = geomDet->surface(); } /// Printout the DetUnits in the CSC chamber diff --git a/Alignment/MuonAlignment/src/AlignableCSCEndcap.cc b/Alignment/MuonAlignment/src/AlignableCSCEndcap.cc index 62b2f8e764723..c6cc4f3dd0a10 100644 --- a/Alignment/MuonAlignment/src/AlignableCSCEndcap.cc +++ b/Alignment/MuonAlignment/src/AlignableCSCEndcap.cc @@ -21,7 +21,7 @@ AlignableCSCEndcap::AlignableCSCEndcap( const std::vector& theCSCStations.insert( theCSCStations.end(), cscStations.begin(), cscStations.end() ); setSurface( computeSurface() ); - + compConstraintType_ = Alignable::CompConstraintType::POSITION_Z; } diff --git a/Alignment/MuonAlignment/src/AlignableCSCRing.cc b/Alignment/MuonAlignment/src/AlignableCSCRing.cc index 86ce5f20832cd..5e3739aba4f8b 100644 --- a/Alignment/MuonAlignment/src/AlignableCSCRing.cc +++ b/Alignment/MuonAlignment/src/AlignableCSCRing.cc @@ -19,7 +19,7 @@ AlignableCSCRing::AlignableCSCRing( const std::vector& csc theCSCChambers.insert( theCSCChambers.end(), cscChambers.begin(), cscChambers.end() ); setSurface( computeSurface() ); - + compConstraintType_ = Alignable::CompConstraintType::POSITION_Z; } diff --git a/Alignment/MuonAlignment/src/AlignableCSCStation.cc b/Alignment/MuonAlignment/src/AlignableCSCStation.cc index 47fdaa6956cd7..619cc530a22de 100644 --- a/Alignment/MuonAlignment/src/AlignableCSCStation.cc +++ b/Alignment/MuonAlignment/src/AlignableCSCStation.cc @@ -19,7 +19,7 @@ AlignableCSCStation::AlignableCSCStation( const std::vector& theCSCRings.insert( theCSCRings.end(), cscRings.begin(), cscRings.end() ); setSurface( computeSurface() ); - + compConstraintType_ = Alignable::CompConstraintType::POSITION_Z; } diff --git a/Alignment/MuonAlignment/src/AlignableDTBarrel.cc b/Alignment/MuonAlignment/src/AlignableDTBarrel.cc index bc790048dfcc8..48c80460f2b1f 100644 --- a/Alignment/MuonAlignment/src/AlignableDTBarrel.cc +++ b/Alignment/MuonAlignment/src/AlignableDTBarrel.cc @@ -21,7 +21,7 @@ AlignableDTBarrel::AlignableDTBarrel( const std::vector& dtWh theDTWheels.insert( theDTWheels.end(), dtWheels.begin(), dtWheels.end() ); setSurface( computeSurface() ); - + compConstraintType_ = Alignable::CompConstraintType::POSITION_Z; } diff --git a/Alignment/MuonAlignment/src/AlignableDTStation.cc b/Alignment/MuonAlignment/src/AlignableDTStation.cc index 4de27a0722795..e1e63106383cd 100644 --- a/Alignment/MuonAlignment/src/AlignableDTStation.cc +++ b/Alignment/MuonAlignment/src/AlignableDTStation.cc @@ -19,7 +19,7 @@ AlignableDTStation::AlignableDTStation( const std::vector& theDTChambers.insert( theDTChambers.end(), dtChambers.begin(), dtChambers.end() ); setSurface( computeSurface() ); - + compConstraintType_ = Alignable::CompConstraintType::POSITION_Z; } diff --git a/Alignment/MuonAlignment/src/AlignableDTWheel.cc b/Alignment/MuonAlignment/src/AlignableDTWheel.cc index 15ff6f5543521..f2f13c1c3894e 100644 --- a/Alignment/MuonAlignment/src/AlignableDTWheel.cc +++ b/Alignment/MuonAlignment/src/AlignableDTWheel.cc @@ -19,7 +19,7 @@ AlignableDTWheel::AlignableDTWheel( const std::vector& dtSt theDTStations.insert( theDTStations.end(), dtStations.begin(), dtStations.end() ); setSurface( computeSurface() ); - + compConstraintType_ = Alignable::CompConstraintType::POSITION_Z; } diff --git a/Alignment/MuonAlignment/src/AlignableMuon.cc b/Alignment/MuonAlignment/src/AlignableMuon.cc index 4d4aafbd0bdd9..6855b0d0a6812 100644 --- a/Alignment/MuonAlignment/src/AlignableMuon.cc +++ b/Alignment/MuonAlignment/src/AlignableMuon.cc @@ -63,8 +63,23 @@ AlignableMuon::~AlignableMuon() } -//-------------------------------------------------------------------------------------------------- -void AlignableMuon::buildDTBarrel( const DTGeometry* pDT ) +//------------------------------------------------------------------------------ +void AlignableMuon::update(const DTGeometry* dtGeometry , + const CSCGeometry* cscGeometry) +{ + // update the muon barrel + buildDTBarrel(dtGeometry, /* update = */ true); + + // update the muon end caps + buildCSCEndcap(cscGeometry, /* update = */ true); + + edm::LogInfo("Alignment") + << "@SUB=AlignableMuon::update" << "Updating alignable muon objects DONE"; +} + + +//------------------------------------------------------------------------------ +void AlignableMuon::buildDTBarrel(const DTGeometry* pDT, bool update) { LogDebug("Position") << "Constructing AlignableDTBarrel"; @@ -81,12 +96,12 @@ void AlignableMuon::buildDTBarrel( const DTGeometry* pDT ) for( int ist = 1 ; ist < 5 ; ist++ ){ // Loop over geom DT Chambers - std::vector theSLs; - for(auto det = pDT->chambers().begin(); - det != pDT->chambers().end(); ++det ){ + int iChamber{0}; + std::vector theSLs; // FIXME: What is this vector meant to be for? Probably redundant since super layers are handled inside of the AlignableDTChamber. + for(const auto& det: pDT->chambers()){ // Get the chamber ID - DTChamberId chamberId = (*det)->id(); - + DTChamberId chamberId = det->id(); + // Get wheel,station and sector of the chamber int wh = chamberId.wheel(); int st = chamberId.station(); @@ -95,70 +110,76 @@ void AlignableMuon::buildDTBarrel( const DTGeometry* pDT ) // Select the chambers in a given wheel in a given station if ( iwh == wh && ist == st ){ - // Create the alignable DT chamber - AlignableDTChamber* tmpDTChamber = new AlignableDTChamber( *det ); - - // Store the DT chambers in a given DT Station and Wheel - tmpDTChambersInStation.push_back( tmpDTChamber ); + if (update) { + // Update the alignable DT chamber + theDTBarrel.back()->wheel(iwh+2).station(ist-1).chamber(iChamber).update(det); + } else { + // Create the alignable DT chamber + AlignableDTChamber* tmpDTChamber = new AlignableDTChamber(det); - // End chamber selection - } + // Store the DT chambers in a given DT Station and Wheel + tmpDTChambersInStation.push_back( tmpDTChamber ); + } - // End loop over chambers + ++iChamber; + // End chamber selection + } + + // End loop over chambers } - - // Store the DT chambers - theDTChambers.insert( theDTChambers.end(), tmpDTChambersInStation.begin(), - tmpDTChambersInStation.end() ); - // Create the alignable DT station with chambers in a given station and wheel - AlignableDTStation* tmpDTStation = new AlignableDTStation( tmpDTChambersInStation ); - - // Store the DT stations in a given wheel - tmpDTStationsInWheel.push_back( tmpDTStation ); + if (!update) { + // Store the DT chambers + theDTChambers.insert(theDTChambers.end(), tmpDTChambersInStation.begin(), + tmpDTChambersInStation.end()); + + // Create the alignable DT station with chambers in a given station and wheel + AlignableDTStation* tmpDTStation = new AlignableDTStation(tmpDTChambersInStation); - // Clear the temporary vector of chambers in a station - tmpDTChambersInStation.clear(); + // Store the DT stations in a given wheel + tmpDTStationsInWheel.push_back(tmpDTStation); + // Clear the temporary vector of chambers in a station + tmpDTChambersInStation.clear(); + } // End loop over stations } - // Store The DT stations - theDTStations.insert( theDTStations.end(), tmpDTStationsInWheel.begin(), - tmpDTStationsInWheel.end() ); + if (!update) { + // Store The DT stations + theDTStations.insert(theDTStations.end(),tmpDTStationsInWheel.begin(), + tmpDTStationsInWheel.end()); - // Create the alignable DT wheel - AlignableDTWheel* tmpWheel = new AlignableDTWheel( tmpDTStationsInWheel ); + // Create the alignable DT wheel + AlignableDTWheel* tmpWheel = new AlignableDTWheel(tmpDTStationsInWheel); - // Store the DT wheels - theDTWheels.push_back( tmpWheel ); - - // Clear temporary vector of stations in a wheel - tmpDTStationsInWheel.clear(); + // Store the DT wheels + theDTWheels.push_back(tmpWheel); + // Clear temporary vector of stations in a wheel + tmpDTStationsInWheel.clear(); + } // End loop over wheels } - // Create the alignable Muon Barrel - AlignableDTBarrel* tmpDTBarrel = new AlignableDTBarrel( theDTWheels ); - - // Store the barrel - theDTBarrel.push_back( tmpDTBarrel ); - - // Store the barrel in the muon - theMuonComponents.push_back( tmpDTBarrel ); + if (!update) { + // Create the alignable Muon Barrel + AlignableDTBarrel* tmpDTBarrel = new AlignableDTBarrel(theDTWheels); + // Store the barrel + theDTBarrel.push_back(tmpDTBarrel); + // Store the barrel in the muon + theMuonComponents.push_back(tmpDTBarrel); + } } -//-------------------------------------------------------------------------------------------------- - - -void AlignableMuon::buildCSCEndcap( const CSCGeometry* pCSC ) +//------------------------------------------------------------------------------ +void AlignableMuon::buildCSCEndcap(const CSCGeometry* pCSC, bool update) { LogDebug("Position") << "Constructing AlignableCSCBarrel"; @@ -180,92 +201,110 @@ void AlignableMuon::buildCSCEndcap( const CSCGeometry* pCSC ) // Loop over rings ( 1..4 ) for ( int iri = 1; iri < 5; iri++ ){ - - // Loop over geom CSC Chambers - const CSCGeometry::ChamberContainer& vc = pCSC->chambers(); - for( auto det = vc.begin(); det != vc.end(); ++det ){ - // Get the CSCDet ID - CSCDetId cscId = (*det)->id(); + // Loop over geom CSC Chambers + int iChamber{0}; + const CSCGeometry::ChamberContainer& vc = pCSC->chambers(); + for(const auto& det: vc){ - // Get chamber, station, ring, layer and endcap labels of the CSC chamber - int ec = cscId.endcap(); - int st = cscId.station(); - int ri = cscId.ring(); - //int ch = cscId.chamber(); + // Get the CSCDet ID + CSCDetId cscId = det->id(); - // Select the chambers in a given endcap, station, and ring - if ( iec == ec && ist == st && iri == ri ) { + // Get chamber, station, ring, layer and endcap labels of the CSC chamber + int ec = cscId.endcap(); + int st = cscId.station(); + int ri = cscId.ring(); + //int ch = cscId.chamber(); - // Create the alignable CSC chamber - AlignableCSCChamber* tmpCSCChamber = new AlignableCSCChamber( *det ); - - // Store the alignable CSC chambers - tmpCSCChambersInRing.push_back( tmpCSCChamber ); + // Select the chambers in a given endcap, station, and ring + if ( iec == ec && ist == st && iri == ri ) { + + if (update) { + // Update the alignable CSC chamber + theCSCEndcaps[iec-1]->station(ist-1).ring(iri-1).chamber(iChamber).update(det); + } else { + AlignableCSCChamber* tmpCSCChamber = new AlignableCSCChamber(det); - // End If chamber selection - } + // Store the alignable CSC chambers + tmpCSCChambersInRing.push_back(tmpCSCChamber); + } - // End loop over geom CSC chambers - } + ++iChamber; + // End If chamber selection + } - // Not all stations have 4 rings: only add the rings that exist (have chambers associated with them) - if (tmpCSCChambersInRing.size() > 0) { + // End loop over geom CSC chambers + } - // Store the alignable CSC chambers - theCSCChambers.insert( theCSCChambers.end(), tmpCSCChambersInRing.begin(), tmpCSCChambersInRing.end() ); + if (!update) { + // Not all stations have 4 rings: only add the rings that exist (have chambers associated with them) + if (tmpCSCChambersInRing.size() > 0) { - // Create the alignable CSC ring with chambers in a given ring - AlignableCSCRing* tmpCSCRing = new AlignableCSCRing( tmpCSCChambersInRing ); + // Store the alignable CSC chambers + theCSCChambers.insert(theCSCChambers.end(), + tmpCSCChambersInRing.begin(), + tmpCSCChambersInRing.end()); - // Store the CSC rings in a given station - tmpCSCRingsInStation.push_back( tmpCSCRing ); + // Create the alignable CSC ring with chambers in a given ring + AlignableCSCRing* tmpCSCRing = new AlignableCSCRing(tmpCSCChambersInRing); - // Clear the temporary vector of chambers in ring - tmpCSCChambersInRing.clear(); + // Store the CSC rings in a given station + tmpCSCRingsInStation.push_back(tmpCSCRing); - // End if this ring exists - } + // Clear the temporary vector of chambers in ring + tmpCSCChambersInRing.clear(); - // End loop over rings + // End if this ring exists + } + } + + // End loop over rings } - // Create the alignable CSC station with rings in a given station - AlignableCSCStation* tmpCSCStation = new AlignableCSCStation( tmpCSCRingsInStation ); - - // Store the alignable CSC rings - theCSCRings.insert( theCSCRings.end(), tmpCSCRingsInStation.begin(), tmpCSCRingsInStation.end() ); + if (!update) { + // Create the alignable CSC station with rings in a given station + AlignableCSCStation* tmpCSCStation = new AlignableCSCStation(tmpCSCRingsInStation); - // Store the CSC stations in a given endcap - tmpCSCStationsInEndcap.push_back( tmpCSCStation ); + // Store the alignable CSC rings + theCSCRings.insert(theCSCRings.end(), tmpCSCRingsInStation.begin(), + tmpCSCRingsInStation.end()); - // Clear the temporary vector of rings in station - tmpCSCRingsInStation.clear(); + // Store the CSC stations in a given endcap + tmpCSCStationsInEndcap.push_back(tmpCSCStation); - // End loop over stations + // Clear the temporary vector of rings in station + tmpCSCRingsInStation.clear(); + } + + // End loop over stations } - // Create the alignable CSC endcap - AlignableCSCEndcap* tmpEndcap = new AlignableCSCEndcap( tmpCSCStationsInEndcap ); - - // Store the alignable CSC stations - theCSCStations.insert( theCSCStations.end(), tmpCSCStationsInEndcap.begin(), tmpCSCStationsInEndcap.end() ); + if (!update) { + // Create the alignable CSC endcap + AlignableCSCEndcap* tmpEndcap = new AlignableCSCEndcap(tmpCSCStationsInEndcap); - // Store the alignable CSC endcaps - theCSCEndcaps.push_back( tmpEndcap ); + // Store the alignable CSC stations + theCSCStations.insert(theCSCStations.end(), tmpCSCStationsInEndcap.begin(), + tmpCSCStationsInEndcap.end()); - // Clear the temporary vector of stations in endcap - tmpCSCStationsInEndcap.clear(); + // Store the alignable CSC endcaps + theCSCEndcaps.push_back(tmpEndcap); - // End loop over endcaps - } + // Clear the temporary vector of stations in endcap + tmpCSCStationsInEndcap.clear(); + } - // Store the encaps in the muon components - theMuonComponents.insert( theMuonComponents.end(), theCSCEndcaps.begin(), theCSCEndcaps.end() ); + // End loop over endcaps + } - + if (!update) { + // Store the encaps in the muon components + theMuonComponents.insert(theMuonComponents.end(), theCSCEndcaps.begin(), + theCSCEndcaps.end()); + } } + //-------------------------------------------------------------------------------------------------- align::Alignables AlignableMuon::DTLayers() { diff --git a/Alignment/TrackerAlignment/interface/AlignableTracker.h b/Alignment/TrackerAlignment/interface/AlignableTracker.h index 38cf548f7af8e..1a8103636f332 100644 --- a/Alignment/TrackerAlignment/interface/AlignableTracker.h +++ b/Alignment/TrackerAlignment/interface/AlignableTracker.h @@ -29,9 +29,13 @@ class AlignableTracker : public AlignableComposite { /// Return alignables of subdet and hierarchy level determined by name /// as defined in tracker part of Alignment/CommonAlignment/StructureType.h Alignables& subStructures(const std::string &subStructName) { - return alignableMap.find(subStructName); + return alignableMap_.find(subStructName); } + /// Updater using TrackerGeometry and TrackerTopology. + /// The given geometry and topology have to match the current ones. + void update(const TrackerGeometry*, const TrackerTopology*); + /// Return TOB half barrels Alignables& outerHalfBarrels() { return this->subStructures(alignableObjectId_.typeToName(align::TOBHalfBarrel)); @@ -167,7 +171,7 @@ class AlignableTracker : public AlignableComposite { const TrackerTopology* tTopo_; align::TrackerNameSpace trackerNameSpace_; AlignableObjectId alignableObjectId_; - AlignableMap alignableMap; + AlignableMap alignableMap_; }; diff --git a/Alignment/TrackerAlignment/interface/AlignableTrackerBuilder.h b/Alignment/TrackerAlignment/interface/AlignableTrackerBuilder.h index 867dee118bacb..757ef4cd65cc0 100644 --- a/Alignment/TrackerAlignment/interface/AlignableTrackerBuilder.h +++ b/Alignment/TrackerAlignment/interface/AlignableTrackerBuilder.h @@ -25,7 +25,7 @@ class AlignableTrackerBuilder { /// Builds all Alignables (units and composites) of the tracker, based on /// the given TrackerGeometry. - void buildAlignables(AlignableTracker*); + void buildAlignables(AlignableTracker*, bool update = false); /// Return tracker name space derived from the tracker's topology const align::TrackerNameSpace& trackerNameSpace() const { @@ -39,22 +39,25 @@ class AlignableTrackerBuilder { private: //================================================================== /// Builds Alignables on module-level for each part of the tracker. - void buildAlignableDetUnits(); + void buildAlignableDetUnits(bool update = false); /// Decides whether a GeomDet is from Pixel- or Strip-Detector and calls /// the according method to build the Alignable. void convertGeomDetsToAlignables(const TrackingGeometry::DetContainer&, - const std::string& moduleName); + const std::string& moduleName, + bool update = false); /// Converts GeomDetUnits of PXB and PXE to AlignableDetUnits. void buildPixelDetectorAlignable(const GeomDet*, int subdetId, - Alignables& aliDets, Alignables& aliDetUnits); + Alignables& aliDets, Alignables& aliDetUnits, + bool update = false); /// Converts GeomDets of TIB, TID, TOB and TEC either to AlignableDetUnits /// or AlignableSiStripDet, depending on the module-type (2D or 1D). void buildStripDetectorAlignable(const GeomDet*, int subdetId, - Alignables& aliDets, Alignables& aliDetUnits); + Alignables& aliDets, Alignables& aliDetUnits, + bool update = false); /// Builds all composite Alignables for the tracker. The hierarchy and /// numbers of components are determined in TrackerAlignmentLevelBuilder. - void buildAlignableComposites(); + void buildAlignableComposites(bool update = false); /// Builds the PixelDetector by hand. void buildPixelDetector(AlignableTracker*); /// Builds the StripDetector by hand. @@ -63,11 +66,11 @@ class AlignableTrackerBuilder { //========================== PRIVATE DATA =================================== //=========================================================================== - const TrackerGeometry* trackerGeometry; - const TrackerTopology* trackerTopology; + const TrackerGeometry* trackerGeometry_; + const TrackerTopology* trackerTopology_; const AlignableObjectId alignableObjectId_; - AlignableMap* alignableMap; + AlignableMap* alignableMap_; TrackerAlignmentLevelBuilder trackerAlignmentLevelBuilder_; diff --git a/Alignment/TrackerAlignment/src/AlignableTracker.cc b/Alignment/TrackerAlignment/src/AlignableTracker.cc index 752947d5dfdc8..923a8e82b5d10 100644 --- a/Alignment/TrackerAlignment/src/AlignableTracker.cc +++ b/Alignment/TrackerAlignment/src/AlignableTracker.cc @@ -29,6 +29,14 @@ ::AlignableTracker(const TrackerGeometry* trackerGeometry, alignableObjectId_ = builder.objectIdProvider(); } +//_____________________________________________________________________________ +void AlignableTracker::update(const TrackerGeometry* trackerGeometry, + const TrackerTopology* trackerTopology) +{ + AlignableTrackerBuilder builder(trackerGeometry, trackerTopology); + builder.buildAlignables(this, /* update = */ true); +} + //_____________________________________________________________________________ align::Alignables AlignableTracker::merge( const Alignables& list1, const Alignables& list2 ) const diff --git a/Alignment/TrackerAlignment/src/AlignableTrackerBuilder.cc b/Alignment/TrackerAlignment/src/AlignableTrackerBuilder.cc index 7d18a0c8e65e5..a695a5ec71686 100644 --- a/Alignment/TrackerAlignment/src/AlignableTrackerBuilder.cc +++ b/Alignment/TrackerAlignment/src/AlignableTrackerBuilder.cc @@ -25,10 +25,10 @@ AlignableTrackerBuilder ::AlignableTrackerBuilder(const TrackerGeometry* trackerGeometry, const TrackerTopology* trackerTopology) : - trackerGeometry(trackerGeometry), - trackerTopology(trackerTopology), + trackerGeometry_(trackerGeometry), + trackerTopology_(trackerTopology), alignableObjectId_(trackerGeometry, nullptr, nullptr), - alignableMap(nullptr), + alignableMap_(nullptr), trackerAlignmentLevelBuilder_(trackerTopology, trackerGeometry) { std::ostringstream ss; @@ -49,14 +49,17 @@ ::AlignableTrackerBuilder(const TrackerGeometry* trackerGeometry, //_____________________________________________________________________________ void AlignableTrackerBuilder -::buildAlignables(AlignableTracker* trackerAlignables) +::buildAlignables(AlignableTracker* trackerAlignables, bool update) { - alignableMap = &trackerAlignables->alignableMap; + alignableMap_ = &trackerAlignables->alignableMap_; // first, build Alignables on module-level (AlignableDetUnits) - buildAlignableDetUnits(); + buildAlignableDetUnits(update); + // now build the composite Alignables (Ladders, Layers etc.) - buildAlignableComposites(); + buildAlignableComposites(update); + + if (update) return; // everything else not needed for the update // create pixel-detector buildPixelDetector(trackerAlignables); @@ -64,7 +67,7 @@ ::buildAlignables(AlignableTracker* trackerAlignables) buildStripDetector(trackerAlignables); // tracker itself is of course also an Alignable - alignableMap->get("Tracker").push_back(trackerAlignables); + alignableMap_->get("Tracker").push_back(trackerAlignables); // id is the id of first component (should be TPBBarrel) trackerAlignables->theId = trackerAlignables->components()[0]->id(); } @@ -77,60 +80,66 @@ ::buildAlignables(AlignableTracker* trackerAlignables) //_____________________________________________________________________________ void AlignableTrackerBuilder -::buildAlignableDetUnits() +::buildAlignableDetUnits(bool update) { // PixelBarrel convertGeomDetsToAlignables( - trackerGeometry->detsPXB(), alignableObjectId_.idToString(align::TPBModule) + trackerGeometry_->detsPXB(), alignableObjectId_.idToString(align::TPBModule), + update ); // PixelEndcap convertGeomDetsToAlignables( - trackerGeometry->detsPXF(), alignableObjectId_.idToString(align::TPEModule) + trackerGeometry_->detsPXF(), alignableObjectId_.idToString(align::TPEModule), + update ); // TIB convertGeomDetsToAlignables( - trackerGeometry->detsTIB(), alignableObjectId_.idToString(align::TIBModule) + trackerGeometry_->detsTIB(), alignableObjectId_.idToString(align::TIBModule), + update ); // TID convertGeomDetsToAlignables( - trackerGeometry->detsTID(), alignableObjectId_.idToString(align::TIDModule) + trackerGeometry_->detsTID(), alignableObjectId_.idToString(align::TIDModule), + update ); // TOB convertGeomDetsToAlignables( - trackerGeometry->detsTOB(), alignableObjectId_.idToString(align::TOBModule) + trackerGeometry_->detsTOB(), alignableObjectId_.idToString(align::TOBModule), + update ); // TEC convertGeomDetsToAlignables( - trackerGeometry->detsTEC(), alignableObjectId_.idToString(align::TECModule) + trackerGeometry_->detsTEC(), alignableObjectId_.idToString(align::TECModule), + update ); } //_____________________________________________________________________________ void AlignableTrackerBuilder ::convertGeomDetsToAlignables(const TrackingGeometry::DetContainer& geomDets, - const std::string& moduleName) + const std::string& moduleName, bool update) { numDetUnits = 0; - auto& alignables = alignableMap->get(moduleName); - alignables.reserve(geomDets.size()); + auto& alignables = alignableMap_->get(moduleName); + if (!update) alignables.reserve(geomDets.size()); // units are added for each moduleName, which are at moduleName + "Unit" // in the pixel Module and ModuleUnit are equivalent - auto & aliUnits = alignableMap->get(moduleName+"Unit"); - aliUnits.reserve(geomDets.size()); // minimal number space needed + auto & aliUnits = alignableMap_->get(moduleName+"Unit"); + if (!update) aliUnits.reserve(geomDets.size()); // minimal number space needed for (auto& geomDet : geomDets) { int subdetId = geomDet->geographicalId().subdetId(); //don't check det()==Tracker if (subdetId == PixelSubdetector::PixelBarrel || subdetId == PixelSubdetector::PixelEndcap) { - buildPixelDetectorAlignable(geomDet, subdetId, alignables, aliUnits); + buildPixelDetectorAlignable(geomDet, subdetId, alignables, aliUnits, update); } else if (subdetId == SiStripDetId::TIB || subdetId == SiStripDetId::TID || @@ -138,7 +147,7 @@ ::convertGeomDetsToAlignables(const TrackingGeometry::DetContainer& geomDets, subdetId == SiStripDetId::TEC) { // for strip we create also ModuleUnit list // for 1D components of 2D layers - buildStripDetectorAlignable(geomDet, subdetId, alignables, aliUnits); + buildStripDetectorAlignable(geomDet, subdetId, alignables, aliUnits, update); } else { throw cms::Exception("LogicError") @@ -163,7 +172,8 @@ ::convertGeomDetsToAlignables(const TrackingGeometry::DetContainer& geomDets, //_____________________________________________________________________________ void AlignableTrackerBuilder ::buildPixelDetectorAlignable(const GeomDet* geomDetUnit, int subdetId, - Alignables& aliDets, Alignables& aliDetUnits) + Alignables& aliDets, Alignables& aliDetUnits, + bool update) { // treat all pixel dets in same way with one AlignableDetUnit if (!geomDetUnit->isLeaf()) { @@ -172,15 +182,39 @@ ::buildPixelDetectorAlignable(const GeomDet* geomDetUnit, int subdetId, << ") is not a GeomDetUnit."; } - aliDets.push_back(new AlignableDetUnit(geomDetUnit)); - aliDetUnits.push_back(aliDets.back()); + if (update) { + auto ali = + std::find_if(aliDets.cbegin(), aliDets.cend(), + [&geomDetUnit](const auto& i) { + return i->id() == geomDetUnit->geographicalId().rawId(); }); + if (ali != aliDets.end()) { + // add dynamic cast here to get AlignableDetUnit! + auto aliDetUnit = dynamic_cast(*ali); + if (aliDetUnit) { + aliDetUnit->update(geomDetUnit); + } else { + throw cms::Exception("LogicError") + << "[AlignableTrackerBuilder::buildPixelDetectorAlignable] " + << "cast to 'AlignableDetUnit*' failed while it should not\n"; + } + } else { + throw cms::Exception("GeometryMismatch") + << "[AlignableTrackerBuilder::buildPixelDetectorAlignable] " + << "GeomDet with DetId " << geomDetUnit->geographicalId().rawId() + << " not found in current geometry.\n"; + } + } else { + aliDets.push_back(new AlignableDetUnit(geomDetUnit)); + aliDetUnits.push_back(aliDets.back()); + } numDetUnits += 1; } //_____________________________________________________________________________ void AlignableTrackerBuilder ::buildStripDetectorAlignable(const GeomDet* geomDet, int subdetId, - Alignables& aliDets, Alignables& aliDetUnits) + Alignables& aliDets, Alignables& aliDetUnits, + bool update) { // In strip we have: // 1) 'Pure' 1D-modules like TOB layers 3-6 (not glued): AlignableDetUnit @@ -201,17 +235,42 @@ ::buildStripDetectorAlignable(const GeomDet* geomDet, int subdetId, } // components (AlignableDetUnits) constructed within - aliDets.push_back(new AlignableSiStripDet(gluedGeomDet)); + if (update) { + auto ali = + std::find_if(aliDets.cbegin(), aliDets.cend(), + [&gluedGeomDet](const auto& i) { + return i->id() == gluedGeomDet->geographicalId().rawId(); }); + if (ali != aliDets.end()) { + auto aliSiStripDet = dynamic_cast(*ali); + if (aliSiStripDet) { + aliSiStripDet->update(gluedGeomDet); + } else { + throw cms::Exception("LogicError") + << "[AlignableTrackerBuilder::buildStripDetectorAlignable] " + << "cast to 'AlignableSiStripDet*' failed while it should not\n"; + } + } else { + throw cms::Exception("GeometryMismatch") + << "[AlignableTrackerBuilder::buildStripDetectorAlignable] " + << "GeomDet with DetId " << gluedGeomDet->geographicalId().rawId() + << " not found in current geometry.\n"; + } + } else { + aliDets.push_back(new AlignableSiStripDet(gluedGeomDet)); + } const auto& addAliDetUnits = aliDets.back()->components(); const auto& nAddedUnits = addAliDetUnits.size(); - // reserve space for the additional units: - aliDetUnits.reserve(aliDetUnits.size() + nAddedUnits -1); - aliDetUnits.insert(aliDetUnits.end(), addAliDetUnits.begin(), addAliDetUnits.end()); + + if (!update) { + // reserve space for the additional units: + aliDetUnits.reserve(aliDetUnits.size() + nAddedUnits -1); + aliDetUnits.insert(aliDetUnits.end(), addAliDetUnits.begin(), addAliDetUnits.end()); + } numDetUnits += nAddedUnits; } else { // no components: pure 1D-module - buildPixelDetectorAlignable(geomDet, subdetId, aliDets, aliDetUnits); + buildPixelDetectorAlignable(geomDet, subdetId, aliDets, aliDetUnits, update); } } // no else: glued components of AlignableDet constructed within // AlignableSiStripDet -> AlignableDet, see above @@ -221,7 +280,7 @@ ::buildStripDetectorAlignable(const GeomDet* geomDet, int subdetId, //_____________________________________________________________________________ void AlignableTrackerBuilder -::buildAlignableComposites() +::buildAlignableComposites(bool update) { unsigned int numCompositeAlignables = 0; @@ -230,7 +289,7 @@ ::buildAlignableComposites() // to get the namespace w/o building the levels auto trackerLevels = trackerAlignmentLevelBuilder_.build(); TrackerAlignableIndexer trackerIndexer{trackerAlignmentLevelBuilder_.trackerNameSpace()}; - AlignableCompositeBuilder compositeBuilder{trackerTopology, trackerGeometry, trackerIndexer}; + AlignableCompositeBuilder compositeBuilder{trackerTopology_, trackerGeometry_, trackerIndexer}; for (auto& trackerSubLevels: trackerLevels) { // first add all levels of the current subdetector to the builder @@ -238,7 +297,7 @@ ::buildAlignableComposites() compositeBuilder.addAlignmentLevel(std::move(level)); } // now build this tracker-level - numCompositeAlignables += compositeBuilder.buildAll(*alignableMap); + numCompositeAlignables += compositeBuilder.buildAll(*alignableMap_, update); // finally, reset the builder compositeBuilder.clearAlignmentLevels(); } @@ -257,9 +316,9 @@ ::buildPixelDetector(AlignableTracker* trackerAlignables) const std::string& pxeName = alignableObjectId_.idToString(align::TPEEndcap); const std::string& pixelName = alignableObjectId_.idToString(align::Pixel); - auto& pxbAlignables = alignableMap->find(pxbName); - auto& pxeAlignables = alignableMap->find(pxeName); - auto& pixelAlignables = alignableMap->get (pixelName); + auto& pxbAlignables = alignableMap_->find(pxbName); + auto& pxeAlignables = alignableMap_->find(pxeName); + auto& pixelAlignables = alignableMap_->get (pixelName); pixelAlignables.push_back( new AlignableComposite(pxbAlignables[0]->id(), align::Pixel, align::RotationType()) @@ -287,11 +346,11 @@ ::buildStripDetector(AlignableTracker* trackerAlignables) const std::string& tecName = alignableObjectId_.idToString(align::TECEndcap); const std::string& stripName = alignableObjectId_.idToString(align::Strip); - auto& tibAlignables = alignableMap->find(tibName); - auto& tidAlignables = alignableMap->find(tidName); - auto& tobAlignables = alignableMap->find(tobName); - auto& tecAlignables = alignableMap->find(tecName); - auto& stripAlignables = alignableMap->get (stripName); + auto& tibAlignables = alignableMap_->find(tibName); + auto& tidAlignables = alignableMap_->find(tidName); + auto& tobAlignables = alignableMap_->find(tobName); + auto& tecAlignables = alignableMap_->find(tecName); + auto& stripAlignables = alignableMap_->get (stripName); stripAlignables.push_back( new AlignableComposite(tibAlignables[0]->id(), align::Strip, align::RotationType()) diff --git a/Alignment/TrackerAlignment/test/alignment_forGeomComp_cfg_TEMPLATE.py b/Alignment/TrackerAlignment/test/alignment_forGeomComp_cfg_TEMPLATE.py index 9d00cf690927e..13a90cee7724f 100644 --- a/Alignment/TrackerAlignment/test/alignment_forGeomComp_cfg_TEMPLATE.py +++ b/Alignment/TrackerAlignment/test/alignment_forGeomComp_cfg_TEMPLATE.py @@ -24,7 +24,7 @@ # take your favourite global tag from Configuration.AlCa.GlobalTag import GlobalTag process.GlobalTag = GlobalTag(process.GlobalTag, "GLOBALTAG") -usedGlobalTag = process.GlobalTag.globaltag._value +usedGlobalTag = process.GlobalTag.globaltag.value() process.load("FWCore.MessageService.MessageLogger_cfi") process.MessageLogger.LOGFILE = cms.untracked.PSet( @@ -69,6 +69,17 @@ ) ) +# explicitely specify run ranges to convince algorithm that multi-IOV input is fine +process.AlignmentProducer.RunRangeSelection = [ + cms.PSet( + RunRanges = cms.vstring("RUNNUMBER"), + selector = process.AlignmentProducer.ParameterBuilder.Selector.alignParams + ) +] # end of process.AlignmentProducer.RunRangeSelection + +# enable alignable updates to convince algorithm that multi-IOV input is fine +process.AlignmentProducer.enableAlignableUpdates = True + process.AlignmentProducer.doMisalignmentScenario = False #True process.AlignmentProducer.applyDbAlignment = True # either globalTag or trackerAlignment process.AlignmentProducer.checkDbAlignmentValidity = False