From 4b8d2bb97aed7dc3585ef6b2c0904b3482d621ab Mon Sep 17 00:00:00 2001 From: Gregor Mittag Date: Thu, 26 May 2016 16:41:15 +0200 Subject: [PATCH 1/8] Added 'Mille::resetOutputFile()'. - first step towards binary FileBlob as lumi product - general clean-up of the 'Mille' class --- .../MillePedeAlignmentAlgorithm/src/Mille.cc | 173 ++++++++++-------- .../MillePedeAlignmentAlgorithm/src/Mille.h | 31 ++-- 2 files changed, 112 insertions(+), 92 deletions(-) diff --git a/Alignment/MillePedeAlignmentAlgorithm/src/Mille.cc b/Alignment/MillePedeAlignmentAlgorithm/src/Mille.cc index 5bc1254ecb72a..e34d124014c63 100644 --- a/Alignment/MillePedeAlignmentAlgorithm/src/Mille.cc +++ b/Alignment/MillePedeAlignmentAlgorithm/src/Mille.cc @@ -10,23 +10,26 @@ #include "Mille.h" #include +#include "FWCore/MessageLogger/interface/MessageLogger.h" //___________________________________________________________________________ -Mille::Mille(const char *outFileName, bool asBinary, bool writeZero) : - myOutFile(outFileName, (asBinary ? (std::ios::binary | std::ios::out) : std::ios::out)), - myAsBinary(asBinary), myWriteZero(writeZero), myBufferPos(-1), myHasSpecial(false) +Mille::Mille(const char *outFileName, bool asBinary, bool writeZero) : + fileMode_(asBinary ? (std::ios::binary | std::ios::out) : std::ios::out), + fileName_(outFileName), + outFile_(fileName_, fileMode_), + asBinary_(asBinary), writeZero_(writeZero), bufferPos_(-1), hasSpecial_(false) { // opens outFileName, by default as binary file - // Instead myBufferPos(-1), myHasSpecial(false) and the following two lines + // Instead bufferPos_(-1), hasSpecial_(false) and the following two lines // we could call newSet() and kill()... - myBufferInt[0] = 0; - myBufferFloat[0] = 0.; + bufferInt_[0] = 0; + bufferFloat_[0] = 0.; - if (!myOutFile.is_open()) { - std::cerr << "Mille::Mille: Could not open " << outFileName - << " as output file." << std::endl; + if (!outFile_.is_open()) { + edm::LogError("Alignment") + << "Mille::Mille: Could not open " << fileName_ << " as output file."; } } @@ -35,48 +38,49 @@ Mille::Mille(const char *outFileName, bool asBinary, bool writeZero) : Mille::~Mille() { // closes file - myOutFile.close(); + outFile_.close(); } //___________________________________________________________________________ void Mille::mille(int NLC, const float *derLc, - int NGL, const float *derGl, const int *label, - float rMeas, float sigma) + int NGL, const float *derGl, const int *label, + float rMeas, float sigma) { if (sigma <= 0.) return; - if (myBufferPos == -1) this->newSet(); // start, e.g. new track + if (bufferPos_ == -1) this->newSet(); // start, e.g. new track if (!this->checkBufferSize(NLC, NGL)) return; // first store measurement - ++myBufferPos; - myBufferFloat[myBufferPos] = rMeas; - myBufferInt [myBufferPos] = 0; + ++bufferPos_; + bufferFloat_[bufferPos_] = rMeas; + bufferInt_ [bufferPos_] = 0; // store local derivatives and local 'lables' 1,...,NLC for (int i = 0; i < NLC; ++i) { - if (derLc[i] || myWriteZero) { // by default store only non-zero derivatives - ++myBufferPos; - myBufferFloat[myBufferPos] = derLc[i]; // local derivatives - myBufferInt [myBufferPos] = i+1; // index of local parameter + if (derLc[i] || writeZero_) { // by default store only non-zero derivatives + ++bufferPos_; + bufferFloat_[bufferPos_] = derLc[i]; // local derivatives + bufferInt_ [bufferPos_] = i+1; // index of local parameter } } // store uncertainty of measurement in between locals and globals - ++myBufferPos; - myBufferFloat[myBufferPos] = sigma; - myBufferInt [myBufferPos] = 0; + ++bufferPos_; + bufferFloat_[bufferPos_] = sigma; + bufferInt_ [bufferPos_] = 0; // store global derivatives and their lables for (int i = 0; i < NGL; ++i) { - if (derGl[i] || myWriteZero) { // by default store only non-zero derivatives - if ((label[i] > 0 || myWriteZero) && label[i] <= myMaxLabel) { // and for valid labels - ++myBufferPos; - myBufferFloat[myBufferPos] = derGl[i]; // global derivatives - myBufferInt [myBufferPos] = label[i]; // index of global parameter + if (derGl[i] || writeZero_) { // by default store only non-zero derivatives + if ((label[i] > 0 || writeZero_) && label[i] <= maxLabel_) { // and for valid labels + ++bufferPos_; + bufferFloat_[bufferPos_] = derGl[i]; // global derivatives + bufferInt_ [bufferPos_] = label[i]; // index of global parameter } else { - std::cerr << "Mille::mille: Invalid label " << label[i] - << " <= 0 or > " << myMaxLabel << std::endl; + edm::LogError("Alignment") + << "Mille::mille: Invalid label " << label[i] + << " <= 0 or > " << maxLabel_; } } } @@ -86,33 +90,33 @@ void Mille::mille(int NLC, const float *derLc, void Mille::special(int nSpecial, const float *floatings, const int *integers) { if (nSpecial == 0) return; - if (myBufferPos == -1) this->newSet(); // start, e.g. new track - if (myHasSpecial) { - std::cerr << "Mille::special: Special values already stored for this record." - << std::endl; + if (bufferPos_ == -1) this->newSet(); // start, e.g. new track + if (hasSpecial_) { + edm::LogError("Alignment") + << "Mille::special: Special values already stored for this record."; return; } if (!this->checkBufferSize(nSpecial, 0)) return; - myHasSpecial = true; // after newSet() (Note: MILLSP sets to buffer position...) + hasSpecial_ = true; // after newSet() (Note: MILLSP sets to buffer position...) - // myBufferFloat[.] | myBufferInt[.] + // bufferFloat_[.] | bufferInt_[.] // ------------------------------------ // 0.0 | 0 // -float(nSpecial) | 0 // The above indicates special data, following are nSpecial floating and nSpecial integer data. - ++myBufferPos; // zero pair - myBufferFloat[myBufferPos] = 0.; - myBufferInt [myBufferPos] = 0; + ++bufferPos_; // zero pair + bufferFloat_[bufferPos_] = 0.; + bufferInt_ [bufferPos_] = 0; - ++myBufferPos; // nSpecial and zero - myBufferFloat[myBufferPos] = -nSpecial; // automatic conversion to float - myBufferInt [myBufferPos] = 0; + ++bufferPos_; // nSpecial and zero + bufferFloat_[bufferPos_] = -nSpecial; // automatic conversion to float + bufferInt_ [bufferPos_] = 0; for (int i = 0; i < nSpecial; ++i) { - ++myBufferPos; - myBufferFloat[myBufferPos] = floatings[i]; - myBufferInt [myBufferPos] = integers[i]; + ++bufferPos_; + bufferFloat_[bufferPos_] = floatings[i]; + bufferInt_ [bufferPos_] = integers[i]; } } @@ -121,7 +125,7 @@ void Mille::special(int nSpecial, const float *floatings, const int *integers) void Mille::kill() { // reset buffers, i.e. kill derivatives accumulated for current set - myBufferPos = -1; + bufferPos_ = -1; } //___________________________________________________________________________ @@ -129,7 +133,20 @@ void Mille::kill() void Mille::flushOutputFile() { // flush output file - myOutFile.flush(); + outFile_.flush(); +} + +//___________________________________________________________________________ + + +void Mille::resetOutputFile() { + // flush output file + outFile_.close(); + outFile_.open(fileName_, fileMode_); + if (!outFile_.is_open()) { + edm::LogError("Alignment") + << "Mille::resetOutputFile: Could not reopen " << fileName_ << "."; + } } //___________________________________________________________________________ @@ -137,30 +154,30 @@ void Mille::flushOutputFile() { void Mille::end() { // write set of derivatives with same local parameters to file - if (myBufferPos > 0) { // only if anything stored... - const int numWordsToWrite = (myBufferPos + 1)*2; - - if (myAsBinary) { - myOutFile.write(reinterpret_cast(&numWordsToWrite), - sizeof(numWordsToWrite)); - myOutFile.write(reinterpret_cast(myBufferFloat), - (myBufferPos+1) * sizeof(myBufferFloat[0])); - myOutFile.write(reinterpret_cast(myBufferInt), - (myBufferPos+1) * sizeof(myBufferInt[0])); + if (bufferPos_ > 0) { // only if anything stored... + const int numWordsToWrite = (bufferPos_ + 1)*2; + + if (asBinary_) { + outFile_.write(reinterpret_cast(&numWordsToWrite), + sizeof(numWordsToWrite)); + outFile_.write(reinterpret_cast(bufferFloat_), + (bufferPos_+1) * sizeof(bufferFloat_[0])); + outFile_.write(reinterpret_cast(bufferInt_), + (bufferPos_+1) * sizeof(bufferInt_[0])); } else { - myOutFile << numWordsToWrite << "\n"; - for (int i = 0; i < myBufferPos+1; ++i) { - myOutFile << myBufferFloat[i] << " "; + outFile_ << numWordsToWrite << "\n"; + for (int i = 0; i < bufferPos_+1; ++i) { + outFile_ << bufferFloat_[i] << " "; } - myOutFile << "\n"; - - for (int i = 0; i < myBufferPos+1; ++i) { - myOutFile << myBufferInt[i] << " "; + outFile_ << "\n"; + + for (int i = 0; i < bufferPos_+1; ++i) { + outFile_ << bufferInt_[i] << " "; } - myOutFile << "\n"; + outFile_ << "\n"; } } - myBufferPos = -1; // reset buffer for next set of derivatives + bufferPos_ = -1; // reset buffer for next set of derivatives } //___________________________________________________________________________ @@ -168,10 +185,10 @@ void Mille::end() void Mille::newSet() { // initilise for new set of locals, e.g. new track - myBufferPos = 0; - myHasSpecial = false; - myBufferFloat[0] = 0.0; - myBufferInt [0] = 0; // position 0 used as error counter + bufferPos_ = 0; + hasSpecial_ = false; + bufferFloat_[0] = 0.0; + bufferInt_ [0] = 0; // position 0 used as error counter } //___________________________________________________________________________ @@ -180,14 +197,14 @@ bool Mille::checkBufferSize(int nLocal, int nGlobal) { // enough space for next nLocal + nGlobal derivatives incl. measurement? - if (myBufferPos + nLocal + nGlobal + 2 >= myBufferSize) { - ++(myBufferInt[0]); // increase error count - std::cerr << "Mille::checkBufferSize: Buffer too short (" - << myBufferSize << ")," - << "\n need space for nLocal (" << nLocal<< ")" - << "/nGlobal (" << nGlobal << ") local/global derivatives, " - << myBufferPos + 1 << " already stored!" - << std::endl; + if (bufferPos_ + nLocal + nGlobal + 2 >= bufferSize_) { + ++(bufferInt_[0]); // increase error count + edm::LogError("Alignment") + << "Mille::checkBufferSize: Buffer too short (" + << bufferSize_ << ")," + << "\n need space for nLocal (" << nLocal<< ")" + << "/nGlobal (" << nGlobal << ") local/global derivatives, " + << bufferPos_ + 1 << " already stored!"; return false; } else { return true; diff --git a/Alignment/MillePedeAlignmentAlgorithm/src/Mille.h b/Alignment/MillePedeAlignmentAlgorithm/src/Mille.h index c1774a173f046..30cb244233391 100644 --- a/Alignment/MillePedeAlignmentAlgorithm/src/Mille.h +++ b/Alignment/MillePedeAlignmentAlgorithm/src/Mille.h @@ -9,11 +9,11 @@ * Class to write a C binary (cf. below) file of a given name and to fill it * with information used as input to pede. * Use its member functions mille(...), special(...), kill() and end() as you would - * use the fortran MILLE and its entry points MILLSP, KILLE and ENDLE. + * use the fortran MILLE and its entry points MILLSP, KILLE and ENDLE. * * For debugging purposes constructor flags enable switching to text output and/or * to write also derivatives and lables which are ==0. - * But note that pede will not be able to read text output and has not been tested with + * But note that pede will not be able to read text output and has not been tested with * derivatives/labels ==0. * * \author : Gero Flucke @@ -23,33 +23,36 @@ * (last update by $Author: flucke $) */ -class Mille +class Mille { public: Mille(const char *outFileName, bool asBinary = true, bool writeZero = false); ~Mille(); void mille(int NLC, const float *derLc, int NGL, const float *derGl, - const int *label, float rMeas, float sigma); + const int *label, float rMeas, float sigma); void special(int nSpecial, const float *floatings, const int *integers); void kill(); - void flushOutputFile(); + void flushOutputFile(); + void resetOutputFile(); void end(); private: void newSet(); bool checkBufferSize(int nLocal, int nGlobal); - std::ofstream myOutFile; // C-binary for output - bool myAsBinary; // if false output as text - bool myWriteZero; // if true also write out derivatives/lables ==0 + const std::ios_base::openmode fileMode_; // file open mode of the binary + const std::string fileName_; // file name of the binary + std::ofstream outFile_; // C-binary for output + bool asBinary_; // if false output as text + bool writeZero_; // if true also write out derivatives/lables ==0 - enum {myBufferSize = 5000}; - int myBufferInt[myBufferSize]; // to collect labels etc. - float myBufferFloat[myBufferSize]; // to collect derivatives etc. - int myBufferPos; - bool myHasSpecial; // if true, special(..) already called for this record + enum {bufferSize_ = 5000}; + int bufferInt_[bufferSize_]; // to collect labels etc. + float bufferFloat_[bufferSize_]; // to collect derivatives etc. + int bufferPos_; + bool hasSpecial_; // if true, special(..) already called for this record - enum {myMaxLabel = (0xFFFFFFFF - (1 << 31))}; // largest label allowed: 2^31 - 1 + enum {maxLabel_ = (0xFFFFFFFF - (1 << 31))}; // largest label allowed: 2^31 - 1 }; #endif From df29429a5f1100326ae9463103769f7092fc7ae6 Mon Sep 17 00:00:00 2001 From: Gregor Mittag Date: Thu, 26 May 2016 17:06:33 +0200 Subject: [PATCH 2/8] Implement luminosity block begin and end methods for MillePede. --- .../interface/MillePedeAlignmentAlgorithm.h | 6 ++++++ .../src/MillePedeAlignmentAlgorithm.cc | 12 ++++++++++++ 2 files changed, 18 insertions(+) diff --git a/Alignment/MillePedeAlignmentAlgorithm/interface/MillePedeAlignmentAlgorithm.h b/Alignment/MillePedeAlignmentAlgorithm/interface/MillePedeAlignmentAlgorithm.h index fbf3ddca7fa20..421bd2bdc2ca4 100644 --- a/Alignment/MillePedeAlignmentAlgorithm/interface/MillePedeAlignmentAlgorithm.h +++ b/Alignment/MillePedeAlignmentAlgorithm/interface/MillePedeAlignmentAlgorithm.h @@ -90,6 +90,12 @@ class MillePedeAlignmentAlgorithm : public AlignmentAlgorithmBase // This one will be called since it matches the interface of the base class virtual void endRun(const EndRunInfo &runInfo, const edm::EventSetup &setup); + /// called at begin of luminosity block (resets Mille binary in mille mode) + virtual void beginLuminosityBlock(const edm::EventSetup&) override; + + /// called at end of luminosity block + virtual void endLuminosityBlock(const edm::EventSetup&) override; + /* virtual void beginLuminosityBlock(const edm::EventSetup &setup) {} */ /* virtual void endLuminosityBlock(const edm::EventSetup &setup) {} */ diff --git a/Alignment/MillePedeAlignmentAlgorithm/src/MillePedeAlignmentAlgorithm.cc b/Alignment/MillePedeAlignmentAlgorithm/src/MillePedeAlignmentAlgorithm.cc index 7f385cdaa68b7..b58eeef772aa7 100644 --- a/Alignment/MillePedeAlignmentAlgorithm/src/MillePedeAlignmentAlgorithm.cc +++ b/Alignment/MillePedeAlignmentAlgorithm/src/MillePedeAlignmentAlgorithm.cc @@ -530,6 +530,18 @@ void MillePedeAlignmentAlgorithm::endRun(const EndRunInfo &runInfo, const edm::E if(this->isMode(myMilleBit)) theMille->flushOutputFile(); } +//____________________________________________________ +void MillePedeAlignmentAlgorithm::beginLuminosityBlock(const edm::EventSetup&) +{ + if(this->isMode(myMilleBit)) theMille->resetOutputFile(); +} + +//____________________________________________________ +void MillePedeAlignmentAlgorithm::endLuminosityBlock(const edm::EventSetup&) +{ + if(this->isMode(myMilleBit)) theMille->flushOutputFile(); +} + //____________________________________________________ int MillePedeAlignmentAlgorithm::addMeasurementData(const edm::EventSetup &setup, From 18f53e6049741d324529583adaf5ccf727fbb265 Mon Sep 17 00:00:00 2001 From: Gregor Mittag Date: Thu, 26 May 2016 18:22:28 +0200 Subject: [PATCH 3/8] Make the producer and the analyzer of the binary FileBlobCollection lumi-based. - updated 'MillePedeFileExtractor' to edm::one::Analyzer - general clean-up of the 'MillePedeFileConverter' and 'MillePedeFileExtractor' --- .../plugins/MillePedeFileConverter.cc | 33 +++++++++-------- .../plugins/MillePedeFileConverter.h | 12 +++---- .../plugins/MillePedeFileExtractor.cc | 35 +++++++++---------- .../plugins/MillePedeFileExtractor.h | 17 +++++---- 4 files changed, 49 insertions(+), 48 deletions(-) diff --git a/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileConverter.cc b/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileConverter.cc index c7db44deef9a2..661bc414f559c 100644 --- a/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileConverter.cc +++ b/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileConverter.cc @@ -10,41 +10,40 @@ #include MillePedeFileConverter::MillePedeFileConverter(const edm::ParameterSet& iConfig) - : theInputDir(iConfig.getParameter("fileDir")), - theInputFileName(iConfig.getParameter("inputBinaryFile")), - theFileBlobLabel(iConfig.getParameter("fileBlobLabel")) { + : inputDir_(iConfig.getParameter("fileDir")), + inputFileName_(iConfig.getParameter("inputBinaryFile")), + fileBlobLabel_(iConfig.getParameter("fileBlobLabel")) { // We define what this producer produces: A FileBlobCollection - produces(theFileBlobLabel); + produces(fileBlobLabel_); } MillePedeFileConverter::~MillePedeFileConverter() {} -void MillePedeFileConverter::endRunProduce(edm::Run& iRun, - const edm::EventSetup& iSetup) { +void MillePedeFileConverter::endLuminosityBlockProduce(edm::LuminosityBlock& iLumi, + const edm::EventSetup& iSetup) { edm::LogInfo("MillePedeFileActions") - << "Inserting all data from file " << theInputDir + theInputFileName - << " as a FileBlob to the run, using label \"" << theFileBlobLabel + << "Inserting all data from file " << inputDir_ + inputFileName_ + << " as a FileBlob to the lumi, using label \"" << fileBlobLabel_ << "\"."; // Preparing the FileBlobCollection: - std::unique_ptr theFileBlobCollection( - new FileBlobCollection()); - FileBlob theFileBlob; + auto fileBlobCollection = std::make_unique(); + FileBlob fileBlob; try { // Creating the FileBlob: // (The FileBlob will signal problems with the file itself.) - theFileBlob = FileBlob(theInputDir + theInputFileName, true); + fileBlob = FileBlob(inputDir_ + inputFileName_, true); } catch (...) { // When creation of the FileBlob fails: edm::LogError("MillePedeFileActions") << "Error: No FileBlob could be created from the file \"" - << theInputDir + theInputFileName << "\"."; + << inputDir_ + inputFileName_ << "\"."; throw; } - if (theFileBlob.size() > 0) { - // Adding the FileBlob to the run: - theFileBlobCollection->addFileBlob(theFileBlob); - iRun.put(std::move(theFileBlobCollection), theFileBlobLabel); + if (fileBlob.size() > 0) { + // Adding the FileBlob to the lumi: + fileBlobCollection->addFileBlob(fileBlob); + iLumi.put(std::move(fileBlobCollection), fileBlobLabel_); } } diff --git a/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileConverter.h b/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileConverter.h index f9e0d074c30f4..9613820ca2393 100644 --- a/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileConverter.h +++ b/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileConverter.h @@ -26,7 +26,7 @@ #include "FWCore/ParameterSet/interface/ParameterSet.h" class MillePedeFileConverter - : public edm::one::EDProducer { + : public edm::one::EDProducer { public: explicit MillePedeFileConverter(const edm::ParameterSet&); ~MillePedeFileConverter(); @@ -34,12 +34,12 @@ class MillePedeFileConverter private: virtual void produce(edm::Event&, const edm::EventSetup&) override {} - virtual void endRunProduce(edm::Run& run, - const edm::EventSetup& iSetup) override final; + virtual void endLuminosityBlockProduce(edm::LuminosityBlock&, + const edm::EventSetup&) override final; - std::string theInputDir; - std::string theInputFileName; - std::string theFileBlobLabel; + const std::string inputDir_; + const std::string inputFileName_; + const std::string fileBlobLabel_; }; // define this as a plug-in diff --git a/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileExtractor.cc b/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileExtractor.cc index 0f2673540a0b2..dfce80880f2e1 100644 --- a/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileExtractor.cc +++ b/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileExtractor.cc @@ -1,8 +1,6 @@ // Original Author: Broen van Besien // Created: Mon, 23 Mar 2015 14:56:15 GMT -#include - #include "Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileExtractor.h" #include "FWCore/MessageLogger/interface/MessageLogger.h" @@ -11,43 +9,44 @@ #include "FWCore/Utilities/interface/EDGetToken.h" MillePedeFileExtractor::MillePedeFileExtractor(const edm::ParameterSet& iConfig) - : theOutputDir(iConfig.getParameter("fileDir")), - theOutputFileName(iConfig.getParameter("outputBinaryFile")) { + : outputDir_(iConfig.getParameter("fileDir")), + outputFileName_(iConfig.getParameter("outputBinaryFile")) { - edm::InputTag fileBlobInputTag = iConfig.getParameter("fileBlobInputTag"); - theFileBlobToken = consumes(fileBlobInputTag); + auto fileBlobInputTag = iConfig.getParameter("fileBlobInputTag"); + fileBlobToken_ = consumes(fileBlobInputTag); // nothing else in the constructor } MillePedeFileExtractor::~MillePedeFileExtractor() {} -void MillePedeFileExtractor::endRun(const edm::Run& iRun, - edm::EventSetup const&) { +void MillePedeFileExtractor::endLuminosityBlock(const edm::LuminosityBlock& iLumi, + const edm::EventSetup&) +{ // Getting our hands on the vector of FileBlobs - edm::Handle theFileBlobCollection; - iRun.getByToken(theFileBlobToken, theFileBlobCollection); - if (theFileBlobCollection.isValid()) { + edm::Handle fileBlobCollection; + iLumi.getByToken(fileBlobToken_, fileBlobCollection); + if (fileBlobCollection.isValid()) { // Logging the amount of FileBlobs in the vector - int theVectorSize = theFileBlobCollection->size(); + int theVectorSize = fileBlobCollection->size(); edm::LogInfo("MillePedeFileActions") << "Root file contains " << theVectorSize << " FileBlob(s)."; // Loop over the FileBlobs in the vector, and write them to files: for (std::vector::const_iterator it = - theFileBlobCollection->begin(); - it != theFileBlobCollection->end(); ++it) { + fileBlobCollection->begin(); + it != fileBlobCollection->end(); ++it) { // We format the filename with a number, starting from 0 to the size of // our vector. // For this to work, the outputBinaryFile config parameter must contain a // formatting directive for a number, like %04d. char theNumberedOutputFileName[200]; - int theNumber = it - theFileBlobCollection->begin(); - sprintf(theNumberedOutputFileName, theOutputFileName.c_str(), theNumber); + int theNumber = it - fileBlobCollection->begin(); + sprintf(theNumberedOutputFileName, outputFileName_.c_str(), theNumber); // Log the filename to which we will write... edm::LogInfo("MillePedeFileActions") << "Writing FileBlob file to file " - << theOutputDir + theNumberedOutputFileName << "."; + << outputDir_ + theNumberedOutputFileName << "."; // ...and perform the writing operation. - it->write(theOutputDir + theNumberedOutputFileName); + it->write(outputDir_ + theNumberedOutputFileName); // Carefull, it seems that when writing to an impossible file, this is // swallowed by the FileBlob->write operation and no error is thrown. } diff --git a/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileExtractor.h b/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileExtractor.h index 2d27e0ec681c0..2105b3dd555d3 100644 --- a/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileExtractor.h +++ b/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileExtractor.h @@ -22,26 +22,29 @@ */ #include "FWCore/Framework/interface/Frameworkfwd.h" -#include "FWCore/Framework/interface/EDAnalyzer.h" +#include "FWCore/Framework/interface/one/EDAnalyzer.h" #include "FWCore/Framework/interface/Event.h" #include "FWCore/Framework/interface/MakerMacros.h" #include "FWCore/ParameterSet/interface/ParameterSet.h" #include "CondFormats/Common/interface/FileBlobCollection.h" -class MillePedeFileExtractor : public edm::EDAnalyzer { +class MillePedeFileExtractor : public edm::one::EDAnalyzer { public: explicit MillePedeFileExtractor(const edm::ParameterSet&); ~MillePedeFileExtractor(); static void fillDescriptions(edm::ConfigurationDescriptions& descriptions); private: - virtual void endRun(edm::Run const&, edm::EventSetup const&) override; - void analyze(const edm::Event&, const edm::EventSetup&) {} + virtual void beginLuminosityBlock(const edm::LuminosityBlock&, + const edm::EventSetup&) override {} + virtual void endLuminosityBlock(const edm::LuminosityBlock&, + const edm::EventSetup&) override; + virtual void analyze(const edm::Event&, const edm::EventSetup&) override {} - std::string theOutputDir; - std::string theOutputFileName; + const std::string outputDir_; + const std::string outputFileName_; - edm::EDGetTokenT theFileBlobToken; + edm::EDGetTokenT fileBlobToken_; }; From ae6be4f18adecc6dec35e057fa5a94c6c6313d84 Mon Sep 17 00:00:00 2001 From: Gregor Mittag Date: Thu, 26 May 2016 18:33:25 +0200 Subject: [PATCH 4/8] Add a switch to select between offline use and PCL. - at PCL the binaries are reset at lumi-section boundaries -> this is not wanted for the offline use where one binary is produced per mille job -> default value of 'runAtPCL' is False --- .../python/ALCARECOPromptCalibProdSiPixelAli0T_cff.py | 1 + .../python/ALCARECOPromptCalibProdSiPixelAli_cff.py | 1 + .../interface/MillePedeAlignmentAlgorithm.h | 2 ++ .../python/MillePedeAlignmentAlgorithm_cfi.py | 2 ++ .../src/MillePedeAlignmentAlgorithm.cc | 5 ++++- 5 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Alignment/CommonAlignmentProducer/python/ALCARECOPromptCalibProdSiPixelAli0T_cff.py b/Alignment/CommonAlignmentProducer/python/ALCARECOPromptCalibProdSiPixelAli0T_cff.py index a9e4f64ff07bc..39c838c73a326 100644 --- a/Alignment/CommonAlignmentProducer/python/ALCARECOPromptCalibProdSiPixelAli0T_cff.py +++ b/Alignment/CommonAlignmentProducer/python/ALCARECOPromptCalibProdSiPixelAli0T_cff.py @@ -99,6 +99,7 @@ SiPixelAliMilleAlignmentProducer.algoConfig = MillePedeAlignmentAlgorithm SiPixelAliMilleAlignmentProducer.algoConfig.mode = 'mille' +SiPixelAliMilleAlignmentProducer.algoConfig.runAtPCL = True SiPixelAliMilleAlignmentProducer.algoConfig.mergeBinaryFiles = cms.vstring() SiPixelAliMilleAlignmentProducer.algoConfig.binaryFile = 'milleBinary_0.dat' SiPixelAliMilleAlignmentProducer.algoConfig.TrajectoryFactory = cms.PSet( diff --git a/Alignment/CommonAlignmentProducer/python/ALCARECOPromptCalibProdSiPixelAli_cff.py b/Alignment/CommonAlignmentProducer/python/ALCARECOPromptCalibProdSiPixelAli_cff.py index 22b54c6f3fe37..6162d8c7cbab4 100644 --- a/Alignment/CommonAlignmentProducer/python/ALCARECOPromptCalibProdSiPixelAli_cff.py +++ b/Alignment/CommonAlignmentProducer/python/ALCARECOPromptCalibProdSiPixelAli_cff.py @@ -97,6 +97,7 @@ SiPixelAliMilleAlignmentProducer.algoConfig = MillePedeAlignmentAlgorithm SiPixelAliMilleAlignmentProducer.algoConfig.mode = 'mille' +SiPixelAliMilleAlignmentProducer.algoConfig.runAtPCL = True SiPixelAliMilleAlignmentProducer.algoConfig.mergeBinaryFiles = cms.vstring() SiPixelAliMilleAlignmentProducer.algoConfig.binaryFile = 'milleBinary_0.dat' SiPixelAliMilleAlignmentProducer.algoConfig.TrajectoryFactory = cms.PSet( diff --git a/Alignment/MillePedeAlignmentAlgorithm/interface/MillePedeAlignmentAlgorithm.h b/Alignment/MillePedeAlignmentAlgorithm/interface/MillePedeAlignmentAlgorithm.h index 421bd2bdc2ca4..0327be43bab15 100644 --- a/Alignment/MillePedeAlignmentAlgorithm/interface/MillePedeAlignmentAlgorithm.h +++ b/Alignment/MillePedeAlignmentAlgorithm/interface/MillePedeAlignmentAlgorithm.h @@ -260,6 +260,8 @@ class MillePedeAlignmentAlgorithm : public AlignmentAlgorithmBase // CHK for GBL std::unique_ptr theBinary; bool theGblDoubleBinary; + + const bool runAtPCL_; }; #endif diff --git a/Alignment/MillePedeAlignmentAlgorithm/python/MillePedeAlignmentAlgorithm_cfi.py b/Alignment/MillePedeAlignmentAlgorithm/python/MillePedeAlignmentAlgorithm_cfi.py index 59c04b085bcd5..2b262471cd371 100644 --- a/Alignment/MillePedeAlignmentAlgorithm/python/MillePedeAlignmentAlgorithm_cfi.py +++ b/Alignment/MillePedeAlignmentAlgorithm/python/MillePedeAlignmentAlgorithm_cfi.py @@ -21,6 +21,8 @@ monitorFile = cms.untracked.string('millePedeMonitor.root'), ## if empty: no monitoring... + runAtPCL = cms.bool(False), # at the PCL the mille binaries are reset at lumi-section boundaries + # PSet that allows to configure the pede labeler, i.e. select the actual # labeler plugin to use and parameters for the selected plugin pedeLabeler = cms.PSet( diff --git a/Alignment/MillePedeAlignmentAlgorithm/src/MillePedeAlignmentAlgorithm.cc b/Alignment/MillePedeAlignmentAlgorithm/src/MillePedeAlignmentAlgorithm.cc index b58eeef772aa7..febf9d24ab2e5 100644 --- a/Alignment/MillePedeAlignmentAlgorithm/src/MillePedeAlignmentAlgorithm.cc +++ b/Alignment/MillePedeAlignmentAlgorithm/src/MillePedeAlignmentAlgorithm.cc @@ -95,7 +95,8 @@ MillePedeAlignmentAlgorithm::MillePedeAlignmentAlgorithm(const edm::ParameterSet theMinNumHits(cfg.getParameter("minNumHits")), theMaximalCor2D(cfg.getParameter("max2Dcorrelation")), theLastWrittenIov(0), - theGblDoubleBinary(cfg.getParameter("doubleBinary")) + theGblDoubleBinary(cfg.getParameter("doubleBinary")), + runAtPCL_(cfg.getParameter("runAtPCL")) { if (!theDir.empty() && theDir.find_last_of('/') != theDir.size()-1) theDir += '/';// may need '/' edm::LogInfo("Alignment") << "@SUB=MillePedeAlignmentAlgorithm" << "Start in mode '" @@ -533,12 +534,14 @@ void MillePedeAlignmentAlgorithm::endRun(const EndRunInfo &runInfo, const edm::E //____________________________________________________ void MillePedeAlignmentAlgorithm::beginLuminosityBlock(const edm::EventSetup&) { + if (!runAtPCL_) return; if(this->isMode(myMilleBit)) theMille->resetOutputFile(); } //____________________________________________________ void MillePedeAlignmentAlgorithm::endLuminosityBlock(const edm::EventSetup&) { + if (!runAtPCL_) return; if(this->isMode(myMilleBit)) theMille->flushOutputFile(); } From 777955c0facc8c37f002008cef63ade9b2a5e1cb Mon Sep 17 00:00:00 2001 From: Gregor Mittag Date: Fri, 27 May 2016 10:00:42 +0200 Subject: [PATCH 5/8] Fix binary numbering for lumi-based MillePedeFileExtractor. --- .../plugins/MillePedeFileExtractor.cc | 19 ++++++++----------- .../plugins/MillePedeFileExtractor.h | 4 +++- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileExtractor.cc b/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileExtractor.cc index dfce80880f2e1..056e701391dba 100644 --- a/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileExtractor.cc +++ b/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileExtractor.cc @@ -27,28 +27,25 @@ void MillePedeFileExtractor::endLuminosityBlock(const edm::LuminosityBlock& iLum iLumi.getByToken(fileBlobToken_, fileBlobCollection); if (fileBlobCollection.isValid()) { // Logging the amount of FileBlobs in the vector - int theVectorSize = fileBlobCollection->size(); - edm::LogInfo("MillePedeFileActions") << "Root file contains " - << theVectorSize << " FileBlob(s)."; + edm::LogInfo("MillePedeFileActions") + << "Root file contains " << fileBlobCollection->size() << " FileBlob(s)."; // Loop over the FileBlobs in the vector, and write them to files: - for (std::vector::const_iterator it = - fileBlobCollection->begin(); - it != fileBlobCollection->end(); ++it) { + for (const auto& blob: *fileBlobCollection) { // We format the filename with a number, starting from 0 to the size of // our vector. // For this to work, the outputBinaryFile config parameter must contain a // formatting directive for a number, like %04d. char theNumberedOutputFileName[200]; - int theNumber = it - fileBlobCollection->begin(); - sprintf(theNumberedOutputFileName, outputFileName_.c_str(), theNumber); + sprintf(theNumberedOutputFileName, outputFileName_.c_str(), nBinaries_); // Log the filename to which we will write... edm::LogInfo("MillePedeFileActions") << "Writing FileBlob file to file " << outputDir_ + theNumberedOutputFileName << "."; // ...and perform the writing operation. - it->write(outputDir_ + theNumberedOutputFileName); - // Carefull, it seems that when writing to an impossible file, this is - // swallowed by the FileBlob->write operation and no error is thrown. + blob.write(outputDir_ + theNumberedOutputFileName); + // Careful, it seems that when writing to an impossible file, this is + // swallowed by the FileBlob.write operation and no error is thrown. + ++nBinaries_; } } else { edm::LogError("MillePedeFileActions") diff --git a/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileExtractor.h b/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileExtractor.h index 2105b3dd555d3..c978dd7c79fec 100644 --- a/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileExtractor.h +++ b/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileExtractor.h @@ -28,7 +28,8 @@ #include "FWCore/ParameterSet/interface/ParameterSet.h" #include "CondFormats/Common/interface/FileBlobCollection.h" -class MillePedeFileExtractor : public edm::one::EDAnalyzer { +class MillePedeFileExtractor : + public edm::one::EDAnalyzer { public: explicit MillePedeFileExtractor(const edm::ParameterSet&); ~MillePedeFileExtractor(); @@ -46,6 +47,7 @@ class MillePedeFileExtractor : public edm::one::EDAnalyzer fileBlobToken_; + int nBinaries_{0}; }; // define this as a plug-in From 5a84e1aa8855df5cf0fb4f21a073bb906e4855fd Mon Sep 17 00:00:00 2001 From: Gregor Mittag Date: Fri, 27 May 2016 10:52:49 +0200 Subject: [PATCH 6/8] Fix wrong warning for missing binary in PCL pede mode. --- .../src/MillePedeAlignmentAlgorithm.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Alignment/MillePedeAlignmentAlgorithm/src/MillePedeAlignmentAlgorithm.cc b/Alignment/MillePedeAlignmentAlgorithm/src/MillePedeAlignmentAlgorithm.cc index febf9d24ab2e5..9b5eb61f73445 100644 --- a/Alignment/MillePedeAlignmentAlgorithm/src/MillePedeAlignmentAlgorithm.cc +++ b/Alignment/MillePedeAlignmentAlgorithm/src/MillePedeAlignmentAlgorithm.cc @@ -351,11 +351,14 @@ std::vector MillePedeAlignmentAlgorithm::getExistingFormattedFiles( } } else { // The file doesn't exist, break out of the loop - edm::LogWarning("Alignment") - << "The input file '" << strippedInputFileName << "' does not exist."; break; } } + // warning if unformatted (-> theNumber stays at 0) does not exist + if (theNumber == 0 && (files.size() == 0 || files.back() != plainFile)) { + edm::LogWarning("Alignment") + << "The input file '" << plainFile << "' does not exist."; + } } return files; } From e17054938bea74c174248937acdea00d74ba3e64 Mon Sep 17 00:00:00 2001 From: Gregor Mittag Date: Fri, 27 May 2016 16:33:19 +0200 Subject: [PATCH 7/8] Put FileBlobCollection into Lumi even if it is empty. - fixes a bug in case of merging products from files which (partly) contain no collection --- .../plugins/MillePedeFileConverter.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileConverter.cc b/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileConverter.cc index 661bc414f559c..d57bfea781bad 100644 --- a/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileConverter.cc +++ b/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileConverter.cc @@ -43,8 +43,8 @@ void MillePedeFileConverter::endLuminosityBlockProduce(edm::LuminosityBlock& iLu if (fileBlob.size() > 0) { // Adding the FileBlob to the lumi: fileBlobCollection->addFileBlob(fileBlob); - iLumi.put(std::move(fileBlobCollection), fileBlobLabel_); } + iLumi.put(std::move(fileBlobCollection), fileBlobLabel_); } // Manage the parameters for the module: From f31bc4635e9185e0096653b3f20199e1f2b9489a Mon Sep 17 00:00:00 2001 From: Gregor Mittag Date: Tue, 31 May 2016 18:03:35 +0200 Subject: [PATCH 8/8] Introduce configurable limit on extracted number of binaries. --- .../plugins/MillePedeFileExtractor.cc | 18 +++++++++++++++--- .../plugins/MillePedeFileExtractor.h | 5 +++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileExtractor.cc b/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileExtractor.cc index 056e701391dba..f18b58df2220f 100644 --- a/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileExtractor.cc +++ b/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileExtractor.cc @@ -6,15 +6,20 @@ #include "FWCore/MessageLogger/interface/MessageLogger.h" #include "CondFormats/Common/interface/FileBlob.h" #include "FWCore/Utilities/interface/InputTag.h" -#include "FWCore/Utilities/interface/EDGetToken.h" +#include "FWCore/Utilities/interface/EDGetToken.h" MillePedeFileExtractor::MillePedeFileExtractor(const edm::ParameterSet& iConfig) : outputDir_(iConfig.getParameter("fileDir")), - outputFileName_(iConfig.getParameter("outputBinaryFile")) { + outputFileName_(iConfig.getParameter("outputBinaryFile")), + maxNumberOfBinaries_(iConfig.getParameter("maxNumberOfBinaries")) { auto fileBlobInputTag = iConfig.getParameter("fileBlobInputTag"); fileBlobToken_ = consumes(fileBlobInputTag); - // nothing else in the constructor + if (hasBinaryNumberLimit()) { + edm::LogInfo("MillePedeFileActions") + << "Limiting the number of extracted binary files to " + << maxNumberOfBinaries_; + } } MillePedeFileExtractor::~MillePedeFileExtractor() {} @@ -22,6 +27,8 @@ MillePedeFileExtractor::~MillePedeFileExtractor() {} void MillePedeFileExtractor::endLuminosityBlock(const edm::LuminosityBlock& iLumi, const edm::EventSetup&) { + if (enoughBinaries()) return; + // Getting our hands on the vector of FileBlobs edm::Handle fileBlobCollection; iLumi.getByToken(fileBlobToken_, fileBlobCollection); @@ -31,6 +38,7 @@ void MillePedeFileExtractor::endLuminosityBlock(const edm::LuminosityBlock& iLum << "Root file contains " << fileBlobCollection->size() << " FileBlob(s)."; // Loop over the FileBlobs in the vector, and write them to files: for (const auto& blob: *fileBlobCollection) { + if (enoughBinaries()) break; // We format the filename with a number, starting from 0 to the size of // our vector. // For this to work, the outputBinaryFile config parameter must contain a @@ -53,6 +61,7 @@ void MillePedeFileExtractor::endLuminosityBlock(const edm::LuminosityBlock& iLum } } + // Manage the parameters for the module: // (Note that this will autogenerate the _cfi.py file.) void MillePedeFileExtractor::fillDescriptions( @@ -74,6 +83,9 @@ void MillePedeFileExtractor::fillDescriptions( "root file. Make sure you overwrite this, if you have changed " "this is the configuration of the MillePedeFileConverter."); + desc.add("maxNumberOfBinaries", 1000)->setComment( + "Number of binaries to be extracted from the input files. " + "Use a negative value to apply no limit."); descriptions.add("millePedeFileExtractor", desc); descriptions.setComment( diff --git a/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileExtractor.h b/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileExtractor.h index c978dd7c79fec..cf52e32e14f6e 100644 --- a/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileExtractor.h +++ b/Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileExtractor.h @@ -42,11 +42,16 @@ class MillePedeFileExtractor : const edm::EventSetup&) override; virtual void analyze(const edm::Event&, const edm::EventSetup&) override {} + bool enoughBinaries() { + return (nBinaries_ >= maxNumberOfBinaries_) && hasBinaryNumberLimit(); } + bool hasBinaryNumberLimit() { return maxNumberOfBinaries_ > -1; } + const std::string outputDir_; const std::string outputFileName_; edm::EDGetTokenT fileBlobToken_; + const int maxNumberOfBinaries_; int nBinaries_{0}; };