Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

new ecal classes added for uploading with dropbox #7650

Merged
merged 1 commit into from Feb 23, 2015
Merged

new ecal classes added for uploading with dropbox #7650

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
247 changes: 100 additions & 147 deletions CondCore/Utilities/bin/conddb_validate.cpp
View file
Open in desktop
@@ -1,28 +1,23 @@
#include "CondCore/DBCommon/interface/DbSession.h"
#include "CondCore/DBCommon/interface/DbScopedTransaction.h"
#include "CondCore/DBCommon/interface/DbTransaction.h"
#include "CondCore/DBCommon/interface/Exception.h"
//#include "CondCore/DBCommon/interface/Exception.h"
#include "CondCore/DBCommon/interface/Auth.h"

#include "CondCore/CondDB/interface/ConnectionPool.h"
#include "CondCore/CondDB/interface/Utils.h"
#include "CondCore/CondDB/interface/IOVEditor.h"
#include "CondCore/CondDB/interface/IOVProxy.h"
#include "CondCore/CondDB/src/DbCore.h"

#include "CondCore/MetaDataService/interface/MetaData.h"

#include "CondCore/IOVService/interface/IOVProxy.h"

#include "CondCore/Utilities/interface/Utilities.h"
#include "CondCore/Utilities/interface/CondDBImport.h"
//#include "CondCore/Utilities/interface/CondDBImport.h"
#include "CondCore/Utilities/interface/CondDBTools.h"
#include <iostream>
#include <boost/filesystem.hpp>

// for the xml dump
#include "TFile.h"
#include "Cintex/Cintex.h"
#include <sstream>
#include <stdio.h>

namespace cond {

Expand All @@ -33,88 +28,13 @@ namespace cond {
int execute();
};

std::string writeTag( const std::string& tag, const std::string& label, persistency::Session& session, std::string tmpDir ){
session.transaction().start();
persistency::IOVProxy p = session.readIov( tag, true );
std::cout <<" "<<p.loadedSize()<<" iovs loaded."<<std::endl;
std::string destFile = label+".db";
boost::filesystem::path destFilePath( destFile );
if( !tmpDir.empty() ) {
destFilePath = boost::filesystem::path( tmpDir ) / destFilePath;
}
destFile = destFilePath.string();
persistency::ConnectionPool localPool;
persistency::Session writeSession = localPool.createSession( "sqlite_file:"+destFile, true );
writeSession.transaction().start( false );
persistency::IOVEditor writeEditor = writeSession.createIov( p.payloadObjectType(), tag, p.timeType(), p.synchronizationType() );
writeEditor.setValidationMode();
writeEditor.setDescription("Validation");
for( auto iov : p ){
std::pair<std::string,boost::shared_ptr<void> > readBackPayload = fetch( iov.payloadId, session );
cond::Hash ph = import( session, iov.payloadId, readBackPayload.first, readBackPayload.second.get(), writeSession );
writeEditor.insert( iov.since, ph );
}
writeEditor.flush();
writeSession.transaction().commit();
session.transaction().commit();
return destFile;
}

bool compareFiles( const std::string& refFileName, const std::string& candFileName ){
FILE* refFile = fopen( refFileName.c_str(), "r" );
if( refFile == NULL ){
throwException("Can't open file "+refFileName, "compareFiles" );
}
FILE* candFile = fopen( candFileName.c_str(), "r" );
if( candFile == NULL ){
throwException("Can't open file "+candFileName, "compareFiles" );
}
int N = 10000;
char buf1[N];
char buf2[N];

bool cmpOk = true;
do {
size_t r1 = fread( buf1, 1, N, refFile );
size_t r2 = fread( buf2, 1, N, candFile );

if( r1 != r2 || memcmp( buf1, buf2, r1)) {
cmpOk = false;
break;
}
} while(!feof(refFile) || !feof(candFile));

fclose(refFile);
fclose( candFile );

return cmpOk;
}

void flushFile( const std::string& account, const std::string& tag, const std::string& fileName ){
boost::filesystem::path accountDir( account );
if( !boost::filesystem::exists( accountDir ) ) boost::filesystem::create_directory( accountDir );
boost::filesystem::path tagDir( tag.c_str() );
tagDir = accountDir / tagDir;
if( !boost::filesystem::exists( tagDir ) ) boost::filesystem::create_directory( tagDir );
boost::filesystem::path sourceFilePath( fileName );
boost::filesystem::path destFilePath = tagDir / sourceFilePath.leaf();
if( boost::filesystem::exists(destFilePath) ) boost::filesystem::remove(destFilePath);
boost::filesystem::copy_file( sourceFilePath, destFilePath );
}

void cleanUp( const std::string& fileName ){
boost::filesystem::path fp( fileName );
if( boost::filesystem::exists(fp) ) boost::filesystem::remove( fp );
}

}

cond::ValidateUtilities::ValidateUtilities():Utilities("conddb_validate"){
addConnectOption("reference","r","reference database connection string (required)");
addConnectOption("candidate","c","candidate 1 connection string (required)");
cond::ValidateUtilities::ValidateUtilities():Utilities("conddb_validate2"){
addConnectOption("sourceConnect","s","source connection string(required)");
addConnectOption("destConnect","d","destionation connection string(required)");
addAuthenticationOptions();
addOption<std::string>("tag","t","migrate only the tag (optional)");
addOption<std::string>("dir","d","tmp folder to dump the temporary files for the comparison (optional)");
addOption<std::string>("tag","t","validate only the tag (optional)");
ROOT::Cintex::Cintex::Enable();
}

Expand All @@ -123,86 +43,119 @@ cond::ValidateUtilities::~ValidateUtilities(){

int cond::ValidateUtilities::execute(){

initializeForDbConnection();
bool debug = hasDebug();
int filterPosition = -1;
std::string tag("");
if( hasOptionValue("tag") ) tag = getOptionValue<std::string>("tag");
std::string dir("");
if( hasOptionValue("dir") ) dir = getOptionValue<std::string>("dir");
if( hasOptionValue("tag")) {
tag = getOptionValue<std::string>("tag");
if(debug){
std::cout << "tag " << tag << std::endl;
}
if( tag[0] == '*' ){
tag = tag.substr(1);
filterPosition = 0;
}
if( tag[tag.size()-1] == '*' ){
tag = tag.substr(0,tag.size()-1);
filterPosition = 1;
}
}
std::string destConnect = getOptionValue<std::string>("destConnect" );
std::string sourceConnect = getOptionValue<std::string>("sourceConnect");

bool debug = hasDebug();
std::tuple<std::string,std::string,std::string> connPars = persistency::parseConnectionString( sourceConnect );
if( std::get<0>( connPars ) == "frontier" ) throwException("Cannot validate data from FronTier cache.","MigrateUtilities::execute");

std::string refConnect = getOptionValue<std::string>("reference");
std::string candidate = getOptionValue<std::string>("candidate" );

std::tuple<std::string,std::string,std::string> connPars = persistency::parseConnectionString( refConnect );
if( std::get<0>( connPars ) == "frontier" ) throwException("Cannot validate data from FronTier cache.","ValidateUtilities::execute");

std::string refDbName = std::get<1>( connPars )+"_"+std::get<2>( connPars );

if(debug){
std::cout << "tag " << tag << std::endl;
}


std::cout <<"# Connecting to source database on "<<sourceConnect<<std::endl;
cond::DbSession sourcedb = openDbSession( "sourceConnect", cond::Auth::COND_READER_ROLE, true );
sourcedb.transaction().start( true );
cond::MetaData metadata(sourcedb);
std::vector<std::string> tagToProcess;
if( !tag.empty() ){
if( !tag.empty() && filterPosition == -1 ){
tagToProcess.push_back( tag );
} else {
cond::DbSession refdb = openDbSession( "reference", cond::Auth::COND_READER_ROLE, true );
refdb.transaction().start( true );
cond::MetaData metadata(refdb);
metadata.listAllTags( tagToProcess );
refdb.transaction().commit();
if( filterPosition != -1 ) {
std::vector<std::string> filteredList;
for( const auto& t: tagToProcess ) {
size_t ptr = t.find( tag );
if( ptr != std::string::npos && ptr < filterPosition ) filteredList.push_back( t );
}
tagToProcess = filteredList;
}
}

persistency::ConnectionPool connPool;
std::cout <<"# Opening session on reference database..."<<std::endl;
persistency::Session session0 = connPool.createSession( refConnect );
std::cout <<"# Opening session on candidates database..."<<std::endl;
persistency::Session session1 = connPool.createSession( candidate );
session1.transaction().start();
if( !session1.existsDatabase() ) throwException( "Candidate DB \""+candidate+" does not exist.",
"MigrateUtilities::execute" );
if( hasDebug() ) {
connPool.setMessageVerbosity( coral::Debug );
connPool.configure();
}
persistency::Session sourceSession = connPool.createSession( sourceConnect );

std::cout <<"# Opening session on destination database..."<<std::endl;
persistency::Session destSession = connPool.createSession( destConnect );

bool existsDestDb = false;
destSession.transaction().start( true );
existsDestDb = destSession.existsDatabase();
destSession.transaction().commit();
if( !existsDestDb ) {
std::cout <<"# ERROR: Database does not exist in the destination "<<destConnect<<std::endl;
return 1;
}

std::cout <<"# "<<tagToProcess.size()<<" tag(s) to process."<<std::endl;
std::cout <<std::endl;
size_t nt = 0;
size_t tid = 0;
size_t nt_validated = 0;
size_t nt_invalid = 0;
for( auto t : tagToProcess ){
tid++;
std::cout <<"--> Processing tag["<<tid<<"]: "<<t<<std::endl;
std::string refFileName("");
std::string candFileName("");
try{
std::cout <<" Writing reference db"<<std::endl;
refFileName = writeTag( t, "ref", session0, dir );
std::string destTag("");
cond::MigrationStatus status;
if( !session1.checkMigrationLog( refConnect, t, destTag, status ) ) {
std::cout << " ERROR: Tag "<< t <<" has not been migrated in database " << candidate <<std::endl;
boost::filesystem::remove( boost::filesystem::path(refFileName) );
nt++;
std::cout <<"--> Processing tag["<<nt<<"]: "<<t<<std::endl;

std::string destTag("");
cond::MigrationStatus status = ERROR;
destSession.transaction().start( true );
bool existsEntry = destSession.checkMigrationLog( sourceConnect, t, destTag, status );
if( existsEntry ){
std::cout <<" Tag found. Current status="<<validationStatusText[status]<<std::endl;
} else {
std::cout <<" Tag not migrated."<<std::endl;
continue;
}
destSession.transaction().commit();
if( existsEntry && status == ERROR ){
std::cout <<" Tag has been migrated with Errors."<<std::endl;
continue;
}
//if( status == MIGRATED ){
try{
if( validateTag( t, sourceSession, destTag, destSession ) ){
std::cout <<" Tag validated."<<std::endl;
nt_validated++;
} else {
std::cout <<" Writing candidate db"<<std::endl;
candFileName = writeTag( destTag, "cand", session1, dir );
bool cmp = compareFiles( refFileName, candFileName );
if(!cmp){
std::cout <<" ERROR: Comparison found differences."<<std::endl;
flushFile( refDbName, t, refFileName );
flushFile( refDbName, t, candFileName );
} else {
std::cout <<" Comparison OK."<<std::endl;
}
nt++;
std::cout <<" ERROR: Migrated tag different from reference."<<std::endl;
nt_invalid++;
}
} catch ( const std::exception& e ){
std::cout <<" ERROR:"<<e.what()<<std::endl;
std::cout <<" Tag "<<t<<" will be skipped."<<std::endl;
std::cout <<" ERROR in validation: "<<e.what()<<std::endl;
}
cleanUp( refFileName );
cleanUp( candFileName );
//}
//try{
// persistency::TransactionScope usc( destSession.transaction() );
// usc.start( false );
// if( existsEntry ) {
// destSession.updateMigrationLog( sourceConnect, t, status );
// } else {
// destSession.addToMigrationLog( sourceConnect, t, destTag, status );
// }
// usc.commit();
//} catch ( const std::exception& e ){
// std::cout <<" ERROR updating the status: "<<e.what()<<std::endl;
//}
}

std::cout <<std::endl<<"# "<<nt<<" tag(s) checked for validation."<<std::endl;
std::cout <<"# "<<nt<<" tag(s) processed. Validated: "<<nt_validated<<" Invalid:"<<nt_invalid<<std::endl;

return 0;
}
Expand Down
5 changes: 5 additions & 0 deletions CondCore/Utilities/src/CondDBFetch.cc
View file
Open in desktop
Expand Up @@ -126,6 +126,10 @@ namespace cond {
FETCH_PAYLOAD_CASE( EcalTimeOffsetConstant )
FETCH_PAYLOAD_CASE( EcalWeightXtalGroups )
FETCH_PAYLOAD_CASE( EcalSamplesCorrelation )
FETCH_PAYLOAD_CASE( EcalCondObjectContainer<EcalPulseShape> )
FETCH_PAYLOAD_CASE( EcalPulseShape )
FETCH_PAYLOAD_CASE( EcalCondObjectContainer<EcalPulseCovariance> )
FETCH_PAYLOAD_CASE( EcalPulseCovariance )
FETCH_PAYLOAD_CASE( FileBlob )
FETCH_PAYLOAD_CASE( GBRForest )
FETCH_PAYLOAD_CASE( GBRForestD )
Expand All @@ -139,6 +143,7 @@ namespace cond {
FETCH_PAYLOAD_CASE( HcalL1TriggerObjects )
FETCH_PAYLOAD_CASE( HcalLUTCorrs )
FETCH_PAYLOAD_CASE( HcalLongRecoParams )
FETCH_PAYLOAD_CASE( HcalZDCLowGainFractions )
FETCH_PAYLOAD_CASE( HcalLutMetadata )
FETCH_PAYLOAD_CASE( HcalMCParams )
FETCH_PAYLOAD_CASE( HcalPFCorrs )
Expand Down
4 changes: 4 additions & 0 deletions CondCore/Utilities/src/CondDBImport.cc
View file
Open in desktop
Expand Up @@ -141,6 +141,10 @@ namespace cond {
IMPORT_PAYLOAD_CASE( EcalTimeDependentCorrections )
IMPORT_PAYLOAD_CASE( EcalWeightXtalGroups )
IMPORT_PAYLOAD_CASE( EcalSamplesCorrelation )
IMPORT_PAYLOAD_CASE( EcalCondObjectContainer<EcalPulseShape> )
IMPORT_PAYLOAD_CASE( EcalPulseShape )
IMPORT_PAYLOAD_CASE( EcalCondObjectContainer<EcalPulseCovariance> )
IMPORT_PAYLOAD_CASE( EcalPulseCovariance )
IMPORT_PAYLOAD_CASE( FileBlob )
IMPORT_PAYLOAD_CASE( GBRForest )
IMPORT_PAYLOAD_CASE( GBRForestD )
Expand Down
2 changes: 2 additions & 0 deletions CondCore/Utilities/src/CondFormats.h
View file
Open in desktop
Expand Up @@ -56,6 +56,8 @@
#include "CondFormats/EcalObjects/interface/EcalTimeBiasCorrections.h"
#include "CondFormats/EcalObjects/interface/EcalTimeDependentCorrections.h"
#include "CondFormats/EcalObjects/interface/EcalSamplesCorrelation.h"
#include "CondFormats/EcalObjects/interface/EcalPulseShapes.h"
#include "CondFormats/EcalObjects/interface/EcalPulseCovariances.h"
#include "CondFormats/EgammaObjects/interface/GBRForest.h"
#include "CondFormats/EgammaObjects/interface/GBRForestD.h"
#include "CondFormats/HcalObjects/interface/AbsOOTPileupCorrection.h"
Expand Down