From 76ab8c21d46d8a7bb1ef743b24b644c9f8c1eb28 Mon Sep 17 00:00:00 2001 From: Giacomo Govi Date: Thu, 27 Mar 2014 17:37:10 +0100 Subject: [PATCH] Back-ported fixes and improvements --- CondCore/CondDB/interface/Binary.h | 18 ++-- CondCore/CondDB/interface/ConnectionPool.h | 12 ++- CondCore/CondDB/interface/GTProxy.h | 3 + CondCore/CondDB/interface/Serialization.h | 7 +- CondCore/CondDB/interface/Session.h | 6 +- CondCore/CondDB/interface/Types.h | 4 + CondCore/CondDB/interface/Utils.h | 46 ++++++---- CondCore/CondDB/python/CondDB_cfi.py | 4 +- CondCore/CondDB/src/Binary.cc | 13 ++- CondCore/CondDB/src/ConnectionPool.cc | 63 ++++++++------ CondCore/CondDB/src/DbConnectionString.cc | 84 +++++++++---------- CondCore/CondDB/src/DbConnectionString.h | 5 +- CondCore/CondDB/src/GTProxy.cc | 13 +++ CondCore/CondDB/src/OraDbSchema.cc | 5 +- CondCore/CondDB/src/Session.cc | 5 -- CondCore/CondDB/src/SessionImpl.cc | 59 +++++++++---- CondCore/CondDB/src/SessionImpl.h | 10 ++- .../RelationalAuthenticationService.cc | 17 ++-- CondCore/DBCommon/src/DbSession.cc | 6 +- CondCore/DBCommon/src/DecodingKey.cc | 2 +- CondCore/DBCommon/test/testDbSession.cc | 24 +----- .../src/PoolDBOutputService.cc | 6 +- ...stIOVPayloadAnalyzer_example_oracle_cfg.py | 2 +- .../DBOutputService/test/stubs/writeKeyed.cc | 9 +- CondCore/ESSources/plugins/CondDBESSource.cc | 18 +--- CondCore/ORA/interface/Container.h | 2 + CondCore/ORA/interface/Object.h | 17 ++++ CondCore/ORA/src/Container.cc | 7 ++ CondCore/ORA/src/Database.cc | 4 +- CondCore/ORA/src/Object.cc | 19 +++-- CondCore/Utilities/bin/conddb_migrate.cpp | 15 ++-- CondCore/Utilities/bin/conddb_migrate_gt.cpp | 2 +- .../Utilities/bin/conddb_test_gt_load.cpp | 81 ++++++++++-------- 33 files changed, 342 insertions(+), 246 deletions(-) diff --git a/CondCore/CondDB/interface/Binary.h b/CondCore/CondDB/interface/Binary.h index aa6cf7f87922a..4ac1605bde594 100644 --- a/CondCore/CondDB/interface/Binary.h +++ b/CondCore/CondDB/interface/Binary.h @@ -1,6 +1,8 @@ #ifndef CondCore_CondDB_Binary_h #define CondCore_CondDB_Binary_h +// for the old system, this will go away +#include "CondCore/ORA/interface/Object.h" #include #include // temporarely @@ -13,19 +15,12 @@ namespace coral { namespace cond { - struct Nodelete { - Nodelete(){} - void operator()( void* ptr ){} - }; - class Binary { public: Binary(); template explicit Binary( const T& object ); - explicit Binary( const boost::shared_ptr& objectPtr ); - Binary( const void* data, size_t size ); explicit Binary( const coral::Blob& data ); @@ -44,16 +39,19 @@ namespace cond { size_t size() const; - boost::shared_ptr share() const; + ora::Object oraObject() const; + + void fromOraObject( const ora::Object& object ); private: std::shared_ptr m_data; // - boost::shared_ptr m_object; + // workaround to support the non-streamed, packed objects ( the old system ) + ora::Object m_object; }; template Binary::Binary( const T& object ): - m_object( &const_cast(object), Nodelete() ){ + m_object( object ){ } } diff --git a/CondCore/CondDB/interface/ConnectionPool.h b/CondCore/CondDB/interface/ConnectionPool.h index 45b25271c651b..7a4eba9e81d9d 100644 --- a/CondCore/CondDB/interface/ConnectionPool.h +++ b/CondCore/CondDB/interface/ConnectionPool.h @@ -26,6 +26,7 @@ namespace cond { // enum DbAuthenticationSystem { UndefinedAuthentication=0,CondDbKey, CoralXMLFile }; + // a wrapper for the coral connection service. class ConnectionPool { public: ConnectionPool(); @@ -38,20 +39,23 @@ namespace cond { bool isLoggingEnabled() const; void setParameters( const edm::ParameterSet& connectionPset ); void configure(); - Session createSession( const std::string& connectionString, bool writeCapable=false ); + Session createSession( const std::string& connectionString, bool writeCapable=false, BackendType backType=DEFAULT_DB ); Session createReadOnlySession( const std::string& connectionString, const std::string& transactionId ); private: - Session createSession( const std::string& connectionString, const std::string& transactionId, bool writeCapable=false ); + Session createSession( const std::string& connectionString, + const std::string& transactionId, + bool writeCapable=false, + BackendType backType=DEFAULT_DB ); void configure( coral::IConnectionServiceConfiguration& coralConfig); private: std::string m_authPath; int m_authSys = 0; - coral::MsgLevel m_messageLevel = coral::Info; + coral::MsgLevel m_messageLevel = coral::Error; bool m_loggingEnabled = false; // this one has to be moved! cond::CoralServiceManager* m_pluginManager = 0; - std::vector m_refreshtablelist; + std::map m_dbTypes; }; } } diff --git a/CondCore/CondDB/interface/GTProxy.h b/CondCore/CondDB/interface/GTProxy.h index 807f6aeaf941f..2d10e50eee1f8 100644 --- a/CondCore/CondDB/interface/GTProxy.h +++ b/CondCore/CondDB/interface/GTProxy.h @@ -21,6 +21,9 @@ namespace cond { namespace persistency { + // required in the "bridge" mode, to be removed with the ORA based code after the transition + std::pair parseTag( const std::string& tag ); + class SessionImpl; class GTProxyData; diff --git a/CondCore/CondDB/interface/Serialization.h b/CondCore/CondDB/interface/Serialization.h index 6642034644783..7fc341d239d38 100644 --- a/CondCore/CondDB/interface/Serialization.h +++ b/CondCore/CondDB/interface/Serialization.h @@ -28,8 +28,9 @@ namespace cond { // default payload factory template T* createPayload( const std::string& payloadTypeName ){ - if( demangledName( typeid(T) )!= payloadTypeName ) - throwException(std::string("Type mismatch, target object is type \"")+payloadTypeName+"\"", + std::string userTypeName = demangledName( typeid(T) ); + if( userTypeName != payloadTypeName ) + throwException(std::string("Type mismatch, user type: \""+userTypeName+"\", target type: \"")+payloadTypeName+"\"", "createPayload" ); return new T; } @@ -113,7 +114,7 @@ namespace cond { payload.reset( createPayload(payloadType) ); ia >> (*payload); } else { - payload = boost::static_pointer_cast(payloadData.share()); + payload = boost::static_pointer_cast(payloadData.oraObject().makeShared()); } return payload; } diff --git a/CondCore/CondDB/interface/Session.h b/CondCore/CondDB/interface/Session.h index 4da43ef916083..b8b2ed15848a3 100644 --- a/CondCore/CondDB/interface/Session.h +++ b/CondCore/CondDB/interface/Session.h @@ -64,9 +64,6 @@ namespace cond { // constructor explicit Session( const std::shared_ptr& sessionImpl ); - // constructor - Session( boost::shared_ptr& session, const std::string& connectionString ); - // Session( const Session& rhs ); @@ -141,9 +138,10 @@ namespace cond { coral::ISchema& nominalSchema(); bool isOraSession(); - + private: cond::Hash storePayloadData( const std::string& payloadObjectType, const cond::Binary& payloadData, const boost::posix_time::ptime& creationTime ); + private: std::shared_ptr m_session; diff --git a/CondCore/CondDB/interface/Types.h b/CondCore/CondDB/interface/Types.h index bf5cec39ff6b5..f1c137c75838a 100644 --- a/CondCore/CondDB/interface/Types.h +++ b/CondCore/CondDB/interface/Types.h @@ -19,6 +19,10 @@ namespace cond { + // to be removed after the transition to new DB + typedef enum { UNKNOWN_DB=0, COND_DB, ORA_DB } BackendType; + static constexpr BackendType DEFAULT_DB = ORA_DB; + typedef enum { SYNCHRONIZATION_UNKNOWN = -1, OFFLINE=0, diff --git a/CondCore/CondDB/interface/Utils.h b/CondCore/CondDB/interface/Utils.h index b8bd83f1537f4..3b3ec28661f97 100644 --- a/CondCore/CondDB/interface/Utils.h +++ b/CondCore/CondDB/interface/Utils.h @@ -7,12 +7,13 @@ #include #include #include +#include namespace cond { namespace { - std::string demangledName( const std::type_info& typeInfo ){ + inline std::string demangledName( const std::type_info& typeInfo ){ int status = 0; std::string ret(""); char* realname = abi::__cxa_demangle( typeInfo.name(), 0, 0, &status); @@ -25,24 +26,41 @@ namespace cond { return ret; } - std::tuple parseConnectionString( const std::string& connectionString ){ - size_t ptr = 0; + } + + namespace persistency { + + inline std::string getConnectionProtocol( const std::string& connectionString ){ size_t techEnd = connectionString.find( ':' ); - if( techEnd == std::string::npos ) throwException( "Connection string is invalid (0)","parseConnectionString" ); - std::string technology = connectionString.substr(ptr,techEnd); - std::string service(""); - ptr = techEnd+1; - if( technology != "sqlite_file" ){ - if( connectionString.substr( ptr,2 )!="//" ) throwException( "Connection string is invalid (1)","parseConnectionString" ); + if( techEnd == std::string::npos ) throwException( "Could not resolve the connection protocol on "+connectionString+".", + "getConnectionProtocol" ); + std::string technology = connectionString.substr(0,techEnd); + return technology; + } + + inline std::tuple parseConnectionString( const std::string& connectionString ){ + std::string protocol = getConnectionProtocol( connectionString ); + std::string serviceName(""); + std::string databaseName(""); + if( protocol == "sqlite" || protocol == "sqlite_file" || protocol == "sqlite_fip" ){ + databaseName = connectionString.substr( protocol.size()+1 ); + } else if ( protocol == "oracle" || protocol == "frontier" ){ + size_t ptr = protocol.size()+1; + if( connectionString.substr( ptr,2 )!="//" ) throwException( "Connection string "+connectionString+ + " is invalid format for technology \""+ + protocol+"\".","parseConnectionString" ); ptr += 2; size_t serviceEnd = connectionString.find( '/', ptr ); - if( serviceEnd == std::string::npos ) throwException( "Connection string is invalid (2)","parseConnectionString" ); - service = connectionString.substr( ptr, serviceEnd-ptr ); + if( serviceEnd == std::string::npos ) throwException( "Connection string "+connectionString+" is invalid.", + "parseConnectionString" ); + serviceName = connectionString.substr( ptr, serviceEnd-ptr ); ptr = serviceEnd+1; - } - std::string schema = connectionString.substr( ptr ); - return std::make_tuple( technology, service, schema ); + databaseName = connectionString.substr( ptr ); + } else throwException( "Technology "+protocol+" is not known.","parseConnectionString" ); + + return std::make_tuple( protocol, serviceName, databaseName ); } + } } diff --git a/CondCore/CondDB/python/CondDB_cfi.py b/CondCore/CondDB/python/CondDB_cfi.py index cf2ba08a09eef..e933491abcc36 100644 --- a/CondCore/CondDB/python/CondDB_cfi.py +++ b/CondCore/CondDB/python/CondDB_cfi.py @@ -3,8 +3,10 @@ CondDB = cms.PSet( DBParameters = cms.PSet( authenticationPath = cms.untracked.string(''), + authenticationSystem = cms.untracked.int32(0), messageLevel = cms.untracked.int32(0), ), - connect = cms.string('protocol://db/schema') ##db/schema" + connect = cms.string('protocol://db/schema'), ##db/schema" + dbFormat = cms.untracked.int32(0) ) diff --git a/CondCore/CondDB/src/Binary.cc b/CondCore/CondDB/src/Binary.cc index cb81f0d4684fd..c650bc38d9d57 100644 --- a/CondCore/CondDB/src/Binary.cc +++ b/CondCore/CondDB/src/Binary.cc @@ -12,10 +12,6 @@ cond::Binary::Binary(): m_data( new coral::Blob(0) ){ } -cond::Binary::Binary( const boost::shared_ptr& objectPtr ): - m_object( objectPtr ){ -} - cond::Binary::Binary( const void* data, size_t size ): m_data( new coral::Blob( size ) ){ ::memcpy( m_data->startingAddress(), data, size ); @@ -46,7 +42,7 @@ const coral::Blob& cond::Binary::get() const { void cond::Binary::copy( const std::string& source ){ m_data.reset( new coral::Blob( source.size() ) ); ::memcpy( m_data->startingAddress(), source.c_str(), source.size() ); - m_object.reset(); + m_object = ora::Object(); } const void* cond::Binary::data() const { @@ -63,10 +59,13 @@ size_t cond::Binary::size() const { return m_data->size(); } -boost::shared_ptr cond::Binary::share() const { +ora::Object cond::Binary::oraObject() const { return m_object; } - +void cond::Binary::fromOraObject( const ora::Object& object ){ + m_object = object; + m_data.reset(); +} diff --git a/CondCore/CondDB/src/ConnectionPool.cc b/CondCore/CondDB/src/ConnectionPool.cc index 4f69bc086482b..0eea89e0160f8 100644 --- a/CondCore/CondDB/src/ConnectionPool.cc +++ b/CondCore/CondDB/src/ConnectionPool.cc @@ -1,5 +1,7 @@ #include "CondCore/CondDB/interface/ConnectionPool.h" #include "DbConnectionString.h" +#include "SessionImpl.h" +#include "IOVSchema.h" // #include "CondCore/DBCommon/interface/CoralServiceManager.h" #include "CondCore/DBCommon/interface/Auth.h" @@ -19,24 +21,14 @@ namespace cond { namespace persistency { + + static const std::string POOL_IOV_TABLE_DATA("IOV_DATA"); + static const std::string ORA_IOV_TABLE_1("ORA_C_COND_IOVSEQUENCE"); + static const std::string ORA_IOV_TABLE_2("ORA_C_COND_IOVSEQU_A0"); + static const std::string ORA_IOV_TABLE_3("ORA_C_COND_IOVSEQU_A1"); - ConnectionPool::ConnectionPool(): - m_authPath(), - m_authSys(0), - m_messageLevel( coral::Error ), - m_loggingEnabled( false ), - m_pluginManager( new cond::CoralServiceManager ), - m_refreshtablelist(){ - m_refreshtablelist.reserve(6); - //table names for IOVSequence in the old POOL mapping - m_refreshtablelist.push_back("IOV"); - m_refreshtablelist.push_back("IOV_DATA"); - //table names for IOVSequence in ORA - m_refreshtablelist.push_back("ORA_C_COND_IOVSEQUENCE"); - m_refreshtablelist.push_back("ORA_C_COND_IOVSEQU_A0"); - m_refreshtablelist.push_back("ORA_C_COND_IOVSEQU_A1"); - //table names for IOVSequence in CONDDB - m_refreshtablelist.push_back("TAG"); + ConnectionPool::ConnectionPool(){ + m_pluginManager = new cond::CoralServiceManager; configure(); } @@ -139,19 +131,42 @@ namespace cond { configure( connServ.configuration() ); } - Session ConnectionPool::createSession( const std::string& connectionString, const std::string& transactionId, bool writeCapable ){ + Session ConnectionPool::createSession( const std::string& connectionString, + const std::string& transactionId, + bool writeCapable, + BackendType backType){ coral::ConnectionService connServ; - std::pair fullConnectionPars = getRealConnectionString( connectionString, transactionId ); - if( !fullConnectionPars.second.empty() ) - for( auto tableName : m_refreshtablelist ) connServ.webCacheControl().refreshTable( fullConnectionPars.second, tableName ); + std::pair fullConnectionPars = getConnectionParams( connectionString, transactionId ); + if( !fullConnectionPars.second.empty() ) { + // the olds formats + connServ.webCacheControl().refreshTable( fullConnectionPars.second, POOL_IOV_TABLE_DATA ); + connServ.webCacheControl().refreshTable( fullConnectionPars.second, ORA_IOV_TABLE_1 ); + connServ.webCacheControl().refreshTable( fullConnectionPars.second, ORA_IOV_TABLE_2 ); + connServ.webCacheControl().refreshTable( fullConnectionPars.second, ORA_IOV_TABLE_3 ); + // the new schema... + connServ.webCacheControl().refreshTable( fullConnectionPars.second, TAG::tname ); + connServ.webCacheControl().refreshTable( fullConnectionPars.second, IOV::tname ); + } boost::shared_ptr coralSession( connServ.connect( fullConnectionPars.first, + writeCapable?Auth::COND_WRITER_ROLE:Auth::COND_READER_ROLE, writeCapable?coral::Update:coral::ReadOnly ) ); - return Session( coralSession, connectionString ); + BackendType bt; + auto it = m_dbTypes.find( connectionString); + if( it == m_dbTypes.end() ){ + bt = checkBackendType( coralSession, connectionString ); + if( bt == UNKNOWN_DB && writeCapable) bt = backType; + m_dbTypes.insert( std::make_pair( connectionString, bt ) ).first; + } else { + bt = (BackendType) it->second; + } + + std::shared_ptr impl( new SessionImpl( coralSession, connectionString, bt ) ); + return Session( impl ); } - Session ConnectionPool::createSession( const std::string& connectionString, bool writeCapable ){ - return createSession( connectionString, "", writeCapable ); + Session ConnectionPool::createSession( const std::string& connectionString, bool writeCapable, BackendType backType ){ + return createSession( connectionString, "", writeCapable, backType ); } Session ConnectionPool::createReadOnlySession( const std::string& connectionString, const std::string& transactionId ){ diff --git a/CondCore/CondDB/src/DbConnectionString.cc b/CondCore/CondDB/src/DbConnectionString.cc index 640e5e3bedf76..b99c8608b2a2a 100644 --- a/CondCore/CondDB/src/DbConnectionString.cc +++ b/CondCore/CondDB/src/DbConnectionString.cc @@ -2,6 +2,7 @@ #include "CondCore/CondDB/interface/Utils.h" #include "DbConnectionString.h" // +#include "CondCore/DBCommon/interface/FipProtocolParser.h" #include "FWCore/Catalog/interface/SiteLocalConfig.h" #include "FWCore/ServiceRegistry/interface/Service.h" @@ -9,6 +10,7 @@ namespace cond { namespace persistency { + unsigned int countslash(const std::string& input) { unsigned int count=0; @@ -24,52 +26,48 @@ namespace cond { return count; } - std::pair makeFrontierConnectionString( const std::string& initialConnection, - const std::string& transactionId ){ - std::string realConn = initialConnection; - std::string proto("frontier://"); - std::string::size_type fpos=initialConnection.find(proto); - unsigned int nslash=countslash(initialConnection.substr(proto.size(),initialConnection.size()-fpos)); - if(nslash==1){ - edm::Service localconfservice; - if( !localconfservice.isAvailable() ){ - throwException("SiteLocalConfigService is not available","cond::makeRealConnectString"); + std::pair getConnectionParams( const std::string& connectionString, + const std::string& transactionId ){ + if( connectionString.empty() ) throwException( "The connection string is empty.","getConnectionParams"); + std::string protocol = getConnectionProtocol( connectionString ); + std::string finalConn = connectionString; + std::string refreshConn(""); + if( protocol == "frontier" ){ + std::string protocol("frontier://"); + std::string::size_type fpos=connectionString.find(protocol); + unsigned int nslash=countslash(connectionString.substr(protocol.size(),connectionString.size()-fpos)); + if(nslash==1){ + edm::Service localconfservice; + if( !localconfservice.isAvailable() ){ + throwException("edm::SiteLocalConfigService is not available","getConnectionParams"); + } + finalConn=localconfservice->lookupCalibConnect(connectionString); } - realConn=localconfservice->lookupCalibConnect(initialConnection); - } - if (!transactionId.empty()) { - size_t l = realConn.rfind('/'); - realConn.insert(l,"(freshkey="+transactionId+')'); - } - - std::string refreshConnect; - std::string::size_type startRefresh = realConn.find("://"); - if (startRefresh != std::string::npos){ - startRefresh += 3; - } - std::string::size_type endRefresh=realConn.rfind("/", std::string::npos); - if (endRefresh == std::string::npos){ - refreshConnect = realConn; - }else{ - refreshConnect = realConn.substr(startRefresh, endRefresh-startRefresh); - if(refreshConnect.substr(0,1) != "("){ - //if the connect string is not a complicated parenthesized string, - // an http:// needs to be at the beginning of it - refreshConnect.insert(0, "http://"); + if (!transactionId.empty()) { + size_t l = finalConn.rfind('/'); + finalConn.insert(l,"(freshkey="+transactionId+')'); } + + std::string::size_type startRefresh = finalConn.find("://"); + if (startRefresh != std::string::npos){ + startRefresh += 3; + } + std::string::size_type endRefresh=finalConn.rfind("/", std::string::npos); + if (endRefresh == std::string::npos){ + refreshConn = finalConn; + } else{ + refreshConn = finalConn.substr(startRefresh, endRefresh-startRefresh); + if(refreshConn.substr(0,1) != "("){ + //if the connect string is not a complicated parenthesized string, + // an http:// needs to be at the beginning of it + refreshConn.insert(0, "http://"); + } + } + } else if ( protocol == "sqlite_fip" ){ + cond::FipProtocolParser parser; + finalConn = parser.getRealConnect( connectionString ); } - return std::make_pair(realConn,refreshConnect); - } - - std::pair getRealConnectionString( const std::string& initialConnection ){ - return getRealConnectionString( initialConnection, "" ); - } - - std::pair getRealConnectionString( const std::string& initialConnection, - const std::string& transId ){ - auto connData = parseConnectionString( initialConnection ); - if( std::get<0>(connData) == "frontier" ) return makeFrontierConnectionString( initialConnection, transId ); - return std::make_pair(initialConnection,""); + return std::make_pair( finalConn, refreshConn ); } } diff --git a/CondCore/CondDB/src/DbConnectionString.h b/CondCore/CondDB/src/DbConnectionString.h index 2c92e43685774..6c28219286c75 100644 --- a/CondCore/CondDB/src/DbConnectionString.h +++ b/CondCore/CondDB/src/DbConnectionString.h @@ -1,6 +1,7 @@ #ifndef CondCore_CondDB_DbConnectionString_h #define CondCore_CondDB_DbConnectionString_h +#include "CondCore/CondDB/interface/Utils.h" // #include @@ -8,9 +9,7 @@ namespace cond { namespace persistency { - std::pair getRealConnectionString( const std::string& initialConnection ); - - std::pair getRealConnectionString( const std::string& initialConnection, const std::string& transId ); + std::pair getConnectionParams( const std::string& initialConnection, const std::string& transId ); } diff --git a/CondCore/CondDB/src/GTProxy.cc b/CondCore/CondDB/src/GTProxy.cc index 3d34c3012bb3e..a3212e066dc6b 100644 --- a/CondCore/CondDB/src/GTProxy.cc +++ b/CondCore/CondDB/src/GTProxy.cc @@ -5,6 +5,19 @@ namespace cond { namespace persistency { + std::pair parseTag( const std::string& tag ){ + std::string pfn(""); + std::string t(tag); + size_t pos = tag.rfind('@'); + if( pos != std::string::npos && tag.size() >= pos+3 ){ + if( tag[pos+1]=='[' && tag[tag.size()-1]==']' ) { + pfn = tag.substr( pos+2,tag.size()-pos-3 ); + t = tag.substr( 0, pos ); + } + } + return std::make_pair( t, pfn ); + } + // implementation details... // only hosting data in this case class GTProxyData { diff --git a/CondCore/CondDB/src/OraDbSchema.cc b/CondCore/CondDB/src/OraDbSchema.cc index ffe796e29dae8..3f0cc084b8b84 100644 --- a/CondCore/CondDB/src/OraDbSchema.cc +++ b/CondCore/CondDB/src/OraDbSchema.cc @@ -118,7 +118,7 @@ namespace cond { bool OraPayloadTable::select( const cond::Hash& payloadHash, std::string& objectType, cond::Binary& payloadData ){ ora::Object obj = m_session.getObject( payloadHash ); objectType = obj.typeName(); - payloadData = cond::Binary( obj.makeShared() ); + payloadData.fromOraObject(obj ); return true; } @@ -129,8 +129,7 @@ namespace cond { cond::Hash OraPayloadTable::insertIfNew( const std::string& objectType, const cond::Binary& payloadData, const boost::posix_time::ptime& ){ - void* ptr = payloadData.share().get(); - ora::Object obj( ptr, objectType ); + ora::Object obj = payloadData.oraObject(); std::string tok = m_session.storeObject( obj, objectType ); m_session.flush(); return tok; diff --git a/CondCore/CondDB/src/Session.cc b/CondCore/CondDB/src/Session.cc index 0dbfc54ea144e..109bf0ed850bf 100644 --- a/CondCore/CondDB/src/Session.cc +++ b/CondCore/CondDB/src/Session.cc @@ -45,11 +45,6 @@ namespace cond { m_transaction( *m_session ){ } - Session::Session( boost::shared_ptr& session, const std::string& connectionString ): - m_session( new SessionImpl( session, connectionString ) ), - m_transaction( *m_session ){ - } - Session::Session( const Session& rhs ): m_session( rhs.m_session ), m_transaction( rhs.m_transaction ){ diff --git a/CondCore/CondDB/src/SessionImpl.cc b/CondCore/CondDB/src/SessionImpl.cc index 78a9dfa804381..4cbf1437174e0 100644 --- a/CondCore/CondDB/src/SessionImpl.cc +++ b/CondCore/CondDB/src/SessionImpl.cc @@ -57,13 +57,37 @@ namespace cond { cond::DbSession m_session; }; + BackendType checkBackendType( boost::shared_ptr& coralSession, + const std::string& connectionString ){ + BackendType ret = UNKNOWN_DB; + cond::DbSession oraSession; + oraSession.open( coralSession, connectionString ); + oraSession.transaction().start( true ); + std::unique_ptr iovSchemaHandle( new OraIOVSchema( oraSession ) ); + std::unique_ptr gtSchemaHandle( new OraGTSchema( oraSession ) ); + if( !iovSchemaHandle->exists() && !gtSchemaHandle->exists() ){ + iovSchemaHandle.reset( new IOVSchema( coralSession->nominalSchema() ) ); + if( iovSchemaHandle->exists() ){ + ret = COND_DB; + } + } else { + ret = ORA_DB; + } + oraSession.transaction().commit(); + return ret; + } + SessionImpl::SessionImpl(): - coralSession(){ + coralSession(), + theBackendType( UNKNOWN_DB ){ } - SessionImpl::SessionImpl( boost::shared_ptr& session, const std::string& connectionStr ): + SessionImpl::SessionImpl( boost::shared_ptr& session, + const std::string& connectionStr, + BackendType backType ): coralSession( session ), - connectionString( connectionStr ){ + connectionString( connectionStr ), + theBackendType( backType ){ } SessionImpl::~SessionImpl(){ @@ -86,21 +110,20 @@ namespace cond { void SessionImpl::startTransaction( bool readOnly ){ if( !transaction.get() ){ - cond::DbConnection oraConnection; - cond::DbSession oraSession = oraConnection.createSession(); - oraSession.open( coralSession, connectionString ); - transaction.reset( new OraTransaction( oraSession ) ); - oraSession.transaction().start( readOnly ); - iovSchemaHandle.reset( new OraIOVSchema( oraSession ) ); - gtSchemaHandle.reset( new OraGTSchema( oraSession ) ); - if( !iovSchemaHandle->exists() && !gtSchemaHandle->exists() ){ - std::unique_ptr iovSchema( new IOVSchema( coralSession->nominalSchema() ) ); - std::unique_ptr gtSchema( new GTSchema( coralSession->nominalSchema() ) ); - if( iovSchema->exists() ){ - iovSchemaHandle = std::move(iovSchema); - gtSchemaHandle = std::move(gtSchema); - transaction.reset( new CondDBTransaction( coralSession ) ); - } + if ( theBackendType == ORA_DB ) { + cond::DbSession oraSession; + oraSession.open( coralSession, connectionString ); + oraSession.transaction().start( readOnly ); + iovSchemaHandle.reset( new OraIOVSchema( oraSession ) ); + gtSchemaHandle.reset( new OraGTSchema( oraSession ) ); + transaction.reset( new OraTransaction( oraSession ) ); + } else if ( theBackendType == COND_DB ){ + coralSession->transaction().start( readOnly ); + iovSchemaHandle.reset( new IOVSchema( coralSession->nominalSchema() ) ); + gtSchemaHandle.reset( new GTSchema( coralSession->nominalSchema() ) ); + transaction.reset( new CondDBTransaction( coralSession ) ); + } else { + throwException( "No valid database found.", "SessionImpl::startTransaction" ); } } else { if(!readOnly ) throwException( "An update transaction is already active.", diff --git a/CondCore/CondDB/src/SessionImpl.h b/CondCore/CondDB/src/SessionImpl.h index 782942bdc7bf6..2f23248867404 100644 --- a/CondCore/CondDB/src/SessionImpl.h +++ b/CondCore/CondDB/src/SessionImpl.h @@ -1,6 +1,7 @@ #ifndef CondCore_CondDB_SessionImpl_h #define CondCore_CondDB_SessionImpl_h +#include "CondCore/CondDB/interface/Types.h" #include "IOVSchema.h" #include "GTSchema.h" // @@ -35,13 +36,19 @@ namespace cond { bool isOra = false; size_t clients = 0; }; + + BackendType checkBackendType( boost::shared_ptr& session, + const std::string& connectionString ); class SessionImpl { public: typedef enum { THROW, DO_NOT_THROW, CREATE } FailureOnOpeningPolicy; public: SessionImpl(); - SessionImpl( boost::shared_ptr& session, const std::string& connectionString ); + SessionImpl( boost::shared_ptr& session, + const std::string& connectionString, + BackendType backType ); + ~SessionImpl(); void close(); @@ -63,6 +70,7 @@ namespace cond { boost::shared_ptr coralSession; // not really useful outside the ORA bridging... std::string connectionString; + BackendType theBackendType; std::unique_ptr transaction; std::unique_ptr iovSchemaHandle; std::unique_ptr gtSchemaHandle; diff --git a/CondCore/DBCommon/plugins/RelationalAuthenticationService.cc b/CondCore/DBCommon/plugins/RelationalAuthenticationService.cc index e1dc3613868fe..81bdaff3bad92 100644 --- a/CondCore/DBCommon/plugins/RelationalAuthenticationService.cc +++ b/CondCore/DBCommon/plugins/RelationalAuthenticationService.cc @@ -60,9 +60,11 @@ cond::RelationalAuthenticationService::RelationalAuthenticationService::credenti } creds = m_cache.get( connectionString ); if( ! creds ){ - std::string msg("No Authentication available for connection=\""); - msg += connectionString + "\"."; - throw coral::AuthenticationServiceException( msg, "cond::RelationalAuthenticationService::RelationalAuthenticationService::credentials", ""); + std::string msg("Connection to \""); + msg += connectionString + "\""; + msg += " with role \"COND_DEFAULT_ROLE\" is not available for "; + msg +=m_db.keyPrincipalName(); + cond::throwException( msg, "cond::RelationalAuthenticationService::RelationalAuthenticationService::credentials" ); } return *creds; } @@ -80,10 +82,11 @@ cond::RelationalAuthenticationService::RelationalAuthenticationService::credenti } creds = m_cache.get( connectionString, role ); if( ! creds ){ - std::string msg("No Authentication available for connection=\""); - msg += connectionString + "\"."; - msg += " and role=\"" + role + "\"."; - throw coral::AuthenticationServiceException( msg, "cond::RelationalAuthenticationService::RelationalAuthenticationService::credentials",""); + std::string msg("Connection to \""); + msg += connectionString + "\""; + msg += " with role \"" + role + "\" is not available for "; + msg +=m_db.keyPrincipalName(); + cond::throwException( msg, "cond::RelationalAuthenticationService::RelationalAuthenticationService::credentials" ); } return *creds; } diff --git a/CondCore/DBCommon/src/DbSession.cc b/CondCore/DBCommon/src/DbSession.cc index 1ad0ad2c23f62..3e57b5a63e4a1 100644 --- a/CondCore/DBCommon/src/DbSession.cc +++ b/CondCore/DBCommon/src/DbSession.cc @@ -207,16 +207,12 @@ const std::string& cond::DbSession::blobStreamingService() const cond::DbTransaction& cond::DbSession::transaction() { - if(!m_implementation->connection.get() || !m_implementation->connection->isOpen()) - throw cond::Exception("DbSession::transaction: cannot open transaction. Underlying connection is closed."); if(!m_implementation->transaction.get()) throw cond::Exception("DbSession::transaction: cannot get transaction. Session has not been open."); return *m_implementation->transaction; } ora::Database& cond::DbSession::storage(){ - if(!m_implementation->connection.get() || !m_implementation->connection->isOpen()) - throw cond::Exception("DbSession::storage: cannot access the storage. Underlying connection is closed."); if(!m_implementation->database.get()) throw cond::Exception("DbSession::storage: cannot access the database. Session has not been open."); return *m_implementation->database; @@ -299,7 +295,7 @@ std::string cond::DbSession::classNameForItem( const std::string& objectId ){ std::string ret(""); if( !oid.isInvalid() ){ ora::Container cont = storage().containerHandle( oid.containerId() ); - ret = cont.className(); + ret = cont.realClassName(); } return ret; } diff --git a/CondCore/DBCommon/src/DecodingKey.cc b/CondCore/DBCommon/src/DecodingKey.cc index d8e48d812fcd7..e5a0079f0c628 100644 --- a/CondCore/DBCommon/src/DecodingKey.cc +++ b/CondCore/DBCommon/src/DecodingKey.cc @@ -179,7 +179,7 @@ size_t cond::DecodingKey::init( const std::string& keyFileName, const std::strin } } } else { - std::string msg = "Provided Key File \""+m_fileName+"\" is invalid."; + std::string msg = "Required Key File \""+m_fileName+"\" is missing or unreadable."; throwException(msg,"DecodingKey::init"); } } diff --git a/CondCore/DBCommon/test/testDbSession.cc b/CondCore/DBCommon/test/testDbSession.cc index b6472a99b3a58..d507ee2815371 100644 --- a/CondCore/DBCommon/test/testDbSession.cc +++ b/CondCore/DBCommon/test/testDbSession.cc @@ -82,30 +82,12 @@ int main(){ try { std::cout << "######### test 9"<("connect"), true ); + std::string connectionString = iConfig.getParameter("connect"); + BackendType backType = (BackendType) iConfig.getUntrackedParameter("dbFormat", DEFAULT_DB ); + if( backType == UNKNOWN_DB ) backType = DEFAULT_DB; + m_session = connection.createSession( connectionString, true, backType ); //if( iConfig.exists("logconnect") ){ // m_logConnectionString = iConfig.getUntrackedParameter("logconnect"); diff --git a/CondCore/DBOutputService/test/python/testIOVPayloadAnalyzer_example_oracle_cfg.py b/CondCore/DBOutputService/test/python/testIOVPayloadAnalyzer_example_oracle_cfg.py index 193e0ac516120..8c58376d64557 100644 --- a/CondCore/DBOutputService/test/python/testIOVPayloadAnalyzer_example_oracle_cfg.py +++ b/CondCore/DBOutputService/test/python/testIOVPayloadAnalyzer_example_oracle_cfg.py @@ -3,7 +3,7 @@ process = cms.Process("TEST") process.load("CondCore.CondDB.CondDB_cfi") process.CondDB.connect = 'oracle://cms_orcoff_prep/CMS_COND_WEB' -process.CondDB.DBParameters.authenticationPath = '/afs/cern.ch/cms/DB/conddb/test' +process.CondDB.DBParameters.authenticationPath = '/afs/cern.ch/cms/DB/conddb' #process.CondDB.DBParameters.messageLevel = cms.untracked.int32(3) process.source = cms.Source("EmptyIOVSource", diff --git a/CondCore/DBOutputService/test/stubs/writeKeyed.cc b/CondCore/DBOutputService/test/stubs/writeKeyed.cc index 16deb6ddde8d6..2950d1dcbf834 100644 --- a/CondCore/DBOutputService/test/stubs/writeKeyed.cc +++ b/CondCore/DBOutputService/test/stubs/writeKeyed.cc @@ -40,6 +40,7 @@ writeKeyed::endJob() { edm::Service outdb; + std::map keys; // populated with the keyed payloads (configurations) for ( size_t i=0; iwriteOne(k.m_obj,k.m_key,confcont); + + keys.insert( std::make_pair( k.m_key, k.m_obj ) ); + //outdb->writeOne(k.m_obj,k.m_key,confcont); } + std::cout <<"# uploading keys..."<writeOne( k.second, k.first,confcont ); + + std::cout <<"# uploading master payloads..."< parseTag( const std::string& tag ){ - std::string pfn(""); - std::string t(tag); - size_t pos = tag.rfind('@'); - if( pos != std::string::npos && tag.size() >= pos+3 ){ - if( tag[pos+1]=='[' && tag[tag.size()-1]==']' ) { - pfn = tag.substr( pos+2,tag.size()-pos-3 ); - t = tag.substr( 0, pos ); - } - } - return std::make_pair( t, pfn ); - } - /* utility class to return a IOVs associated to a given "name" This implementation return the IOV associated to a record... It is essentialy a workaround to get the full IOV out of the tag colector @@ -211,7 +197,7 @@ CondDBESSource::CondDBESSource( const edm::ParameterSet& iConfig ) : for(it=itBeg;it!=itEnd;++it){ std::string connStr = m_connectionString; std::string tag = it->second.tagName(); - std::pair tagParams = parseTag( it->second.tagName() ); + std::pair tagParams = cond::persistency::parseTag( it->second.tagName() ); if( !tagParams.second.empty() ) { connStr = tagParams.second; tag = tagParams.first; @@ -407,7 +393,7 @@ CondDBESSource::setIntervalFor( const edm::eventsetup::EventSetupRecordKey& iKey //transId << "long" << m_lastRun; transId << m_lastRun; std::string connStr = m_connectionString; - std::pair tagParams = parseTag( tcIter->second.tagName() ); + std::pair tagParams = cond::persistency::parseTag( tcIter->second.tagName() ); if( !tagParams.second.empty() ) connStr = tagParams.second; std::map >::iterator iSess = m_sessionPool.find( connStr ); bool reopen = false; diff --git a/CondCore/ORA/interface/Container.h b/CondCore/ORA/interface/Container.h index 4828b3bf58e68..75583e68d691b 100644 --- a/CondCore/ORA/interface/Container.h +++ b/CondCore/ORA/interface/Container.h @@ -61,6 +61,8 @@ namespace ora { const std::string& className(); + std::string realClassName(); + const std::string& mappingVersion(); size_t size(); diff --git a/CondCore/ORA/interface/Object.h b/CondCore/ORA/interface/Object.h index 2a69d9a366ece..4588b4778ddcf 100644 --- a/CondCore/ORA/interface/Object.h +++ b/CondCore/ORA/interface/Object.h @@ -12,6 +12,9 @@ namespace ora { class Object { public: Object(); + template + explicit Object( const T& obj ); + Object( const void* ptr, const std::type_info& typeInfo ); Object( const void* ptr, const Reflex::Type& type ); Object( const void* ptr, const std::string& typeName ); Object( const Object& rhs); @@ -30,6 +33,20 @@ namespace ora { void* m_ptr; Reflex::Type m_type; }; + + template<> + inline + Object::Object( const Object& rhs ): + m_ptr( rhs.m_ptr ), + m_type( rhs.m_type ){ + } + +} + +template +inline +ora::Object::Object( const T& obj ): + Object( &obj, typeid(obj) ){ } template diff --git a/CondCore/ORA/src/Container.cc b/CondCore/ORA/src/Container.cc index 967e16ae6dd0e..7ca6363b5c030 100644 --- a/CondCore/ORA/src/Container.cc +++ b/CondCore/ORA/src/Container.cc @@ -76,6 +76,13 @@ const std::string& ora::Container::className(){ return m_dbContainer->className(); } +std::string ora::Container::realClassName(){ + Reflex::Type type = ClassUtils::lookupDictionary( className() ); + std::string ret = ClassUtils::demangledName( type.TypeInfo() ); + ret.erase( std::remove( ret.begin(), ret.end(), ' ' ), ret.end() ); + return ret; +} + const std::string& ora::Container::mappingVersion(){ return m_dbContainer->mappingVersion(); } diff --git a/CondCore/ORA/src/Database.cc b/CondCore/ORA/src/Database.cc index 03952c6c913cf..bba026e8b540f 100644 --- a/CondCore/ORA/src/Database.cc +++ b/CondCore/ORA/src/Database.cc @@ -46,7 +46,7 @@ namespace ora { contHandle = session.createContainer( name, contType ); } else { throwException("Container \""+name+"\" does not exist in the database.", - "Database::insertItem"); + "Database::getContainerFromSession"); } } @@ -301,7 +301,7 @@ ora::Object ora::Database::fetchItem(const OId& oid){ ora::OId ora::Database::insertItem(const std::string& containerName, const Object& dataObject ){ - open( true ); + open( true ); Container cont = getContainerFromSession( containerName, dataObject.type(), *m_impl->m_session ); int itemId = cont.insertItem( dataObject ); return OId( cont.id(), itemId ); diff --git a/CondCore/ORA/src/Object.cc b/CondCore/ORA/src/Object.cc index f031db232feb6..e7f8396126f56 100644 --- a/CondCore/ORA/src/Object.cc +++ b/CondCore/ORA/src/Object.cc @@ -9,14 +9,19 @@ ora::Object::Object(): m_type(){ } +ora::Object::Object( const void* ptr, const std::type_info& typeInfo ): + m_ptr( const_cast(ptr) ){ + m_type = ClassUtils::lookupDictionary( typeInfo, true ); +} + ora::Object::Object( const void* ptr, const Reflex::Type& type ): m_ptr( const_cast(ptr) ), - m_type( type ){ + m_type( type ){ } ora::Object::Object( const void* ptr, const std::string& typeName ): - m_ptr( const_cast(ptr) ), - m_type(Reflex::Type::ByName( typeName )){ + m_ptr( const_cast(ptr) ){ + m_type = ClassUtils::lookupDictionary( typeName, true ); } ora::Object::Object( const Object& rhs): @@ -65,8 +70,12 @@ void* ora::Object::cast( const std::type_info& typeInfo ) const{ return ClassUtils::upCast( m_type, m_ptr, castType ); } -boost::shared_ptr ora::Object::makeShared() const { - return boost::shared_ptr( m_ptr, RflxDeleter( m_type ) ); +boost::shared_ptr ora::Object::makeShared( ) const { + boost::shared_ptr ret; + if( m_ptr ) { + ret = boost::shared_ptr( m_ptr, RflxDeleter( m_type ) ); + } + return ret; } void ora::Object::destruct() { diff --git a/CondCore/Utilities/bin/conddb_migrate.cpp b/CondCore/Utilities/bin/conddb_migrate.cpp index 77e747bb0fbc0..795e604ba47c4 100644 --- a/CondCore/Utilities/bin/conddb_migrate.cpp +++ b/CondCore/Utilities/bin/conddb_migrate.cpp @@ -5,6 +5,7 @@ #include "CondCore/DBCommon/interface/Auth.h" #include "CondCore/CondDB/interface/ConnectionPool.h" +#include "CondCore/CondDB/interface/Utils.h" #include "CondCore/CondDB/interface/IOVEditor.h" #include "CondCore/CondDB/interface/IOVProxy.h" @@ -56,7 +57,7 @@ int cond::MigrateUtilities::execute(){ std::string destConnect = getOptionValue("destConnect" ); std::string sourceConnect = getOptionValue("sourceConnect"); - std::tuple connPars = parseConnectionString( sourceConnect ); + std::tuple connPars = persistency::parseConnectionString( sourceConnect ); if( std::get<0>( connPars ) == "frontier" ) throwException("Cannot migrate data from FronTier cache.","MigrateUtilities::execute"); std::cout <<"# Connecting to source database on "< 0 ) { - payloadType = *(sourceIov.payloadClasses().begin()); - } else { - std::string tk = sourceIov.begin()->token(); - payloadType = sourcedb.classNameForItem( tk ); - } + std::string tk = sourceIov.begin()->token(); + std::string payloadType = sourcedb.classNameForItem( tk ); + std::cout <<" Importing tag. Size:"<( gtitem ); std::cout <<"--> Processing tag "<( connectionData ); if( std::get<1>( connectionData )=="FrontierArc" ) { size_t len = account.size()-5; diff --git a/CondCore/Utilities/bin/conddb_test_gt_load.cpp b/CondCore/Utilities/bin/conddb_test_gt_load.cpp index 7b79e1ef3f81b..144a40f8a4a45 100644 --- a/CondCore/Utilities/bin/conddb_test_gt_load.cpp +++ b/CondCore/Utilities/bin/conddb_test_gt_load.cpp @@ -19,8 +19,6 @@ namespace cond { void load( const std::string& tag ); - void reload(); - void reset(); TimeType timeType() const; @@ -52,9 +50,7 @@ namespace cond { } cond::UntypedPayloadProxy::UntypedPayloadProxy( Session& session ): - m_session( session ), - m_iov( session.iovProxy() ), - m_data(){ + m_session( session ){ m_data.reset( new pimpl ); m_data->current.clear(); } @@ -74,12 +70,9 @@ cond::UntypedPayloadProxy& cond::UntypedPayloadProxy::operator=( const cond::Unt void cond::UntypedPayloadProxy::load( const std::string& tag ){ m_data->current.clear(); - m_iov.load( tag ); -} - -void cond::UntypedPayloadProxy::reload(){ - m_data->current.clear(); - m_iov.reload(); + m_session.transaction().start(); + m_iov = m_session.readIov( tag ); + m_session.transaction().commit(); } void cond::UntypedPayloadProxy::reset(){ @@ -105,6 +98,8 @@ bool cond::UntypedPayloadProxy::get( cond::Time_t targetTime, bool debug ){ // a new payload is required! if( debug )std::cout <<" Searching tag "<(targetTime),"UntypedPayloadProxy::get"); m_data->current = *iIov; @@ -113,10 +108,14 @@ bool cond::UntypedPayloadProxy::get( cond::Time_t targetTime, bool debug ){ std::string payloadType(""); Binary data; loaded = m_session.fetchPayloadData( m_data->current.payloadId, payloadType, data ); + m_session.transaction().commit(); if( !loaded ){ std::cout <<"ERROR: payload with id "<current.payloadId<<" could not be loaded."<current.since<<" - "<current.till<<"]"; @@ -173,21 +172,31 @@ int cond::TestGTLoad::execute(){ ConnectionPool connPool; if( hasDebug() ) connPool.setMessageVerbosity( coral::Debug ); + connPool.configure(); Session session = connPool.createSession( connect ); session.transaction().start(); std::cout <<"Loading Global Tag "< proxies; std::map requests; for( auto t: gt ){ - UntypedPayloadProxy p( session ); + std::pair tagParams = parseTag( t.tagName() ); + std::string tagConnStr = connect; + Session tagSession = session; + if( !tagParams.second.empty() ) { + tagConnStr = tagParams.second; + tagSession = connPool.createSession( tagConnStr ); + } + UntypedPayloadProxy p( tagSession ); try{ - p.load( t.tagName() ); + p.load( tagParams.first ); proxies.push_back( p ); - requests.insert( std::make_pair( t.tagName(), 0 ) ); + requests.insert( std::make_pair( tagParams.first, 0 ) ); } catch ( const cond::Exception& e ){ std::cout <<"ERROR: "<second++; - } else if( ttype==lumiid ){ - p.get( lumi, hasDebug() ); - r->second++; - } else if( ttype==timestamp){ - p.get( ts, hasDebug() ); - r->second++; - } else { - std::cout <<"WARNING: iov request on tag "<second++; + } else if( ttype==lumiid ){ + p.get( lumi, hasDebug() ); + r->second++; + } else if( ttype==timestamp){ + p.get( ts, hasDebug() ); + r->second++; + } else { + std::cout <<"WARNING: iov request on tag "<second<<" Queries:"<& hist = p.history(); - for( auto e: p.history() ) std::cout <<" "<second<<" Queries:"<& hist = p.history(); + for( auto e: p.history() ) std::cout <<" "<