Fixes for RunInfo O2O and cleanup of PopCon interface

CondCore/CondDB/src/IOVEditor.cc

@@ -182,7 +182,11 @@ namespace cond {
       if( m_data->iovBuffer.size() ) {
 	std::sort(m_data->iovBuffer.begin(),m_data->iovBuffer.end(),iovSorter);
 	cond::Time_t l = std::get<0>(m_data->iovBuffer.front());
-	if( m_data->synchronizationType != cond::SYNCH_ANY && m_data->synchronizationType != cond::SYNCH_VALIDATION ){
+  //We do not allow for IOV updates (i.e. insertion in the past or overriding) on tags whose syncrosization is not "ANY" or "VALIDATION".
+  //This policy is stricter than the one deployed in the Condition Upload service,
+  //which allows insertions in the past or overriding for IOVs larger than the first condition safe run for HLT ("HLT"/"EXPRESS" synchronizations) and Tier0 ("PROMPT"/"PCL").
+  //This is intended: in the C++ API we have not got a way to determine the first condition safe runs.
+  if( m_data->synchronizationType != cond::SYNCH_ANY && m_data->synchronizationType != cond::SYNCH_VALIDATION ){
 	  // retrieve the last since
 	  cond::Time_t last = 0;
 	  cond::Hash h;

CondCore/PopCon/interface/PopCon.h

@@ -72,18 +72,17 @@ namespace popcon {
 
     bool m_LoggingOn;
 
-    bool m_IsDestDbCheckedInQueryLog;
-
     std::string m_tag;
 
     cond::TagInfo_t m_tagInfo;
 
     cond::LogDBEntry_t m_logDBEntry;
 
     bool m_close;
+
     Time_t m_lastTill;
 
-
+    static constexpr const char* const s_version = "5.0";
   };
 
 
@@ -103,15 +102,15 @@ namespace popcon {
 
   template<typename Container>
   const std::string displayIovHelper(Container const & payloads) {
-    if (payloads.empty()) return "Nothing to transfer;";
-    std::ostringstream s;    
+    if (payloads.empty()) return std::string("Nothing to transfer;");
+    std::ostringstream s;
     // when only 1 payload is transferred; 
-    if ( payloads.size()==1)  
-      s <<"Since " << (*payloads.begin()).time <<  "; " ;
+    if ( payloads.size()==1)
+      s << "Since " << (*payloads.begin()).time <<  "; " ;
     else{
       // when more than one payload are transferred;  
-      s <<   "first payload Since " <<  (*payloads.begin()).time <<  ","
-	<< "last payload Since "  << (*payloads.rbegin()).time <<  ";" ;  
+      s << "first payload Since " <<  (*payloads.begin()).time <<  ", "
+        << "last payload Since "  << (*payloads.rbegin()).time <<  "; " ;  
     }  
     return s.str();
   }
@@ -129,12 +128,12 @@ namespace popcon {
   								 m_tagInfo,m_logDBEntry); 
     Container const & payloads = *ret.first;
 
-    if(m_LoggingOn)
-      m_dbService->setLogHeaderForRecord(m_record,source.id(),"PopCon v4.0; " + 
-					 displayIovHelper(payloads) +  ret.second);
-    //m_dbService->setLogHeaderForRecord(m_record,source.id(),"PopCon v4.0; " + 
-    //				 cond::userInfo() + displayIovHelper(payloads) +  ret.second);
-
+    if(m_LoggingOn) {
+      std::ostringstream s;
+      s << "PopCon v" << s_version << "; " << displayIovHelper(payloads) << ret.second;
+      //s << "PopCon v" << s_version << "; " << cond::userInfo() << displayIovHelper(payloads) << ret.second;
+      m_dbService->setLogHeaderForRecord(m_record,source.id(),s.str());
+    }
     displayHelper(payloads);
 
     std::for_each(payloads.begin(),payloads.end(),

CondCore/PopCon/src/PopCon.cc

@@ -15,15 +15,14 @@ namespace popcon {
     m_record(pset.getParameter<std::string> ("record")),
     m_payload_name(pset.getUntrackedParameter<std::string> ("name","")),
     m_LoggingOn(pset.getUntrackedParameter< bool > ("loggingOn",true)),
-    m_IsDestDbCheckedInQueryLog( pset.getUntrackedParameter< bool >("IsDestDbCheckedInQueryLog",true)),
     m_close(pset.getUntrackedParameter< bool > ("closeIOV",false)),
     m_lastTill(pset.getUntrackedParameter< bool > ("lastTill",0))
     {
       //TODO set the policy (cfg or global configuration?)
       //Policy if corrupted data found
 
-      edm::LogInfo ("PopCon") << "This is PopCon (Populator of Condition) V4.0\n"
-			      << "Please report any problem and feature request through the savannah portal under the category conditions\n" ; 
+      edm::LogInfo ("PopCon") << "This is PopCon (Populator of Condition) v" << s_version << ".\n"
+                              << "Please report any problem and feature request through the JIRA project CMSCONDDB.\n" ; 
     }
 
   PopCon::~PopCon(){
@@ -53,30 +52,22 @@ namespace popcon {
       m_tagInfo.name = m_tag;
       m_tagInfo.size = iov.sequenceSize();
       if( m_tagInfo.size>0 ){
-	cond::Iov_t last = iov.getLast();
-	m_tagInfo.lastInterval = cond::ValidityInterval( last.since, last.till );
-	m_tagInfo.lastPayloadToken = last.payloadId;
+        cond::Iov_t last = iov.getLast();
+        m_tagInfo.lastInterval = cond::ValidityInterval( last.since, last.till );
+        m_tagInfo.lastPayloadToken = last.payloadId;
       }
 
-      //if( m_IsDestDbCheckedInQueryLog ) {
-      //m_dbService->queryLog().LookupLastEntryByTag( m_tag, connectionStr, m_logDBEntry );
-      //std::cout <<" ------ log info searched in the same db: "<< connectionStr << "------" <<std::endl;
-      //} else {
-      //m_dbService->queryLog().LookupLastEntryByTag( m_tag, m_logDBEntry );
-      //std::cout <<" ------ log info found in another db "<< "------" <<std::endl;
-      //}
-
-      edm::LogInfo ("PopCon") << "DB: " << connectionStr << "\n"
-			      << "TAG: " << m_tag 
-			      << ", last since/till: " <<  m_tagInfo.lastInterval.first
-			      << "/" << m_tagInfo.lastInterval.second
-			      << ", , size: " << m_tagInfo.size << "\n"
-			      << "Last writer: " <<  m_logDBEntry.provenance 
-			      << ", size: " << m_logDBEntry.payloadIdx+1 << std::endl;
+      edm::LogInfo ("PopCon") << "destination DB: " << connectionStr
+                              << ", target DB: " << ( m_targetConnectionString.empty() ? connectionStr : m_targetConnectionString ) << "\n"
+                              << "TAG: " << m_tag
+                              << ", last since/till: " <<  m_tagInfo.lastInterval.first
+                              << "/" << m_tagInfo.lastInterval.second
+                              << ", size: " << m_tagInfo.size << "\n" << std::endl;
     } else {
-      edm::LogInfo ("PopCon") << "DB: " << connectionStr << "\n"
-			      << "TAG: " << m_tag 
-			      << "; First writer to this new tag." << std::endl; 
+      edm::LogInfo ("PopCon") << "destination DB: " << connectionStr
+                              << ", target DB: " << ( m_targetConnectionString.empty() ? connectionStr : m_targetConnectionString ) << "\n"
+                              << "TAG: " << m_tag
+                              << "; First writer to this new tag." << std::endl;
     }
     return m_targetSession;
   }

CondTools/RunInfo/interface/RunInfoHandler.h

@@ -5,7 +5,7 @@
 
 #include "CondCore/PopCon/interface/PopConSourceHandler.h"
 #include "CondFormats/RunInfo/interface/RunInfo.h"
-#include "FWCore/ParameterSet/interface/ParameterSetfwd.h"
+#include "FWCore/ParameterSet/interface/ParameterSet.h"
 
 class RunInfoHandler : public popcon::PopConSourceHandler<RunInfo>{
  public:
@@ -15,14 +15,14 @@ class RunInfoHandler : public popcon::PopConSourceHandler<RunInfo>{
   RunInfoHandler(const edm::ParameterSet& pset); 
 
  private:
-  std::string m_name;
   unsigned long long m_since;
+  std::string m_name;
 
   // for reading from omds
+  std::string m_runinfo_schema;
+  std::string m_dcsenv_schema;
   std::string m_connectionString;
-  std::string m_authpath;
-  std::string m_user;
-  std::string m_pass;
+  edm::ParameterSet m_connectionPset;
 };
 
 #endif 

CondTools/RunInfo/interface/RunInfoRead.h

@@ -1,24 +1,19 @@
 #ifndef CondTools_RunInfo_RunInfoRead_h
 #define CondTools_RunInfo_RunInfoRead_h
 
-#include "CondTools/RunInfo/interface/TestBase.h"
 #include "CondFormats/RunInfo/interface/RunInfo.h"
+#include "FWCore/ParameterSet/interface/ParameterSet.h"
 #include <string>
 
-class RunInfoRead : virtual public TestBase {
+class RunInfoRead {
  public:
   RunInfoRead(const std::string& connectionString,
-	      const std::string& user,
-	      const std::string& pass);
-  virtual ~RunInfoRead();
-  void run();
-  RunInfo readData(const std::string& table, const std::string& column, const int r_number);
+	      const edm::ParameterSet& connectionPset);
+  ~RunInfoRead();
+  RunInfo readData(const std::string& runinfo_schema, const std::string& dcsenv_schema, const int r_number);
  private:
-  std::string m_tableToRead;
-  std::string m_columnToRead;
   std::string m_connectionString;
-  std::string m_user;
-  std::string m_pass;
+  edm::ParameterSet m_connectionPset;
 };
 
 #endif

CondTools/RunInfo/src/RunInfoHandler.cc

@@ -1,55 +1,70 @@
 #include "FWCore/MessageLogger/interface/MessageLogger.h"
-#include "FWCore/ParameterSet/interface/ParameterSet.h"
 #include "CondTools/RunInfo/interface/RunInfoHandler.h"
 #include "CondTools/RunInfo/interface/RunInfoRead.h"
 #include<iostream>
 #include<vector>
 
-RunInfoHandler::RunInfoHandler(const edm::ParameterSet& pset) :
-   m_name( pset.getUntrackedParameter<std::string>( "name", "RunInfoHandler") )
-  ,m_since( pset.getParameter<unsigned long long>( "runNumber" ) )
-  ,m_connectionString( pset.getUntrackedParameter<std::string>( "connectionString", "oracle://cms_omds_adg/CMS_RUNINFO") )
-  ,m_authpath( pset.getUntrackedParameter<std::string>( "authenticationPath", "." ) )
-  ,m_user( pset.getUntrackedParameter<std::string>( "OnlineDBUser", "CMS_RUNINFO_R" ) )
-  ,m_pass( pset.getUntrackedParameter<std::string>( "OnlineDBPass", "PASSWORD") ) {
+RunInfoHandler::RunInfoHandler( const edm::ParameterSet& pset ) :
+   m_since( pset.getParameter<unsigned long long>( "runNumber" ) )
+  ,m_name( pset.getUntrackedParameter<std::string>( "name", "RunInfoHandler" ) )
+  ,m_runinfo_schema( pset.getUntrackedParameter<std::string>( "RunInfoSchema", "CMS_RUNINFO" ) )
+  ,m_dcsenv_schema( pset.getUntrackedParameter<std::string>( "DCSEnvSchema", "CMS_DCS_ENV_PVSS_COND") )
+  ,m_connectionString( pset.getParameter<std::string>( "connect" ) )
+  ,m_connectionPset( pset.getParameter<edm::ParameterSet>( "DBParameters" ) ) {
 }
 
 RunInfoHandler::~RunInfoHandler() {}
 
 void RunInfoHandler::getNewObjects() {
   //check whats already inside of database
-  edm::LogInfo( "RunInfoHandler" ) << "------- " << m_name
-                                   << " - > getNewObjects\n"
-                                   << "got offlineInfo " << tagInfo().name
+  edm::LogInfo( "RunInfoHandler" ) << "[" << "RunInfoHandler::" << __func__ << "]:" << m_name << ": "
+                                   << "Destination Tag Info: name " << tagInfo().name
                                    << ", size " << tagInfo().size
                                    << ", last object valid since " << tagInfo().lastInterval.first
-                                   << " token " << tagInfo().lastPayloadToken << std::endl;
-  edm::LogInfo( "RunInfoHandler" ) << "runnumber/first since = " << m_since << std::endl;
+                                   << ", hash " << tagInfo().lastPayloadToken << std::endl;
+  edm::LogInfo( "RunInfoHandler" ) << "[" << "RunInfoHandler::" << __func__ << "]:" << m_name << ": runnumber/first since = " << m_since << std::endl;
+
+  //check if a transfer is needed:
+  //if the new run number is smaller than or equal to the latest IOV, exit.
+  //This is needed as now the IOV Editor does not always protect for insertions:
+  //ANY and VALIDATION sychronizations are allowed to write in the past.
+  if( tagInfo().size > 0  && tagInfo().lastInterval.first >= m_since ) {
+    edm::LogWarning( "RunInfoHandler" ) << "[" << "RunInfoHandler::" << __func__ << "]:" << m_name << ": "
+                                        << "last IOV " << tagInfo().lastInterval.first
+                                        << ( tagInfo().lastInterval.first == m_since ? " is equal to" : " is larger than" )
+                                        << " the run proposed for insertion " << m_since
+                                        << ". No transfer needed." << std::endl;
+    return;
+  }
+
   RunInfo* r = new RunInfo();
 
   //fill with null runinfo if empty run are found beetween the two last valid ones 
   size_t n_empty_run = 0;
   if( tagInfo().size > 0  && (tagInfo().lastInterval.first + 1) < m_since ) {
     n_empty_run = m_since - tagInfo().lastInterval.first - 1;
-    edm::LogInfo( "RunInfoHandler" ) << "------- " << "entering fake run from "
+    edm::LogInfo( "RunInfoHandler" ) << "[" << "RunInfoHandler::" << __func__ << "]:" << m_name << ": "
+                                     << "entering fake run from "
                                      << tagInfo().lastInterval.first + 1
-                                     <<  "to " << m_since - 1 << "- > getNewObjects"
+                                     <<  " to " << m_since - 1
                                      << std::endl;
   } 
+  std::ostringstream ss;
   // transfer fake run for 1 to since for the first time
   if( tagInfo().size == 0 && m_since != 1 ) {
     m_to_transfer.push_back( std::make_pair( (RunInfo*)(r->Fake_RunInfo()), 1 ) );
+    ss << "fake run number: " << 1 << ", ";
   }
   if ( n_empty_run != 0 ) {
     m_to_transfer.push_back(std::make_pair( (RunInfo*)(r->Fake_RunInfo()), tagInfo().lastInterval.first + 1 ) );
+    ss << "fake run number: " << tagInfo().lastInterval.first + 1 << ", ";
   }
 
   //reading from omds
-  RunInfoRead rn( m_connectionString, m_user, m_pass );
-  *r = rn.readData( "RUNSESSION_PARAMETER", "STRING_VALUE",(int)m_since );
+  RunInfoRead rn( m_connectionString, m_connectionPset );
+  *r = rn.readData( m_runinfo_schema, m_dcsenv_schema, (int)m_since );
   m_to_transfer.push_back( std::make_pair( (RunInfo*)r, m_since) );
-  std::ostringstream ss;
-  ss << "since =" << m_since;
-  m_userTextLog = ss.str() + ";";
-  edm::LogInfo( "RunInfoHandler" ) << "------- " << m_name << " - > getNewObjects" << std::endl;
+  ss << "run number: " << m_since << ";";
+  m_userTextLog = ss.str();
+  edm::LogInfo( "RunInfoHandler" ) << "[" << "RunInfoHandler::" << __func__ << "]:" << m_name << ": END." << std::endl;
 }