Merge pull request #14733 from ggovi/condcore-utilities-o2o-2-81X

Changes to allow to use the db writer for the monitoring

CondCore/Utilities/python/o2o.py

@@ -9,8 +9,8 @@
 import netrc
 import logging
 
-prod_db_service = 'cms_orcon_prod'
-dev_db_service = 'cms_orcoff_prep'
+prod_db_service = ['cms_orcon_prod','o2o_prod']
+dev_db_service = ['cms_orcoff_prep','o2o_dev']
 schema_name = 'CMS_CONDITIONS'
 oracle_tpl = 'oracle://%s:%s@%s'
 private_db = 'sqlite:///o2o_jobs.db'
@@ -31,28 +31,30 @@
 
 class O2OJob(_Base):
     __tablename__      = 'O2O_JOB'
+    __table_args__     = {'schema' : schema_name}
     name               = sqlalchemy.Column(sqlalchemy.String(100),    primary_key=True)
     enabled            = sqlalchemy.Column(sqlalchemy.Integer,        nullable=False)
     tag_name           = sqlalchemy.Column(sqlalchemy.String(100),    nullable=False)
     interval           = sqlalchemy.Column(sqlalchemy.Integer,        nullable=False)
 
 class O2ORun(_Base):
     __tablename__      = 'O2O_RUN'
+    __table_args__     = {'schema' : schema_name}
     job_name           = sqlalchemy.Column(sqlalchemy.String(100),    primary_key=True)
     start_time         = sqlalchemy.Column(sqlalchemy.TIMESTAMP,      primary_key=True)
     end_time           = sqlalchemy.Column(sqlalchemy.TIMESTAMP,      nullable=True)
     status_code        = sqlalchemy.Column(sqlalchemy.Integer,        nullable=False)
     log                = sqlalchemy.Column(sqlalchemy.CLOB,           nullable=True)
 
-def get_db_credentials( serviceName, authFile ):
+def get_db_credentials( db_service, authFile ):
     pwd = None
     if authFile is None:
        if authPathEnvVar in os.environ:
             authPath = os.environ[authPathEnvVar]
             authFile = os.path.join(authPath,'.netrc')
             logging.debug('Retrieving credentials from file %s' %authFile )
-    (username, account, pwd) = netrc.netrc( authFile ).authenticators(serviceName)
-    return pwd
+    (username, account, pwd) = netrc.netrc( authFile ).authenticators(db_service[1])
+    return username,pwd
 
 
 class O2OMgr(object):
@@ -74,14 +76,15 @@ def getSession( self, db_service, auth ):
         else:
             self.logger.info('Getting credentials')
             try:
-                pwd = get_db_credentials( db_service, auth )
+                username, pwd = get_db_credentials( db_service, auth )
             except Exception as e:
                 logging.debug(str(e))
+                username = None
                 pwd = None
-            if not pwd:
-                logging.error('Credentials for service %s are not available',db_service)
+            if username is None:
+                logging.error('Credentials for service %s are not available',db_service[0])
                 return None
-            url = oracle_tpl %(schema_name,pwd,db_service)
+            url = oracle_tpl %(username,pwd,db_service[0])
         session = None
         try:
             self.eng = sqlalchemy.create_engine( url )

CondCore/Utilities/python/popcon2dropbox.py

@@ -17,10 +17,10 @@
 import upload_popcon
 
 class CondMetaData(object):
-   def __init__( self ):
+   def __init__( self, fileName ):
       self.md = {}
       self.datef = datetime.now()
-      with open(confFileName) as jf:
+      with open(fileName) as jf:
          try:
             self.md = json.load(jf)
          except ValueError as e:
@@ -86,7 +86,7 @@ def runO2O( cmsswdir, releasepath, release, arch, jobfilename, logfilename, *p )
 
 def upload_to_dropbox( backend ):
 
-    md = CondMetaData() 
+    md = CondMetaData(confFileName) 
     # check if the expected input file is there...
     if not os.path.exists( dbFileForDropBox ):
        print 'The input sqlite file has not been produced.'
@@ -120,8 +120,8 @@ def upload_to_dropbox( backend ):
        print e
        return False
 
-def upload():
-    md = CondMetaData()
+def upload( cfileName, authPath ):
+    md = CondMetaData(cfileName)
 
     # check if the expected input file is there...                                                                                                                              
     if not os.path.exists( dbFileForDropBox ):
@@ -158,6 +158,8 @@ def upload():
        comment = v.get("comment")
        metadata = md.dumpMetadataForUpload( inputTag, destTag, comment )
        uploadCommand = 'uploadConditions.py %s' %fileNameForDropBox
+       if not authPath is None:
+          uploadCommand += ' -a %s' %authPath
        try:
           pipe = subprocess.Popen( uploadCommand, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
           stdout = pipe.communicate()[0]
@@ -182,14 +184,16 @@ def upload():
           print e
     return ret
 
-def run( jobfilename, *p ):
+def run( jobfilename, authPath ):
+    fns = os.path.splitext( jobfilename )
+    confFile = '%s.json' %fns[0]
     if os.path.exists( '%s.db' %fileNameForDropBox ):
        print "Removing files with name %s" %fileNameForDropBox
        os.remove( '%s.db' %fileNameForDropBox )
     if os.path.exists( '%s.txt' %fileNameForDropBox ):
        os.remove( '%s.txt' %fileNameForDropBox )
     command = 'cmsRun %s ' %jobfilename
-    command += ' '.join(p)
+    command += ' popconConfigFileName=%s' %confFile
     command += ' 2>&1'
     pipe = subprocess.Popen( command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
     stdout = pipe.communicate()[0]
@@ -199,4 +203,4 @@ def run( jobfilename, *p ):
     if retCode!=0:
        print 'O2O job failed. Skipping upload.'
        return retCode
-    return upload()
+    return upload( confFile, authPath )

CondCore/Utilities/python/popcon2dropbox_job_conf.py

@@ -1,7 +1,18 @@
 import FWCore.ParameterSet.Config as cms
-
+import FWCore.ParameterSet.VarParsing as VarParsing
 import popcon2dropbox
-md = popcon2dropbox.CondMetaData()
+
+options = VarParsing.VarParsing()
+options.register('popconConfigFileName',
+                'popcon2dropbox.json',
+                VarParsing.VarParsing.multiplicity.singleton,
+                VarParsing.VarParsing.varType.string,
+                "PopCon config file name")
+
+options.parseArguments()
+
+md = popcon2dropbox.CondMetaData( options.popconConfigFileName )
+
 psetForRec = []
 for k,v in md.records().items():
     psetForRec.append( cms.PSet( record = cms.string(str(k)),
@@ -22,16 +33,17 @@
                                         timetype = cms.untracked.string(str(v.get('timetype')))
                                         )
                               )
-print psetForOutRec
 
-process = cms.Process("TEST")
+destinationDatabase = md.destinationDatabase()
+
+
+process = cms.Process("PopCon")
 process.load("CondCore.CondDB.CondDB_cfi")
 process.CondDB.DBParameters.messageLevel = cms.untracked.int32( 3 )
 process.CondDB.connect = 'sqlite:%s' %popcon2dropbox.dbFileForDropBox
 
 process.PoolDBOutputService = cms.Service("PoolDBOutputService",
     process.CondDB,
-    logconnect = cms.untracked.string('sqlite:%s' %popcon2dropbox.dbLogFile),
     toPut = cms.VPSet( psetForOutRec )
 )
 
@@ -42,5 +54,4 @@
     interval   = cms.uint64(1)
 )
 
-print process.CondDB.connect
 

CondCore/Utilities/scripts/popconRun

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+'''Script that directs the popcon output to the condition uploader
+'''
+import os
+import sys
+from CondCore.Utilities import popcon2dropbox
+
+import optparse
+import argparse
+
+def main( argv ):
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("job_file", type=str, help="wrapper for popcon jobs")
+    parser.add_argument("-a","--auth", type=str,  help="authentication path (for conddb key and .netrc files)")
+    args = parser.parse_args()  
+
+    if args.job_file is None:
+        print 'ERROR: the cfg file name has not been provided.'
+        return -1
+    if not os.path.exists( args.job_file ):
+        print 'ERROR: the specified cfg file %s has not been found in the execution directory' %args.job_file
+        return -2
+    fns = os.path.splitext( args.job_file )
+    confFile = '%s.json' %fns[0]
+    if not os.path.exists( confFile ):
+        print 'ERROR: the metadata file %s has not been found in the execution directory' %confFile
+        return -3
+
+    return popcon2dropbox.run( args.job_file, args.auth )
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv))
+
+

CondCore/Utilities/scripts/uploadConditions.py

@@ -589,9 +589,12 @@ def uploadFile(self, filename, backend = defaultBackend, temporaryFile = default
 
 def authenticateUser(dropBox, options):
 
+    netrcPath = None
+    if options.authPath is not None:
+        netrcPath = os.path.join( options.authPath,'.netrc' )
     try:
         # Try to find the netrc entry
-        (username, account, password) = netrc.netrc().authenticators(options.netrcHost)
+        (username, account, password) = netrc.netrc( netrcPath ).authenticators(options.netrcHost)
     except Exception:
         # netrc entry not found, ask for the username and password
         logging.info(
@@ -813,6 +816,12 @@ def main():
         help = 'The netrc host (machine) from where the username and password will be read. Default: %default',
     )
 
+    parser.add_option('-a', '--authPath',
+        dest = 'authPath',
+        default = None,
+        help = 'The path of the .netrc file for the authentication. Default: $HOME',
+    )
+
     (options, arguments) = parser.parse_args()
 
     if len(arguments) < 1: