From 232ec4253e3d259986cfdb9706f698c39fe34f6c Mon Sep 17 00:00:00 2001 From: Giacomo Govi Date: Thu, 19 Jan 2017 11:07:47 +0100 Subject: [PATCH 1/2] Removing haks no longer required in sqlalchemy 1.1.4 --- CondCore/Utilities/python/conddblib.py | 23 ++--------------------- CondCore/Utilities/python/credentials.py | 7 ++++++- 2 files changed, 8 insertions(+), 22 deletions(-) diff --git a/CondCore/Utilities/python/conddblib.py b/CondCore/Utilities/python/conddblib.py index 905cc9073b53f..256064ccc2ba0 100644 --- a/CondCore/Utilities/python/conddblib.py +++ b/CondCore/Utilities/python/conddblib.py @@ -273,10 +273,8 @@ class IOV: 'payload_hash':(DbRef(Payload,'hash'),_Col.pk) } -# the string 'GLOBAL' being a keyword in sqlalchemy ( upper case ), when used in the model cause the two GT tables to be unreadable ( bug ) -# the only choice is to use lower case names, and rename the tables in sqlite after creation!! class GlobalTag: - __tablename__ = 'global_tag' + __tablename__ = 'GLOBAL_TAG' columns = { 'name':(sqlalchemy.String(name_length),_Col.pk), 'validity': (sqlalchemy.BIGINT,_Col.notNull), 'description':(sqlalchemy.String(description_length),_Col.notNull), @@ -285,7 +283,7 @@ class GlobalTag: 'snapshot_time':(sqlalchemy.TIMESTAMP,_Col.notNull) } class GlobalTagMap: - __tablename__ = 'global_tag_map' + __tablename__ = 'GLOBAL_TAG_MAP' columns = { 'global_tag_name':(DbRef(GlobalTag,'name'),_Col.pk), 'record':(sqlalchemy.String(name_length),_Col.pk), 'label':(sqlalchemy.String(name_length),_Col.pk), @@ -430,23 +428,6 @@ def init(self, drop=False): self.get_dbtype(TagLog).__table__.create(bind = self.engine) self.get_dbtype(GlobalTag).__table__.create(bind = self.engine) self.get_dbtype(GlobalTagMap).__table__.create(bind = self.engine) - #self.metadata.create_all(self.engine) - if self.is_sqlite: - # horrible hack, but no choice because of the sqlalchemy bug ( see comment in the model) - import sqlite3 - import string - conn = sqlite3.connect( self._url.database ) - c = conn.cursor() - stmt = string.Template('ALTER TABLE $before RENAME TO $after') - c.execute( stmt.substitute( before=GlobalTag.__tablename__, after='TMP0' ) ) - c.execute( stmt.substitute( before='TMP0', after=GlobalTag.__tablename__.upper() ) ) - c.execute( stmt.substitute( before=GlobalTagMap.__tablename__, after='TMP1' ) ) - c.execute( stmt.substitute( before='TMP1', after=GlobalTagMap.__tablename__.upper() ) ) - conn.commit() - conn.close() - # TODO: Create indexes - #logger.debug('Creating indexes...') - # Connection helpers def _getCMSFrontierConnectionString(database): diff --git a/CondCore/Utilities/python/credentials.py b/CondCore/Utilities/python/credentials.py index 5b2d0b1889714..d5b1c5ce8219a 100644 --- a/CondCore/Utilities/python/credentials.py +++ b/CondCore/Utilities/python/credentials.py @@ -1,7 +1,12 @@ import netrc import os +netrcFileName = '.netrc' + def get_credentials_from_file( service, authFile=None ): + if not authFile is None: + if os.path.isdir(authFile): + authFile = os.path.join( authFile, netrcFileName ) creds = netrc.netrc( authFile ).authenticators(service) return creds @@ -9,6 +14,6 @@ def get_credentials( authPathEnvVar, service, authFile=None ): if authFile is None: if authPathEnvVar in os.environ: authPath = os.environ[authPathEnvVar] - authFile = os.path.join(authPath,'.netrc') + authFile = os.path.join(authPath, netrcFileName) return get_credentials_from_file( service, authFile ) From 85189e2d60eeb263ce3be2480be50a615161119c Mon Sep 17 00:00:00 2001 From: Giacomo Govi Date: Thu, 19 Jan 2017 11:08:22 +0100 Subject: [PATCH 2/2] Added o2oMgr with new functionalities --- CondCore/Utilities/python/o2o.py | 250 ++++++++++++++++++++----- CondCore/Utilities/scripts/o2oMgr.py | 126 +++++++++++++ CondCore/Utilities/scripts/o2oRun.py | 10 +- CondCore/Utilities/scripts/o2oSetup.py | 52 ----- 4 files changed, 340 insertions(+), 98 deletions(-) create mode 100755 CondCore/Utilities/scripts/o2oMgr.py delete mode 100755 CondCore/Utilities/scripts/o2oSetup.py diff --git a/CondCore/Utilities/python/o2o.py b/CondCore/Utilities/python/o2o.py index a906a58038de9..5c4b8214229c4 100644 --- a/CondCore/Utilities/python/o2o.py +++ b/CondCore/Utilities/python/o2o.py @@ -8,6 +8,7 @@ import sys import logging import string +import json import CondCore.Utilities.credentials as auth @@ -40,19 +41,59 @@ class O2OJob(_Base): tag_name = sqlalchemy.Column(sqlalchemy.String(100), nullable=False) interval = sqlalchemy.Column(sqlalchemy.Integer, nullable=False) +class O2OJobConf(_Base): + __tablename__ = 'O2O_JOB_CONF' + __table_args__ = {'schema' : schema_name} + job_name = sqlalchemy.Column(sqlalchemy.ForeignKey(O2OJob.name), primary_key=True) + insertion_time = sqlalchemy.Column(sqlalchemy.TIMESTAMP, primary_key=True) + configuration = sqlalchemy.Column(sqlalchemy.String(4000), nullable=False) + + job = sqlalchemy.orm.relationship('O2OJob', primaryjoin="O2OJob.name==O2OJobConf.job_name") + class O2ORun(_Base): __tablename__ = 'O2O_RUN' __table_args__ = {'schema' : schema_name} - job_name = sqlalchemy.Column(sqlalchemy.String(100), primary_key=True) + job_name = sqlalchemy.Column(sqlalchemy.ForeignKey(O2OJob.name), primary_key=True) start_time = sqlalchemy.Column(sqlalchemy.TIMESTAMP, primary_key=True) end_time = sqlalchemy.Column(sqlalchemy.TIMESTAMP, nullable=True) status_code = sqlalchemy.Column(sqlalchemy.Integer, nullable=False) log = sqlalchemy.Column(sqlalchemy.CLOB, nullable=True) + job = sqlalchemy.orm.relationship('O2OJob', primaryjoin="O2OJob.name==O2ORun.job_name") + def get_db_credentials( db_service, authFile ): (username, account, pwd) = auth.get_credentials( authPathEnvVar, db_service[1], authFile ) return username,pwd +def print_table( headers, table ): + ws = [] + for h in headers: + ws.append(len(h)) + for row in table: + ind = 0 + for c in row: + if ind ws[ind]: + ws[ind] = len(c) + ind += 1 + + def printf( row ): + line = '' + ind = 0 + for w in ws: + fmt = '{:<%s}' %w + if ind1: + nkeys.append( "-"+keys[1] ) + nkeys.append( "--"+keys[0] ) + action = "store" + if 'type' not in params.keys(): + params['action'] = "store_true" + self.parser.add_argument(*nkeys,**params ) + self.options.add( keys[0] ) + + def addCommand( self, command_name, help_entry, *requiredOptions ): + self.parser.add_argument("--"+command_name, action="store_true", help=help_entry ) + for opt in requiredOptions: + if opt not in self.options: + raise Exception("Option '%s' has not been registered." %opt ) + self.commands[command_name] = requiredOptions + + def setup(): + return + + def execute(self): + self.args = self.parser.parse_args() + executed = False + self.setup() + for k in self.commands.keys(): + if getattr(self.args,k): + if executed: + print 'Ignoring command %s...' %k + else: + required_options = self.commands[k] + for o in required_options: + val = getattr(self.args,o) + if val is None: + raise Exception( "Required option '%s' has not been specified." %o ) + func = getattr(self,k) + func() + executed = True + return executed + +class O2OMgrTool(CommandTool): + def __init__(self): + CommandTool.__init__(self) + self.mgr = None + CommandTool.addOption(self,"name", "n", type=str, help="the o2o job name") + CommandTool.addOption(self,"configFile", "c", type=str, help="the JSON configuration file path") + CommandTool.addOption(self,"interval", "i", type=int, help="the chron job interval") + CommandTool.addOption(self,"db", type=str, help="the target database: pro ( for prod ) or dev ( for prep ). default=pro") + CommandTool.addOption(self,"auth","a", type=str, help="path of the authentication file") + CommandTool.addCommand( self,"create", "create a new O2O job", "name","configFile","interval") + CommandTool.addCommand(self,"setConfig","set a new configuration for the specified job","name","configFile" ) + CommandTool.addCommand(self,"setInterval","set a new execution interval for the specified job","name","interval" ) + CommandTool.addCommand(self,"enable","enable the O2O job","name" ) + CommandTool.addCommand(self,"disable", "disable the O2O job" , "name") + CommandTool.addCommand(self,"migrate", "migrate the tag info for the jobs in configuration entries" ) + CommandTool.addCommand(self,"listJobs", "list the registered jobs" ) + CommandTool.addCommand(self,"listConf", "shows the configurations for the specified job", "name") + + def setup(self): + db_service = o2olib.prod_db_service + if self.args.db is not None: + if self.args.db == 'dev' or self.args.db == 'oradev' : + db_service = o2olib.dev_db_service + elif self.args.db != 'orapro' and self.args.db != 'onlineorapro' and self.args.db != 'pro': + raise Exception("Database '%s' is not known." %self.args.db ) + + self.mgr = o2olib.O2OJobMgr() + return self.mgr.connect( db_service, self.args.auth ) + + def create(self): + self.mgr.add( self.args.name, self.args.configFile, self.args.interval, True ) + + def setConfig(self): + self.mgr.setConfig( self.args.name, self.args.configFile ) + + def setInterval(self): + self.mgr.setConfig( self.args.name, self.args.interval ) + + def enable(self): + self.mgr.setConfig( self.args.name, True ) + + def disable(self): + self.mgr.setConfig( self.args.name, False ) + + def migrate(self): + self.mgr.migrateConfig() + + def listJobs(self): + self.mgr.listJobs() + + def listConf(self): + self.mgr.listConfig( self.args.name ) + +def main( argv ): + + tool = O2OMgrTool() + ret = False + try: + ret = tool.execute() + except Exception as e: + print e + return ret + +if __name__ == '__main__': + + sys.exit(main(sys.argv)) diff --git a/CondCore/Utilities/scripts/o2oRun.py b/CondCore/Utilities/scripts/o2oRun.py index f49a6940fddeb..d8a04e23ae90a 100755 --- a/CondCore/Utilities/scripts/o2oRun.py +++ b/CondCore/Utilities/scripts/o2oRun.py @@ -14,8 +14,7 @@ def main( argv ): parser = argparse.ArgumentParser() parser.add_argument("executable", type=str, help="wrapper for O2O jobs") parser.add_argument("-n","--name", type=str, help="the O2O job name" ) - parser.add_argument("-i","--inputFromDb", action="store_true", help="get input params from the db ( labels: '$db' for the database connection string; '$tag' for the destination tag name") - parser.add_argument("-d","--dev", action="store_true", help="bookkeeping in dev database") + parser.add_argument("--db", type=str, help="the target database: pro ( for prod ) or dev ( for prep ). default=pro") parser.add_argument("-a","--auth", type=str, help="path of the authentication file") parser.add_argument("-v","--verbose", action="store_true", help="job output mirrored to screen (default=logfile only)") args = parser.parse_args() @@ -24,8 +23,11 @@ def main( argv ): parser.error("Job name not given.") db_service = o2olib.prod_db_service - if args.dev: - db_service = o2olib.dev_db_service + if args.db is not None: + if args.db == 'dev' or args.db == 'oradev' : + db_service = o2olib.dev_db_service + elif args.db != 'orapro' and args.db != 'onlineorapro' and args.db != 'pro': + raise Exception("Database '%s' is not known." %self.args.db ) runMgr = o2olib.O2ORunMgr() ret = -1 if runMgr.connect( db_service, args ): diff --git a/CondCore/Utilities/scripts/o2oSetup.py b/CondCore/Utilities/scripts/o2oSetup.py deleted file mode 100755 index c1808ea18716d..0000000000000 --- a/CondCore/Utilities/scripts/o2oSetup.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python - -''' -''' - -__author__ = 'Giacomo Govi' - -import CondCore.Utilities.o2o as o2olib -import sys -import optparse -import argparse - -def main( argv ): - - parser = argparse.ArgumentParser() - parser.add_argument("-c","--create", type=str, help="create a new O2O job" ) - parser.add_argument("-1","--enable", type=str, help="enable the O2O job" ) - parser.add_argument("-0","--disable", type=str, help="disable the O2O job" ) - - parser.add_argument("-t","--tag", type=str, help="the CondDB Tag name") - parser.add_argument("-i","--interval", type=str, help="the chron job interval") - parser.add_argument("-d","--dev", action="store_true", help="bookkeeping in dev database") - parser.add_argument("-a","--auth", type=str, help="path of the authentication file") - args = parser.parse_args() - - if not args.create and not args.enable and not args.disable: - parser.error("Command not given. Possible choices: create, enable, disable") - - db_service = o2olib.prod_db_service - if args.dev: - db_service = o2olib.dev_db_service - mgr = o2olib.O2OJobMgr() - ret = -1 - if mgr.connect( db_service, args.auth ): - if args.create: - if not args.tag: - parser.error("Option 'tag' not provided.") - if not args.interval: - parser.error("Option 'interval' not provided.") - print 'creating...' - created = mgr.add( args.create, args.tag, args.interval, True ) - return created - if args.enable: - mgr.set( args.enable, True ) - if args.disable: - mgr.set( args.disable, False ) - ret = 0 - return ret - -if __name__ == '__main__': - - sys.exit(main(sys.argv))