Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update conddb tool to make a copy of a full GT #10471

Merged
merged 3 commits into from Aug 18, 2015
Merged

Update conddb tool to make a copy of a full GT #10471

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5b95052
update DB name for prep
apfeiffer1 Jul 29, 2015
26050a6
use passed args for first and second instead of option values in _cop…
apfeiffer1 Jul 29, 2015
eefc813
handle copying of GT with intervals correctly. to be tested
apfeiffer1 Jul 29, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
8 changes: 4 additions & 4 deletions CondCore/Utilities/python/conddblib.py
View file
Open in desktop
Expand Up @@ -363,16 +363,16 @@ def connect(database='pro', init=False, verbose=0):
'pro': lambda: _getCMSFrontierSQLAlchemyConnectionString('PromptProd'),
'arc': lambda: _getCMSFrontierSQLAlchemyConnectionString('FrontierArc'),
'int': lambda: _getCMSFrontierSQLAlchemyConnectionString('FrontierInt'),
'dev': lambda: _getCMSFrontierSQLAlchemyConnectionString('FrontierPrep'),
'dev': lambda: _getCMSFrontierSQLAlchemyConnectionString('FrontierPrep', 'cms_conditions_002'),
'boost': lambda: _getCMSFrontierSQLAlchemyConnectionString('FrontierProd', 'cms_conditions'),
'boostprep': lambda: _getCMSFrontierSQLAlchemyConnectionString('FrontierPrep', 'cms_conditions'),
'boostprep': lambda: _getCMSFrontierSQLAlchemyConnectionString('FrontierPrep', 'cms_conditions_002'),

'orapro': lambda: _getCMSOracleSQLAlchemyConnectionString('cms_orcon_adg'),
'oraarc': lambda: _getCMSOracleSQLAlchemyConnectionString('cmsarc_lb'),
'oraint': lambda: _getCMSOracleSQLAlchemyConnectionString('cms_orcoff_int'),
'oradev': lambda: _getCMSOracleSQLAlchemyConnectionString('cms_orcoff_prep'),
'oradev': lambda: _getCMSOracleSQLAlchemyConnectionString('cms_orcoff_prep', 'cms_conditions_002'),
'oraboost': lambda: _getCMSOracleSQLAlchemyConnectionString('cms_orcon_adg' , 'cms_conditions'),
'oraboostprep': lambda: _getCMSOracleSQLAlchemyConnectionString('cms_orcoff_prep', 'cms_conditions'),
'oraboostprep': lambda: _getCMSOracleSQLAlchemyConnectionString('cms_orcoff_prep', 'cms_conditions_002'),

'onlineorapro': lambda: _getCMSOracleSQLAlchemyConnectionString('cms_orcon_prod'),
'onlineoraint': lambda: _getCMSOracleSQLAlchemyConnectionString('cmsintr_lb'),
Expand Down
95 changes: 77 additions & 18 deletions CondCore/Utilities/scripts/conddb
View file
Open in desktop
Expand Up @@ -14,6 +14,7 @@ import stat
import subprocess
import tempfile
import textwrap
import time

import sqlalchemy

Expand Down Expand Up @@ -144,7 +145,7 @@ def _exists(session, primary_key, value):
try:
ret = session.query(primary_key).\
filter(primary_key == value).\
count() != 0
count() != 0
except sqlalchemy.exc.OperationalError:
pass

Expand Down Expand Up @@ -853,14 +854,62 @@ def diff(args):
if not (is_tag1 and is_tag2) and not (is_global_tag1 and is_global_tag2):
raise Exception('There are no tag or global tag pairs named %s and %s in the database(s).' % (args.first, args.second))

def convertRunToTimes(startRun, stopRun=None):

def _copy_tag(args, session1, session2, first, second):
if not stopRun :
stopRun = startRun + 1

startTime1, stopTime1 = runToTime(startRun)
startTime2, stopTime2 = runToTime(stopRun)

timeMap = { 'start' : {
'hash' : None,
'run' : startRun,
'time' : startTime1-15., # the time we get may be a bit delayed (7-10 sec according to Salvatore)
'lumi' : startRun<<32|0x1,
},
'stop' : {
'hash' : None,
'run' : stopRun,
'time' : stopTime2+15., # the time we get may be a bit delayed (7-10 sec according to Salvatore)
'lumi' : stopRun<<32|0x1,
}
}

logging.debug("convertRunToTimes> start: %s stop %s \n timeMap: %s " % (startRun, stopRun, str(timeMap)))

return timeMap

def runToTime(runNr):

connStr = conddb._getCMSFrontierSQLAlchemyConnectionString('PromptProd', 'CMS_CONDITIONS')

connection = conddb.connect(connStr)
session = connection.session()

startIOV = session.query(conddb.IOV.insertion_time).filter(conddb.IOV.tag_name == 'runinfo_start_31X_hlt', conddb.IOV.since == runNr).all()
stopIOV = session.query(conddb.IOV.insertion_time).filter(conddb.IOV.tag_name == 'runinfo_31X_hlt', conddb.IOV.since == runNr).all()

startTime = time.mktime( startIOV[0][0].timetuple() )
stopTime = time.mktime( stopIOV [0][0].timetuple() )

return startTime, stopTime

def _copy_tag(args, session1, session2, first, second, fromIOV=None, toIOV=None, timeMap=None):
logging.info('Copying tag %s to %s ...', str_db_object(args.db, first), str_db_object(args.destdb, second))

# Copy the tag
tag = _rawdict(session1.query(conddb.Tag).get(args.first))
tag['name'] = args.second
tag = _rawdict(session1.query(conddb.Tag).get(first))
tag['name'] = second
tag['end_of_validity'] = 0 # XXX: SQLite does not work with long ints...

if not timeMap and not fromIOV and not toIOV:
raise Exception("_copy_tag> One of timeMap, fromIOV, toIOV needs to be given ...")

if timeMap and not fromIOV and not toIOV:
fromIOV = timeMap['start'][ tag['time_type'].lower().strip() ]
toIOV = timeMap['stop'] [ tag['time_type'].lower().strip() ]

session2.add(conddb.Tag(**tag))

# Get the closest smaller IOV than the given starting point (args.from),
Expand All @@ -869,25 +918,29 @@ def _copy_tag(args, session1, session2, first, second):
# In the case where the starting point is before any IOV, we do not need
# to cut the query anyway.
prev_iov = None
if getattr(args, 'from') is not None:
if fromIOV is not None:
fromVal = fromIOV
logging.debug("checking FROM %s of type %s for tag: %s " % (fromIOV, tag['time_type'], str(tag['name'])) )
prev_iov = session1.query(conddb.IOV.since).\
filter(
conddb.IOV.tag_name == args.first,
conddb.IOV.since <= getattr(args, 'from'),
conddb.IOV.tag_name == first,
conddb.IOV.since <= fromVal,
).\
order_by(conddb.IOV.since.desc()).\
limit(1).\
scalar()
logging.debug('The closest smaller IOV than the given starting one (--from) is %s...', prev_iov)
logging.debug('The closest smaller IOV than the given starting one (--from %s) is %s...', fromVal, prev_iov)


# Copy the distinct payloads referenced in the IOVs of the tag
# FIXME: Put the DISTINCT query as a subquery (we can't directly use distinct on BLOBs)
query = session1.query(conddb.IOV.payload_hash).filter(conddb.IOV.tag_name == args.first)
query = session1.query(conddb.IOV.payload_hash).filter(conddb.IOV.tag_name == first)
if prev_iov is not None:
query = query.filter(conddb.IOV.since >= prev_iov)
if args.to is not None:
query = query.filter(conddb.IOV.since <= args.to)
if toIOV is not None:
toVal = toIOV
logging.debug("filtering with TO %s of type %s for tag: %s to " % (toIOV, tag['time_type'], str(tag['name'])) )
query = query.filter(conddb.IOV.since <= toVal)
query = query.distinct()
for (payload_hash, ) in query:
if _exists(session2, conddb.Payload.hash, payload_hash):
Expand All @@ -897,15 +950,15 @@ def _copy_tag(args, session1, session2, first, second):
session2.add(conddb.Payload(** _rawdict(session1.query(conddb.Payload).filter(conddb.Payload.hash == payload_hash).one())))

# Copy the IOVs of the tag
query = session1.query(conddb.IOV).filter(conddb.IOV.tag_name == args.first)
query = session1.query(conddb.IOV).filter(conddb.IOV.tag_name == first)
if prev_iov is not None:
query = query.filter(conddb.IOV.since >= prev_iov)
if args.to is not None:
query = query.filter(conddb.IOV.since <= args.to)
if toIOV is not None:
query = query.filter(conddb.IOV.since <= toIOV)
for iov in query:
logging.debug('Copying IOV %s -> %s...', str_iov(iov.since, iov.insertion_time), iov.payload_hash)
iov = _rawdict(iov)
iov['tag_name'] = args.second
iov['tag_name'] = second

# In the first IOV of the tag we need to use the starting point given
# by the user, instead of the one coming from the source tag; unless
Expand Down Expand Up @@ -947,7 +1000,7 @@ def copy(args):
if args.second is None:
args.second = args.first

_copy_tag(args, session1, session2, args.first, args.second)
_copy_tag(args, session1, session2, args.first, args.second, getattr(args, 'from'), args.to)

_confirm_changes(args)
session2.commit()
Expand All @@ -957,20 +1010,26 @@ def copy(args):
if args.second is None:
args.second = args.first

timeMap = convertRunToTimes(getattr(args, 'from'), args.to)

logging.info('Copying global tag %s to %s ...', str_db_object(args.db, args.first), str_db_object(args.destdb, args.second))

# Copy the global tag
global_tag = _rawdict(session1.query(conddb.GlobalTag).get(args.first))
global_tag['name'] = args.second
global_tag['validity'] = 0 # XXX: SQLite does not work with long ints...
session2.add(conddb.GlobalTag(**global_tag))

# Copy the tags of the global tag
logging.debug('Creating query for tag %s filter %s ...', conddb.GlobalTagMap.tag_name, args.first)
query = session1.query(conddb.GlobalTagMap.tag_name).filter(conddb.GlobalTagMap.global_tag_name == args.first).distinct()
for (tag, ) in query:
if _exists(session2, conddb.Tag.name, tag):
logging.debug('Copying tag %s to %s for GT %s ...', str_db_object(args.db, tag), str_db_object(args.destdb, tag), str_db_object(args.destdb, args.second))
if _exists(session2, conddb.Tag.name, tag ):
logging.warn('Skipping copy of tag %s to %s since it already exists... *The tags may differ in content*', str_db_object(args.db, tag), str_db_object(args.destdb, tag))
else:
_copy_tag(args, session1, session2, tag, tag)
logging.debug('going to copy tag %s to %s ... ', str_db_object(args.db, tag), str_db_object(args.destdb, tag))
_copy_tag(args, session1, session2, tag, tag, timeMap=timeMap)

# Copy the map of the global tag
query = session1.query(conddb.GlobalTagMap).filter(conddb.GlobalTagMap.global_tag_name == args.first)
Expand Down