Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

62X DBS3 API transition #2313

Merged
1 commit merged into from Feb 6, 2014
Merged

62X DBS3 API transition #2313

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
77 changes: 36 additions & 41 deletions PhysicsTools/PatAlgos/python/patInputFiles_cff.py
View file
Open in desktop
Expand Up @@ -2,51 +2,46 @@
from PhysicsTools.PatAlgos.tools.cmsswVersionTools import pickRelValInputFiles

filesRelValProdTTbarAODSIM = cms.untracked.vstring(
pickRelValInputFiles( cmsswVersion = 'CMSSW_6_2_0_pre8'
, relVal = 'RelValProdTTbar'
, globalTag = 'PRE_ST62_V8'
, dataTier = 'AODSIM'
, maxVersions = 1
, numberOfFiles = 1
)
#pickRelValInputFiles( cmsswVersion = 'CMSSW_6_2_0_pre8'
#, relVal = 'RelValProdTTbar'
#, globalTag = 'PRE_ST62_V8'
#, dataTier = 'AODSIM'
#, maxVersions = 1
#, numberOfFiles = 1
#, useDAS = True
#)
'/store/relval/CMSSW_6_2_0_pre8/RelValProdTTbar/AODSIM/PRE_ST62_V8-v1/00000/AC6C769D-4AE0-E211-AFD0-5404A63886AC.root'
)
filesRelValProdTTbarGENSIMRECO = cms.untracked.vstring(
pickRelValInputFiles( cmsswVersion = 'CMSSW_6_2_0_pre5' # event content in 620pre8 broken
, relVal = 'RelValProdTTbar'
, globalTag = 'PRE_ST61_V1'
, dataTier = 'GEN-SIM-RECO'
, maxVersions = 1
, numberOfFiles = 1
)
#pickRelValInputFiles( cmsswVersion = 'CMSSW_6_2_0_pre5' # event content in 620pre8 broken
#, relVal = 'RelValProdTTbar'
#, globalTag = 'PRE_ST61_V1'
#, dataTier = 'GEN-SIM-RECO'
#, maxVersions = 1
#, numberOfFiles = 1
#, useDAS = True
#)
'/store/relval/CMSSW_6_2_0_pre5/RelValProdTTbar/GEN-SIM-RECO/PRE_ST61_V1-v1/00000/EA73A371-77A4-E211-98E0-003048F0E320.root'
)
filesRelValTTbarPileUpGENSIMRECO = cms.untracked.vstring(
pickRelValInputFiles( cmsswVersion = 'CMSSW_6_2_0_pre5' # event content in 620pre8 broken
, relVal = 'RelValTTbar'
, globalTag = 'PU_PRE_ST61_V1'
, dataTier = 'GEN-SIM-RECO'
, maxVersions = 1
, numberOfFiles = 1
)
#pickRelValInputFiles( cmsswVersion = 'CMSSW_6_2_0_pre5' # event content in 620pre8 broken
#, relVal = 'RelValTTbar'
#, globalTag = 'PU_PRE_ST61_V1'
#, dataTier = 'GEN-SIM-RECO'
#, maxVersions = 1
#, numberOfFiles = 1
#, useDAS = True
#)
'/store/relval/CMSSW_6_2_0_pre5/RelValTTbar/GEN-SIM-RECO/PU_PRE_ST61_V1-v1/00000/E877A0D2-DEA5-E211-9EA4-002481E0D73C.root'
)
filesSingleMuRECO = cms.untracked.vstring(
pickRelValInputFiles( cmsswVersion = 'CMSSW_6_2_0_pre8'
, relVal = 'SingleMu'
, dataTier = 'RECO'
, globalTag = 'PRE_62_V8_RelVal_mu2012D'
, maxVersions = 1
, numberOfFiles = 1
)
# only one block available at CERN
# FIXME: need to fix DBS query in 'pickRelValInputFiles' to identify them properly
# ==> query for file requiring dataset AND site does not work in DBS :-(
#'/store/relval/CMSSW_6_2_0_pre8/SingleMu/RECO/PRE_62_V8_RelVal_mu2012B-v1/00000/029F8FA5-D7E0-E211-BCCF-001E67398430.root',
#'/store/relval/CMSSW_6_2_0_pre8/SingleMu/RECO/PRE_62_V8_RelVal_mu2012B-v1/00000/1A9A0FE7-D5E0-E211-9868-003048F01164.root',
#'/store/relval/CMSSW_6_2_0_pre8/SingleMu/RECO/PRE_62_V8_RelVal_mu2012B-v1/00000/4E0F44F0-D5E0-E211-9A0A-003048CF6780.root',
#'/store/relval/CMSSW_6_2_0_pre8/SingleMu/RECO/PRE_62_V8_RelVal_mu2012B-v1/00000/6E43C4C0-DBE0-E211-B452-003048D37366.root',
#'/store/relval/CMSSW_6_2_0_pre8/SingleMu/RECO/PRE_62_V8_RelVal_mu2012B-v1/00000/A426EC8A-DAE0-E211-BE58-D8D385FF4A94.root',
#'/store/relval/CMSSW_6_2_0_pre8/SingleMu/RECO/PRE_62_V8_RelVal_mu2012B-v1/00000/A45F6ADF-D8E0-E211-948B-003048FE9D54.root',
#'/store/relval/CMSSW_6_2_0_pre8/SingleMu/RECO/PRE_62_V8_RelVal_mu2012B-v1/00000/CE1D92A1-D9E0-E211-B7F0-C860001BD934.root',
#'/store/relval/CMSSW_6_2_0_pre8/SingleMu/RECO/PRE_62_V8_RelVal_mu2012B-v1/00000/D61C12DD-E9E0-E211-8A9E-5404A63886D2.root',
#'/store/relval/CMSSW_6_2_0_pre8/SingleMu/RECO/PRE_62_V8_RelVal_mu2012B-v1/00000/E4F44285-DFE0-E211-BEA9-0025B3203918.root',
#'/store/relval/CMSSW_6_2_0_pre8/SingleMu/RECO/PRE_62_V8_RelVal_mu2012B-v1/00000/F848CDC3-D6E0-E211-BA88-003048F009C4.root'
#pickRelValInputFiles( cmsswVersion = 'CMSSW_6_2_0_pre8'
#, relVal = 'SingleMu'
#, dataTier = 'RECO'
#, globalTag = 'PRE_62_V8_RelVal_mu2012D'
#, maxVersions = 1
#, numberOfFiles = 1
#, useDAS = True
#)
'/store/relval/CMSSW_6_2_0_pre8/SingleMu/RECO/PRE_62_V8_RelVal_mu2012D-v1/00001/FCF80FBD-ECDF-E211-A99E-001E6739721F.root'
)
111 changes: 53 additions & 58 deletions PhysicsTools/PatAlgos/python/tools/cmsswVersionTools.py
View file
Open in desktop
Expand Up @@ -268,15 +268,12 @@ def apply( self ):
print '%s DEBUG: Querying dataset \'%s\' with'%( self._label, dataset )
print ' \'%s\''%( dasQuery )
# partially stolen from das_client.py for option '--format=plain', needs filter ("grep") in the query
dasData = das_client.get_data( 'https://cmsweb.cern.ch', dasQuery, 0, dasLimit, False )
jsondict = json.loads( dasData )
jsondict = das_client.get_data( 'https://cmsweb.cern.ch', dasQuery, 0, dasLimit, False )
if debug:
print '%s DEBUG: Received DAS data:'%( self._label )
print ' \'%s\''%( dasData )
print '%s DEBUG: Determined JSON dictionary:'%( self._label )
print '%s DEBUG: Received DAS JSON dictionary:'%( self._label )
print ' \'%s\''%( jsondict )
if jsondict[ 'status' ] != 'ok':
print 'There was a problem while querying DAS with query \'%s\'. Server reply was:\n %s' % (dasQuery, dasData)
print 'There was a problem while querying DAS with query \'%s\'. Server reply was:\n %s' % (dasQuery, jsondict)
exit( 1 )
mongo_query = jsondict[ 'mongo_query' ]
filters = mongo_query[ 'filters' ]
Expand All @@ -289,20 +286,17 @@ def apply( self ):
print '%s DEBUG: Data in JSON dictionary:'%( self._label )
print ' \'%s\''%( data )
for row in data:
filePath = [ r for r in das_client.get_value( row, filters ) ][ 0 ]
filePath = [ r for r in das_client.get_value( row, filters[ 'grep' ] ) ][ 0 ]
if debug:
print '%s DEBUG: Testing file entry \'%s\''%( self._label, filePath )
if len( filePath ) > 0:
if validVersion != version:
dasTest = das_client.get_data( 'https://cmsweb.cern.ch', 'site dataset=%s | grep site.name'%( dataset ), 0, 999, False )
jsontestdict = json.loads( dasTest )
jsontestdict = das_client.get_data( 'https://cmsweb.cern.ch', 'site dataset=%s | grep site.name'%( dataset ), 0, 999, False )
mongo_testquery = jsontestdict[ 'mongo_query' ]
testfilters = mongo_testquery[ 'filters' ]
testdata = jsontestdict[ 'data' ]
if debug:
print '%s DEBUG: Received DAS data (site test):'%( self._label )
print ' \'%s\''%( dasTest )
print '%s DEBUG: Determined JSON dictionary (site test):'%( self._label )
print '%s DEBUG: Received DAS JSON dictionary (site test):'%( self._label )
print ' \'%s\''%( jsontestdict )
print '%s DEBUG: Query in JSON dictionary (site test):'%( self._label )
print ' \'%s\''%( mongo_testquery )
Expand All @@ -312,7 +306,7 @@ def apply( self ):
print ' \'%s\''%( testdata )
foundSE = False
for testrow in testdata:
siteName = [ tr for tr in das_client.get_value( testrow, testfilters ) ][ 0 ]
siteName = [ tr for tr in das_client.get_value( testrow, testfilters[ 'grep' ] ) ][ 0 ]
if siteName == domainSE:
foundSE = True
break
Expand Down Expand Up @@ -343,57 +337,58 @@ def apply( self ):
else:
if debug:
print '%s DEBUG: Using DBS query'%( self._label )
for version in range( maxVersions, 0, -1 ):
filePaths = []
fileCount = 0
dataset = '/%s/%s-%s-v%i/%s'%( relVal, cmsswVersion, globalTag, version, dataTier )
dbsQuery = 'find file where dataset = %s'%( dataset )
if debug:
print '%s DEBUG: Querying dataset \'%s\' with'%( self._label, dataset )
print ' \'%s\''%( dbsQuery )
foundSE = False
for line in os.popen( 'dbs search --query="%s"'%( dbsQuery ) ):
if line.find( '.root' ) != -1:
if validVersion != version:
if not foundSE:
dbsSiteQuery = 'find dataset where dataset = %s and site = %s'%( dataset, domainSE )
if debug:
print '%s DEBUG: Querying site \'%s\' with'%( self._label, domainSE )
print ' \'%s\''%( dbsSiteQuery )
for lineSite in os.popen( 'dbs search --query="%s"'%( dbsSiteQuery ) ):
if lineSite.find( dataset ) != -1:
foundSE = True
break
if not foundSE:
if debug:
print '%s DEBUG: Possible version \'v%s\' not available on SE \'%s\''%( self._label, version, domainSE )
break
validVersion = version
if debug:
print '%s DEBUG: Valid version set to \'v%i\''%( self._label, validVersion )
if numberOfFiles == 0:
break
filePath = line.replace( '\n', '' )
if debug:
print '%s DEBUG: File \'%s\' found'%( self._label, filePath )
fileCount += 1
if fileCount > skipFiles:
filePaths.append( filePath )
if not numberOfFiles < 0:
if numberOfFiles <= len( filePaths ):
break
if validVersion > 0:
if numberOfFiles == 0 and debug:
print '%s DEBUG: No files requested'%( self._label )
break
print '%s WARNING: DBS query disabled for DBS3 transition to new API'%( self._label )
#for version in range( maxVersions, 0, -1 ):
#filePaths = []
#fileCount = 0
#dataset = '/%s/%s-%s-v%i/%s'%( relVal, cmsswVersion, globalTag, version, dataTier )
#dbsQuery = 'find file where dataset = %s'%( dataset )
#if debug:
#print '%s DEBUG: Querying dataset \'%s\' with'%( self._label, dataset )
#print ' \'%s\''%( dbsQuery )
#foundSE = False
#for line in os.popen( 'dbs search --query="%s"'%( dbsQuery ) ).readlines():
#if line.find( '.root' ) != -1:
#if validVersion != version:
#if not foundSE:
#dbsSiteQuery = 'find dataset where dataset = %s and site = %s'%( dataset, domainSE )
#if debug:
#print '%s DEBUG: Querying site \'%s\' with'%( self._label, domainSE )
#print ' \'%s\''%( dbsSiteQuery )
#for lineSite in os.popen( 'dbs search --query="%s"'%( dbsSiteQuery ) ).readlines():
#if lineSite.find( dataset ) != -1:
#foundSE = True
#break
#if not foundSE:
#if debug:
#print '%s DEBUG: Possible version \'v%s\' not available on SE \'%s\''%( self._label, version, domainSE )
#break
#validVersion = version
#if debug:
#print '%s DEBUG: Valid version set to \'v%i\''%( self._label, validVersion )
#if numberOfFiles == 0:
#break
#filePath = line.replace( '\n', '' )
#if debug:
#print '%s DEBUG: File \'%s\' found'%( self._label, filePath )
#fileCount += 1
#if fileCount > skipFiles:
#filePaths.append( filePath )
#if not numberOfFiles < 0:
#if numberOfFiles <= len( filePaths ):
#break
#if validVersion > 0:
#if numberOfFiles == 0 and debug:
#print '%s DEBUG: No files requested'%( self._label )
#break

# Check output and return
if validVersion == 0:
print '%s INFO : No RelVal file(s) found at all in datasets \'%s*\' on SE \'%s\''%( self._label, datasetAll, domainSE )
print '%s WARNING : No RelVal file(s) found at all in datasets \'%s*\' on SE \'%s\''%( self._label, datasetAll, domainSE )
if debug:
self.messageEmptyList()
elif len( filePaths ) == 0:
print '%s INFO : No RelVal file(s) picked up in dataset \'%s\''%( self._label, dataset )
print '%s WARNING : No RelVal file(s) picked up in dataset \'%s\''%( self._label, dataset )
if debug:
self.messageEmptyList()
elif len( filePaths ) < numberOfFiles:
Expand Down
2 changes: 1 addition & 1 deletion TopQuarkAnalysis/TopEventProducers/test/tqaf_woGeneratorInfo_cfg.py
View file
Open in desktop
Expand Up @@ -25,7 +25,7 @@
process.load("Configuration.Geometry.GeometryIdeal_cff")
process.load("Configuration.StandardSequences.FrontierConditions_GlobalTag_cff")
from Configuration.AlCa.GlobalTag import GlobalTag
process.GlobalTag = GlobalTag(process.GlobalTag, 'auto:com10_7E33v4')
process.GlobalTag = GlobalTag(process.GlobalTag, 'auto:com10_8E33v2')
process.load("Configuration.StandardSequences.MagneticField_cff")

#-------------------------------------------------
Expand Down