Skip to content

Commit

Permalink
Merge pull request #14264 from mmusich/DropboxMetadata2016_80X
Browse files Browse the repository at this point in the history
DropBox metadata - 2016 (81X)
  • Loading branch information
davidlange6 committed Apr 28, 2016
2 parents a5daab9 + 997d623 commit b08c187
Show file tree
Hide file tree
Showing 13 changed files with 197 additions and 112 deletions.
8 changes: 2 additions & 6 deletions CondFormats/Common/test/BeamSpotObjectsRcdByLumi_prep.json
@@ -1,11 +1,7 @@
{
"destinationDatabase": "oracle://cms_orcoff_prep/CMS_COND_BEAMSPOT",
"destinationDatabase": "oracle://cms_orcoff_prep/CMS_CONDITIONS",
"destinationTags": {
"BeamSpotObjects_PCL_byLumi_v1_prompt": {
"dependencies": {
},
"synchronizeTo": "prompt"
}
"BeamSpotObjects_PCL_byLumi_v1_prompt": {}
},
"inputTag": "BeamSpotObject_ByLumi",
"since": null,
Expand Down
8 changes: 2 additions & 6 deletions CondFormats/Common/test/BeamSpotObjectsRcdByLumi_prod.json
@@ -1,11 +1,7 @@
{
"destinationDatabase": "oracle://cms_orcon_prod/CMS_COND_31X_BEAMSPOT",
"destinationDatabase": "oracle://cms_orcon_prod/CMS_CONDITIONS",
"destinationTags": {
"BeamSpotObjects_PCL_byLumi_v1_prompt": {
"dependencies": {
},
"synchronizeTo": "prompt"
}
"BeamSpotObjects_PCL_byLumi_v0_prompt": {}
},
"inputTag": "BeamSpotObject_ByLumi",
"since": null,
Expand Down
13 changes: 4 additions & 9 deletions CondFormats/Common/test/BeamSpotObjectsRcdByRun_prep.json
@@ -1,14 +1,9 @@
{
"destinationDatabase": "oracle://cms_orcoff_prep/CMS_COND_BEAMSPOT",
"destinationDatabase": "oracle://cms_orcoff_prep/CMS_CONDITIONS",
"destinationTags": {
"BeamSpotObjects_PCL_byRun_v1_offline": {
"dependencies": {
"BeamSpotObjects_PCL_byRun_v1_hlt": "hlt",
"BeamSpotObjects_PCL_byRun_v1_express": "express",
"BeamSpotObjects_PCL_byRun_v1_prompt": "prompt"
},
"synchronizeTo": "offline"
}
"BeamSpotObjects_PCL_byRun_v1_prompt": {},
"BeamSpotObjects_PCL_byRun_v1_express": {},
"BeamSpotObjects_PCL_byRun_v1_hlt": {}
},
"inputTag": "BeamSpotObject_ByRun",
"since": null,
Expand Down
13 changes: 4 additions & 9 deletions CondFormats/Common/test/BeamSpotObjectsRcdByRun_prod.json
@@ -1,14 +1,9 @@
{
"destinationDatabase": "oracle://cms_orcon_prod/CMS_COND_31X_BEAMSPOT",
"destinationDatabase": "oracle://cms_orcon_prod/CMS_CONDITIONS",
"destinationTags": {
"BeamSpotObjects_PCL_byRun_v1_offline": {
"dependencies": {
"BeamSpotObjects_PCL_byRun_v1_hlt": "hlt",
"BeamSpotObjects_PCL_byRun_v1_express": "express",
"BeamSpotObjects_PCL_byRun_v1_prompt": "prompt"
},
"synchronizeTo": "offline"
}
"BeamSpotObjects_PCL_byRun_v0_prompt": {},
"BeamSpotObjects_2009_v1_express": {},
"BeamSpotObjects_PCL_byRun_v0_hlt": {}
},
"inputTag": "BeamSpotObject_ByRun",
"since": null,
Expand Down
12 changes: 12 additions & 0 deletions CondFormats/Common/test/DropBoxMetadata_example.txt
@@ -0,0 +1,12 @@
{
"destinationDatabase": "oracle://cms_orcon_prod/CMS_COND_42X_RUN_INFO",
"destinationTags": {
"DropBoxMetadata_v1_express": {
"dependencies": {},
"synchronizeTo": "express"
}
},
"inputTag": "DropBoxMetadata",
"since": 1,
"userText": "Update drop-box metadata for PCL"
}
124 changes: 64 additions & 60 deletions CondFormats/Common/test/ProduceDropBoxMetadata.py
@@ -1,62 +1,47 @@
import FWCore.ParameterSet.Config as cms

process = cms.Process("myprocess")
process.load("CondCore.CondDB.CondDB_cfi")

process.CondDBCommon.connect = 'sqlite_file:DropBoxMetadata.db'
process.load("CondCore.CondDB.CondDB_cfi")

process.CondDB.connect = 'sqlite_file:DropBoxMetadata.db'

process.maxEvents = cms.untracked.PSet(
input = cms.untracked.int32(1)
)
process.source = cms.Source("EmptySource",
firstRun = cms.untracked.uint32(165200)
firstRun = cms.untracked.uint32(300000)
)

# process.PoolDBOutputService.DBParameters.messageLevel = 3
import json
# beamspot by run
BeamSpotObjectsRcdByRun_prod_file = open("BeamSpotObjectsRcdByRun_prod.json")
BeamSpotObjectsRcdByRun_prod_json = json.load(BeamSpotObjectsRcdByRun_prod_file)
BeamSpotObjectsRcdByRun_prod_file.close()
BeamSpotObjectsRcdByRun_prod_str = json.JSONEncoder().encode(BeamSpotObjectsRcdByRun_prod_json).replace('"',""")

def encodeJsonInString(filename):
"""This function open the json file and encodes it in a string replacing probelamtic characters"""
thefile = open(filename)
thejson = json.load(thefile)
thefile.close()
return json.JSONEncoder().encode(thejson).replace('"',""")


BeamSpotObjectsRcdByRun_prep_file = open("BeamSpotObjectsRcdByRun_prep.json")
BeamSpotObjectsRcdByRun_prep_json = json.load(BeamSpotObjectsRcdByRun_prep_file)
BeamSpotObjectsRcdByRun_prep_file.close()
BeamSpotObjectsRcdByRun_prep_str = json.JSONEncoder().encode(BeamSpotObjectsRcdByRun_prep_json).replace('"',""")

# beamspot by run
BeamSpotObjectsRcdByRun_prod_str = encodeJsonInString("BeamSpotObjectsRcdByRun_prod.json")
BeamSpotObjectsRcdByRun_prep_str = encodeJsonInString("BeamSpotObjectsRcdByRun_prep.json")

# beamspot by lumi
BeamSpotObjectsRcdByLumi_prod_file = open("BeamSpotObjectsRcdByLumi_prod.json")
BeamSpotObjectsRcdByLumi_prod_json = json.load(BeamSpotObjectsRcdByLumi_prod_file)
BeamSpotObjectsRcdByLumi_prod_file.close()
BeamSpotObjectsRcdByLumi_prod_str = json.JSONEncoder().encode(BeamSpotObjectsRcdByLumi_prod_json).replace('"',""")


BeamSpotObjectsRcdByLumi_prep_file = open("BeamSpotObjectsRcdByLumi_prep.json")
BeamSpotObjectsRcdByLumi_prep_json = json.load(BeamSpotObjectsRcdByLumi_prep_file)
BeamSpotObjectsRcdByLumi_prep_file.close()
BeamSpotObjectsRcdByLumi_prep_str = json.JSONEncoder().encode(BeamSpotObjectsRcdByLumi_prep_json).replace('"',""")


BeamSpotObjectsRcdByLumi_prod_str = encodeJsonInString("BeamSpotObjectsRcdByLumi_prod.json")
BeamSpotObjectsRcdByLumi_prep_str = encodeJsonInString("BeamSpotObjectsRcdByLumi_prep.json")

#SiStripBadStripRcd
SiStripBadStripRcd_prod_file = open("SiStripBadStripRcd_prod.json")
SiStripBadStripRcd_prod_json = json.load(SiStripBadStripRcd_prod_file)
SiStripBadStripRcd_prod_file.close()
SiStripBadStripRcd_prod_str = json.JSONEncoder().encode(SiStripBadStripRcd_prod_json).replace('"',""")



SiStripBadStripRcd_prep_file = open("SiStripBadStripRcd_prep.json")
SiStripBadStripRcd_prep_json = json.load(SiStripBadStripRcd_prep_file)
SiStripBadStripRcd_prep_file.close()
SiStripBadStripRcd_prep_str = json.JSONEncoder().encode(SiStripBadStripRcd_prep_json).replace('"',""")
SiStripBadStripRcd_prod_str = encodeJsonInString("SiStripBadStripRcd_prod.json")
SiStripBadStripRcd_prep_str = encodeJsonInString("SiStripBadStripRcd_prep.json")

#SiStripApvGainRcd
SiStripApvGainRcd_prod_str = encodeJsonInString("SiStripApvGainRcd_prod.json")
SiStripApvGainRcd_prep_str = encodeJsonInString("SiStripApvGainRcd_prep.json")

#SiPixelAli
SiPixelAliRcd_prod_str = encodeJsonInString("SiPixelAliRcd_prod.json")
SiPixelAliRcd_prep_str = encodeJsonInString("SiPixelAliRcd_prep.json")


process.mywriter = cms.EDAnalyzer("ProduceDropBoxMetadata",
Expand All @@ -78,37 +63,56 @@
FileClass = cms.untracked.string("ALCA"),
prodMetaData = cms.untracked.string(SiStripBadStripRcd_prod_str),
prepMetaData = cms.untracked.string(SiStripBadStripRcd_prep_str),
),
cms.PSet(record = cms.untracked.string('SiStripApvGainRcd'),
Source = cms.untracked.string("AlcaHarvesting"),
FileClass = cms.untracked.string("ALCA"),
prodMetaData = cms.untracked.string(SiStripApvGainRcd_prod_str),
prepMetaData = cms.untracked.string(SiStripApvGainRcd_prep_str),
),
cms.PSet(record = cms.untracked.string('TrackerAlignmentRcd'),
Source = cms.untracked.string("AlcaHarvesting"),
FileClass = cms.untracked.string("ALCA"),
prodMetaData = cms.untracked.string(SiPixelAliRcd_prod_str),
prepMetaData = cms.untracked.string(SiPixelAliRcd_prep_str),
)

),
read = cms.untracked.bool(True),
toRead = cms.untracked.vstring("BeamSpotObjectsRcdByRun",'BeamSpotObjectsRcdByLumi','SiStripBadStripRcd') # same strings as fType
toRead = cms.untracked.vstring("BeamSpotObjectsRcdByRun",'BeamSpotObjectsRcdByLumi','SiStripBadStripRcd','SiStripApvGainRcd','TrackerAlignmentRcd') # same strings as fType
)


process.p = cms.Path(process.mywriter)

from CondCore.CondDB.CondDB_cfi import CondDB
CondDB.connect = "sqlite_file:DropBoxMetadata.db"

process.PoolDBOutputService = cms.Service("PoolDBOutputService",
CondDB,
toPut = cms.VPSet(cms.PSet(record = cms.string('DropBoxMetadataRcd'),
tag = cms.string('DropBoxMetadata'),
timetype = cms.untracked.string('runnumber')
)
),
loadBlobStreamer = cms.untracked.bool(False),
# timetype = cms.untracked.string('lumiid')
# timetype = cms.untracked.string('runnumber')
)

if process.mywriter.write:

from CondCore.CondDB.CondDB_cfi import CondDB
CondDB.connect = "sqlite_file:DropBoxMetadata.db"

process.PoolDBOutputService = cms.Service("PoolDBOutputService",
CondDB,
toPut = cms.VPSet(cms.PSet(record = cms.string('DropBoxMetadataRcd'),
tag = cms.string('DropBoxMetadata'),
timetype = cms.untracked.string('runnumber')
)
),
loadBlobStreamer = cms.untracked.bool(False),
# timetype = cms.untracked.string('lumiid')
# timetype = cms.untracked.string('runnumber')
)

process.load('Configuration.StandardSequences.FrontierConditions_GlobalTag_cff')
process.GlobalTag.globaltag = 'GR_E_V31::All'
process.GlobalTag.globaltag = '80X_dataRun2_Express_Queue'

process.GlobalTag.connect = 'frontier://PromptProd/CMS_CONDITIONS'
#process.GlobalTag.connect = 'sqlite_file:/afs/cern.ch/user/c/cerminar/public/Alca/GlobalTag/GR_R_311_V2.db'

process.GlobalTag.toGet = cms.VPSet(
cms.PSet(record = cms.string("DropBoxMetadataRcd"),
tag = cms.string("DropBoxMetadata"),
connect = cms.untracked.string("sqlite_file:DropBoxMetadata.db")
)
)
readsqlite = False
if readsqlite:
process.GlobalTag.toGet = cms.VPSet(
cms.PSet(record = cms.string("DropBoxMetadataRcd"),
tag = cms.string("DropBoxMetadata"),
connect = cms.string("sqlite_file:DropBoxMetadata.db")
)
)
11 changes: 11 additions & 0 deletions CondFormats/Common/test/SiPixelAliRcd_prep.json
@@ -0,0 +1,11 @@
{
"destinationDatabase": "oracle://cms_orcoff_prep/CMS_CONDITIONS",
"destinationTags": {
"SiPixelAli_PCL_v0_prompt": {},
"SiPixelAli_PCL_v0_hlt": {}
},
"inputTag": "SiPixelAli_pcl",
"since": null,
"userText": "T0 PCL Upload for SiPixel Ali. (prep)"
}

11 changes: 11 additions & 0 deletions CondFormats/Common/test/SiPixelAliRcd_prod.json
@@ -0,0 +1,11 @@
{
"destinationDatabase": "oracle://cms_orcoff_prep/CMS_CONDITIONS",
"destinationTags": {
"SiPixelAli_PCL_v0_prompt": {},
"SiPixelAli_PCL_v0_hlt": {}
},
"inputTag": "SiPixelAli_pcl",
"since": null,
"userText": "T0 PCL Upload for SiPixel Ali. (prep)"
}

11 changes: 11 additions & 0 deletions CondFormats/Common/test/SiStripApvGainRcd_prep.json
@@ -0,0 +1,11 @@
{
"destinationDatabase": "oracle://cms_orcoff_prep/CMS_CONDITIONS",
"destinationTags": {
"SiStripApvGain_PCL_v0_prompt": {},
"SiStripApvGain_PCL_v0_hlt": {}
},
"inputTag": "SiStripApvGain_pcl",
"since": null,
"userText": "T0 PCL Upload for SiStrip Gains calib. (prod)"
}

11 changes: 11 additions & 0 deletions CondFormats/Common/test/SiStripApvGainRcd_prod.json
@@ -0,0 +1,11 @@
{
"destinationDatabase": "oracle://cms_orcoff_prep/CMS_CONDITIONS",
"destinationTags": {
"SiStripApvGain_PCL_v0_prompt": {},
"SiStripApvGain_PCL_v0_hlt": {}
},
"inputTag": "SiStripApvGain_pcl",
"since": null,
"userText": "T0 PCL Upload for SiStrip Gains calib. (prod)"
}

13 changes: 4 additions & 9 deletions CondFormats/Common/test/SiStripBadStripRcd_prep.json
@@ -1,16 +1,11 @@
{
"destinationDatabase": "oracle://cms_orcoff_prep/CMS_COND_STRIP",
"destinationDatabase": "oracle://cms_orcoff_prep/CMS_CONDITIONS",
"destinationTags": {
"SiStripBadChannel_PCL_v1_offline": {
"dependencies": {
"SiStripBadChannel_PCL_v1_hlt": "hlt",
"SiStripBadChannel_PCL_v1_prompt": "prompt"
},
"synchronizeTo": "offline"
}
"SiStripBadChannel_PCL_v1_prompt": {},
"SiStripBadChannel_PCL_v1_hlt": {}
},
"inputTag": "SiStripBadStrip_pcl",
"since": null,
"userText": "T0 PCL Upload for SiStrip bad-channles"
"userText": "T0 PCL Upload for SiStrip bad-channels"
}

19 changes: 6 additions & 13 deletions CondFormats/Common/test/SiStripBadStripRcd_prod.json
@@ -1,16 +1,9 @@
{
"destinationDatabase": "oracle://cms_orcoff_prep/CMS_COND_STRIP",
"destinationDatabase": "oracle://cms_orcon_prod/CMS_CONDITIONS",
"destinationTags": {
"SiStripBadChannel_PCL_v1_offline": {
"dependencies": {
"SiStripBadChannel_PCL_v1_hlt": "hlt",
"SiStripBadChannel_PCL_v1_prompt": "prompt"
},
"synchronizeTo": "offline"
}
},
"inputTag": "SiStripBadStrip_pcl",
"since": null,
"userText": "T0 PCL Upload for SiStrip bad-channles"
"SiStripBadChannel_FromOfflineCalibration_GR10_v1_prompt": {}
},
"inputTag": "SiStripBadStrip_pcl",
"since": null,
"userText": "T0 PCL Upload for SiStrip bad-channels"
}

55 changes: 55 additions & 0 deletions CondFormats/Common/test/parse_output.py
@@ -0,0 +1,55 @@
#!/usr/bin/env python
import os
import json


if __name__ == "__main__":
filename = "out_sqlite_v2.txt"
dirname = 'out_sqlite_v3'

filehandler = open(filename, 'r')
lines = filehandler.readlines()



try:
os.stat(dirname)
except:
os.mkdir(dirname)

recordname = None


for line in lines:


if '--- record' in line:
recordname = line.split()[2]
print '-----------------------------------------------------------------'
print 'record: ',recordname

if 'prepMetaData' in line:
dict_text = line.split('value: ')[1]
prep_metadata = json.loads(dict_text.replace('"','"'))
prep_metadata_dump = json.dumps(prep_metadata, sort_keys = True, indent = 4)
print '----- prepMetaData:'
print prep_metadata_dump
outFilePrep = open('%s/%s_prep.json'%(dirname,recordname), 'w')
outFilePrep.write(prep_metadata_dump+'\n')
outFilePrep.close()

if 'prodMetaData' in line:
dict_text = line.split('value: ')[1]
prod_metadata = json.loads(dict_text.replace('"','"'))
prod_metadata_dump = json.dumps(prod_metadata, sort_keys = True, indent = 4)
print '----- prodMetaData:'
print prod_metadata_dump
outFileProd = open('%s/%s_prod.json'%(dirname,recordname), 'w')
outFileProd.write(prod_metadata_dump+'\n')
outFileProd.close()


filehandler.close()
#print line


0 comments on commit b08c187

Please sign in to comment.