Skip to content

Commit

Permalink
Merge pull request #31694 from mmusich/updatePVValidationPerRunSubmis…
Browse files Browse the repository at this point in the history 
…sion_11_2_X

[TkAl] Update PV Validation and PV Resolution per-run submission tools
  • Loading branch information
@cmsbuild
cmsbuild committed Oct 14, 2020
2 parents 70ebca1 + d5bd310 commit a623ea8
Show file tree
Hide file tree
Showing 11 changed files with 1,255 additions and 68 deletions.
77 changes: 59 additions & 18 deletions Alignment/OfflineValidation/scripts/submitPVResolutionJobs.py
View file
Expand Up @@ -3,25 +3,47 @@
Submits per run Primary Vertex Resoltion Alignment validation using the split vertex method,
usage:
python submitPVResolutionJobs.py -i PVResolutionExample.ini -D /JetHT/Run2018C-TkAlMinBias-12Nov2019_UL2018-v2/ALCARECO
submitPVResolutionJobs.py -i PVResolutionExample.ini -D /JetHT/Run2018C-TkAlMinBias-12Nov2019_UL2018-v2/ALCARECO
'''

from __future__ import print_function

__author__ = 'Marco Musich'
__copyright__ = 'Copyright 2020, CERN CMS'
__credits__ = ['Ernesto Migliore', 'Salvatore Di Guida']
__license__ = 'Unknown'
__maintainer__ = 'Marco Musich'
__email__ = 'marco.musich@cern.ch'
__version__ = 1

import os,sys
import getopt
import commands
import time
import json
import ROOT
import urllib
import string
import subprocess
import pprint
import warnings
from subprocess import Popen, PIPE
import multiprocessing
from optparse import OptionParser
import os, shlex, shutil, getpass
import ConfigParser
import configparser as ConfigParser

CopyRights = '##################################\n'
CopyRights += '# submitPVVResolutioJobs.py #\n'
CopyRights += '# marco.musich@cern.ch #\n'
CopyRights += '# October 2020 #\n'
CopyRights += '##################################\n'

##############################################
def get_status_output(*args, **kwargs):
##############################################
p = subprocess.Popen(*args, **kwargs)
stdout, stderr = p.communicate()
return p.returncode, stdout, stderr

##############################################
def check_proxy():
Expand Down Expand Up @@ -116,33 +138,42 @@ def getLuminosity(homedir,minRun,maxRun,isRunBased,verbose):
if(not isRunBased):
return myCachedLumi

## using normtag
#output = subprocess.check_output([homedir+"/.local/bin/brilcalc", "lumi", "-b", "STABLE BEAMS", "--normtag","/cvmfs/cms-bril.cern.ch/cms-lumi-pog/Normtags/normtag_PHYSICS.json", "-u", "/pb", "--begin", str(minRun),"--end",str(maxRun),"--output-style","csv"])
try:
## using normtag
#output = subprocess.check_output([homedir+"/.local/bin/brilcalc", "lumi", "-b", "STABLE BEAMS", "--normtag","/cvmfs/cms-bril.cern.ch/cms-lumi-pog/Normtags/normtag_PHYSICS.json", "-u", "/pb", "--begin", str(minRun),"--end",str(maxRun),"--output-style","csv"])

## no normtag
output = subprocess.check_output([homedir+"/.local/bin/brilcalc", "lumi", "-b", "STABLE BEAMS","-u", "/pb", "--begin", str(minRun),"--end",str(maxRun),"--output-style","csv"])
## no normtag
output = subprocess.check_output([homedir+"/.local/bin/brilcalc", "lumi", "-b", "STABLE BEAMS","-u", "/pb", "--begin", str(minRun),"--end",str(maxRun),"--output-style","csv"])
except:
warnings.warn('ATTENTION! Impossible to query the BRIL DB!')
return myCachedLumi

if(verbose):
print("INSIDE GET LUMINOSITY")
print(output)

for line in output.split("\n"):
for line in output.decode().split("\n"):
if ("#" not in line):
runToCache = line.split(",")[0].split(":")[0]
lumiToCache = line.split(",")[-1].replace("\r", "")
#print("run",runToCache)
#print("lumi",lumiToCache)
myCachedLumi[runToCache] = lumiToCache

#print(myCachedLumi)
if(verbose):
print(myCachedLumi)
return myCachedLumi

##############################################
def isInJSON(run,jsonfile):
##############################################
with open(jsonfile, 'rb') as myJSON:
jsonDATA = json.load(myJSON)
return (run in jsonDATA)
try:
with open(jsonfile, 'r') as myJSON:
jsonDATA = json.load(myJSON)
return (run in jsonDATA)
except:
warnings.warn('ATTENTION! Impossible to find lumi mask! All runs will be used.')
return True

#######################################################
def as_dict(config):
Expand Down Expand Up @@ -229,6 +260,9 @@ def main():

(opts, args) = parser.parse_args()

global CopyRights
print('\n'+CopyRights)

input_CMSSW_BASE = os.environ.get('CMSSW_BASE')

## prepare the eos output directory
Expand All @@ -246,27 +280,33 @@ def main():
try:
config = ConfigParser.ConfigParser()
config.read(opts.iniPathName)
except ConfigParser.MissingSectionHeaderError, e:
raise WrongIniFormatError(`e`)
except ConfigParser.MissingSectionHeaderError as e:
raise WrongIniFormatError(e)

print("Parsed the following configuration \n\n")
inputDict = as_dict(config)
pprint.pprint(inputDict)

if(not bool(inputDict)):
raise SystemExit("\n\n ERROR! Could not parse any input file, perhaps you are submitting this from the wrong folder? \n\n")

## check first there is a valid grid proxy
forward_proxy(".")

runs = commands.getstatusoutput("dasgoclient -query='run dataset="+opts.DATASET+"'")[1].split("\n")
#runs = commands.getstatusoutput("dasgoclient -query='run dataset="+opts.DATASET+"'")[1].split("\n")
runs = get_status_output("dasgoclient -query='run dataset="+opts.DATASET+"'",shell=True, stdout=PIPE, stderr=PIPE)[1].decode().split("\n")
runs.pop()
runs.sort()
print("\n\n Will run on the following runs: \n",runs)

if(not os.path.exists("cfg")):
os.system("mkdir cfg")
os.system("mkdir bash")
os.system("mkdir BASH")
os.system("mkdir harvest")
os.system("mkdir out")

cwd = os.getcwd()
bashdir = os.path.join(cwd,"bash")
bashdir = os.path.join(cwd,"BASH")

runs.sort()
# get from the DB the int luminosities
Expand Down Expand Up @@ -344,7 +384,7 @@ def main():
key = key.split(":", 1)[1]
print("dealing with",key)

os.system("cp PrimaryVertexResolution_templ_cfg.py ./cfg/PrimaryVertexResolution_"+key+"_"+run+"_cfg.py")
os.system("cp "+input_CMSSW_BASE+"/src/Alignment/OfflineValidation/test/PrimaryVertexResolution_templ_cfg.py ./cfg/PrimaryVertexResolution_"+key+"_"+run+"_cfg.py")
os.system("sed -i 's|XXX_FILES_XXX|"+listOfFiles+"|g' "+cwd+"/cfg/PrimaryVertexResolution_"+key+"_"+run+"_cfg.py")
os.system("sed -i 's|XXX_RUN_XXX|"+run+"|g' "+cwd+"/cfg/PrimaryVertexResolution_"+key+"_"+run+"_cfg.py")
os.system("sed -i 's|YYY_KEY_YYY|"+key+"|g' "+cwd+"/cfg/PrimaryVertexResolution_"+key+"_"+run+"_cfg.py")
Expand All @@ -370,5 +410,6 @@ def main():
print(submissionCommand)
os.system(submissionCommand)

###################################################
if __name__ == "__main__":
main()

0 comments on commit a623ea8

Please sign in to comment.