Fixes for GainCalibration

CalibTracker/SiStripChannelGain/plugins/SiStripGainFromCalibTree.cc

@@ -359,7 +359,21 @@ SiStripGainFromCalibTree::algoEndJob() {
    algoComputeMPVandGain();
 
    //FIXME: for the moment the tree is disabled in PCL, among other reasons the fact that the TFileSevice is not available @ Tier0
-   if(AlgoMode != "PCL") storeOnTree();
+   if(AlgoMode != "PCL"){
+      storeOnTree();
+      //also save the 2D monitor elements to this file as TH2D tfs
+      tfs = edm::Service<TFileService>().operator->();
+      tfs->make<TH2F> (*Charge_Vs_Index->getTH2F());
+      tfs->make<TH2F> (*Charge_Vs_Index_Absolute->getTH2F());
+      tfs->make<TH2F> (*Charge_Vs_PathlengthTIB->getTH2F());
+      tfs->make<TH2F> (*Charge_Vs_PathlengthTOB->getTH2F());
+      tfs->make<TH2F> (*Charge_Vs_PathlengthTIDP->getTH2F());
+      tfs->make<TH2F> (*Charge_Vs_PathlengthTIDM->getTH2F());
+      tfs->make<TH2F> (*Charge_Vs_PathlengthTECP1->getTH2F());
+      tfs->make<TH2F> (*Charge_Vs_PathlengthTECP2->getTH2F());
+      tfs->make<TH2F> (*Charge_Vs_PathlengthTECM1->getTH2F());
+      tfs->make<TH2F> (*Charge_Vs_PathlengthTECM2->getTH2F());
+   }
 }
 
 
@@ -400,7 +414,7 @@ void SiStripGainFromCalibTree::algoAnalyzeTheTree()
 {
    for(unsigned int i=0;i<VInputFiles.size();i++){
       printf("Openning file %3i/%3i --> %s\n",i+1, (int)VInputFiles.size(), (char*)(VInputFiles[i].c_str())); fflush(stdout);
-      TChain* tree = new TChain("commonCalibrationTree/tree");
+      TChain* tree = new TChain("gainCalibrationTree/tree");
       tree->Add(VInputFiles[i].c_str());
 
       TString EventPrefix("");

CalibTracker/SiStripChannelGain/python/computeGain_cff.py

@@ -2,6 +2,8 @@
 
 SiStripCalib = cms.EDAnalyzer("SiStripGainFromCalibTree",
     OutputGains         = cms.string('Gains_ASCII.txt'),
+    Tracks              = cms.untracked.InputTag('CalibrationTracksRefit'),
+    AlgoMode            = cms.untracked.string('CalibTree'),
 
     minTrackMomentum    = cms.untracked.double(2),
     minNrEntries        = cms.untracked.double(25),

CalibTracker/SiStripChannelGain/test/7TeVData/Gains_Compute_cfg.py

@@ -59,4 +59,9 @@
         fileName = cms.string('Gains_Tree.root')  
 )
 
-process.p = cms.Path(process.SiStripCalib)
+process.DQMStore = cms.Service("DQMStore")
+process.load("DQMServices.Components.DQMFileSaver_cfi")
+process.dqmSaver.convention = 'Offline'
+process.dqmSaver.workflow = '/Express/PCLTest/ALCAPROMPT'
+
+process.p = cms.Path(process.SiStripCalib * process.dqmSaver) 

CalibTracker/SiStripChannelGain/test/7TeVData/Validation_Compute_cfg.py

@@ -39,6 +39,7 @@
 
 
 
+
 process.SiStripCalibValidation.FirstSetOfConstants = cms.untracked.bool(False)
 process.SiStripCalibValidation.CalibrationLevel    = cms.untracked.int32(0) # 0==APV, 1==Laser, 2==module
 
@@ -47,4 +48,9 @@
         fileName = cms.string('Validation_Tree.root')  
 )
 
-process.p = cms.Path(process.SiStripCalibValidation)
+process.DQMStore = cms.Service("DQMStore")
+process.load("DQMServices.Components.DQMFileSaver_cfi")
+process.dqmSaver.convention = 'Offline'
+process.dqmSaver.workflow = '/Express/PCLTest/ALCAPROMPT'
+
+process.p = cms.Path(process.SiStripCalibValidation * process.dqmSaver)

CalibTracker/SiStripChannelGain/test/7TeVData/automatic_RunOnCalibTree.py

@@ -9,16 +9,17 @@
 def numberOfEvents(file):
 	rootfile = ROOT.TFile.Open(file,'read')
 	tree = ROOT.TTree()
-	rootfile.GetObject("commonCalibrationTree/tree",tree)
+	rootfile.GetObject("gainCalibrationTree/tree",tree)
         NEntries = tree.GetEntries()
         rootfile.Close()
         print file +' --> '+str(NEntries)
 	return NEntries	
 
 
-globaltag = "GR_P_V32" 
-path = "/store/group/tracker/strip/calibration/calibrationtree/GR12" #"/castor/cern.ch/user/m/mgalanti/calibrationtree/GR11"
-#path = "/castor/cern.ch/user/m/mgalanti/calibrationtree/GR12/"
+globaltag = "GR_E_V42" 
+path = "/store/group/dpg_tracker_strip/comm_tracker/Strip/Calibration/calibrationtree/CRUZET15" #no slash at the end
+#path = "/castor/cern.ch/user/m/mgalanti/calibrationtree/GR12"
+#path = "/castor/cern.ch/user/m/mgalanti/calibrationtree/GR11"
 firstRun = -1
 #firstRun = 192701	#value of the first run with the new calibration --> this is needed to avoid mixing runs with different calibrations
 lastRun  = -1
@@ -30,9 +31,9 @@ def numberOfEvents(file):
 #go to parent directory = test directory
 os.chdir("..");
 #identify last run of the previous calibration
-if(firstRun<=0):
-	out = commands.getstatusoutput("ls /afs/cern.ch/cms/tracker/sistrvalidation/WWW/CalibrationValidation/ParticleGain/ | grep Run_ | tail -n 1");
-	firstRun = int(out[1].split('_')[3])+1
+#if(firstRun<=0):
+#	out = commands.getstatusoutput("ls /afs/cern.ch/cms/tracker/sistrvalidation/WWW/CalibrationValidation/ParticleGain/ | grep Run_ | tail -n 1");
+#	firstRun = int(out[1].split('_')[3])+1
 
 #Get List of CalibFiles:
 print("Get the list of calibTree from castor (cmsLs" + path + ")")
@@ -45,7 +46,7 @@ def numberOfEvents(file):
         if(len(info)<1):continue;
 	subParts = info.split();        
 	size = int(subParts[1])/1048576;
-	if(size < 50): continue	#skip file<50MB
+#	if(size < 50): continue	#skip file<50MB  #commented for CRUZET15
 	run = int(subParts[4].replace(path+'/',"").replace("calibTree_","").replace(".root","")) 
 	if(run<firstRun):continue
         if(lastRun>0 and run>lastRun):continue
@@ -54,8 +55,8 @@ def numberOfEvents(file):
 	if(calibTreeList==""):firstRun=run;
 	calibTreeList += '  "root://eoscms//eos/cms'+subParts[4]+'", #' + str(size).rjust(6)+'MB  NEvents='+str(NEvents/1000).rjust(8)+'K\n'
 	NTotalEvents += NEvents;
-	if(NTotalEvents>2500000):
-		break;
+#	if(NTotalEvents>2500000): #commented for CRUZET15
+#		break;
 
 if(lastRun<=0):lastRun = run
 

CalibTracker/SiStripChannelGain/test/7TeVData/automatic_RunOnCalibTree_manual.py

@@ -9,21 +9,22 @@
 def numberOfEvents(file):
 	rootfile = ROOT.TFile.Open(file,'read')
 	tree = ROOT.TTree()
-	rootfile.GetObject("commonCalibrationTree/tree",tree)
+	rootfile.GetObject("gainCalibrationTree/tree",tree)
         NEntries = tree.GetEntries()
         rootfile.Close()
         print file +' --> '+str(NEntries)
 	return NEntries	
 
 
-globaltag = "GR_P_V32" 
-path = "/store/group/tracker/strip/calibration/calibrationtree/GR12" #"/castor/cern.ch/user/m/mgalanti/calibrationtree/GR11"
-#path = "/castor/cern.ch/user/m/mgalanti/calibrationtree/GR12/"
-firstRun = 191919
+globaltag = "GR_E_V42" 
+path = "/store/group/dpg_tracker_strip/comm_tracker/Strip/Calibration/calibrationtree/CRUZET15" #no slash at the end
+#path = "/castor/cern.ch/user/m/mgalanti/calibrationtree/GR12"
+#path = "/castor/cern.ch/user/m/mgalanti/calibrationtree/GR11"
+firstRun = 237201
 #firstRun = 192701	#value of the first run with the new calibration --> this is needed to avoid mixing runs with different calibrations
-lastRun  = 197000
+lastRun  = 237201
 MC=""
-publish = True
+publish = False
 mail = "loic.quertenmont@gmail.com"
 automatic = True;
 
@@ -45,7 +46,7 @@ def numberOfEvents(file):
         if(len(info)<1):continue;
 	subParts = info.split();        
 	size = int(subParts[1])/1048576;
-	if(size < 50): continue	#skip file<50MB
+#	if(size < 50): continue	#skip file<50MB
 	run = int(subParts[4].replace(path+'/',"").replace("calibTree_","").replace(".root","")) 
 	if(run<firstRun):continue
         if(lastRun>0 and run>lastRun):continue
@@ -54,8 +55,8 @@ def numberOfEvents(file):
 	if(calibTreeList==""):firstRun=run;
 	calibTreeList += '  "root://eoscms//eos/cms'+subParts[4]+'", #' + str(size).rjust(6)+'MB  NEvents='+str(NEvents/1000).rjust(8)+'K\n'
 	NTotalEvents += NEvents;
-	if(NTotalEvents>2500000):
-		break;
+#	if(NTotalEvents>2500000):
+#		break;
 
 if(lastRun==0):lastRun = run
 

CalibTracker/SiStripCommon/test/MakeCalibrationTrees/FailledRun.txt

@@ -0,0 +1 @@
+

CalibTracker/SiStripCommon/test/MakeCalibrationTrees/LastRun.txt

@@ -0,0 +1 @@
+0

CalibTracker/SiStripCommon/test/MakeCalibrationTrees/SubmitJobs.py

@@ -0,0 +1,167 @@
+#!/usr/bin/env python
+import urllib
+import string
+import os
+import sys
+import commands
+import time
+
+#DATASET = '/MinimumBias/Commissioning2015-SiStripCalMinBias-PromptReco-v1/ALCARECO'
+DATASET='/Cosmics/Commissioning2015-PromptReco-v1/RECO'
+# Set the correct environment
+CMSSWDIR='/afs/cern.ch/cms/tracker/sistrvalidation/Calibration/CalibrationTree/CMSSW_7_3_2/src/' # CMSSW version to be used for CRUZET2015
+#CMSSWDIR='/afs/cern.ch/cms/tracker/sistrvalidation/Calibration/CalibrationTree/CMSSW_5_3_8_patch3/src' # CMSSW version to be used for 8TeV
+RUNDIR  ='/afs/cern.ch/cms/tracker/sistrvalidation/Calibration/CalibrationTree/CMSSW_7_3_2/src/CalibTracker/SiStripCommon/test/MakeCalibrationTrees' # directory where the job will actually run
+#CMSSWDIR = '/afs/cern.ch/user/q/querten/workspace/public/CalibTreeUpdate/CMSSW_5_3_8_patch3/src'
+#RUNDIR = '/afs/cern.ch/user/q/querten/workspace/public/CalibTreeUpdate/CMSSW_5_3_8_patch3/src/newScript'
+CASTORDIR = '/store/group/dpg_tracker_strip/comm_tracker/Strip/Calibration/calibrationtree/CRUZET15'
+#CASTORDIR = '/castor/cern.ch/cms/store/group/tracker/strip/calibration/calibrationtree/GRIN' # used for GRIN
+#CASTORDIR = '/castor/cern.ch/cms/store/group/tracker/strip/calibration/calibrationtree/GR12'
+os.environ['PATH'] = os.getenv('PATH')+':/afs/cern.ch/cms/sw/common/'
+os.environ['CMS_PATH']='/afs/cern.ch/cms'
+os.environ['FRONTIER_PROXY'] = 'http://cmst0frontier.cern.ch:3128'
+os.environ['SCRAM_ARCH']='slc6_amd64_gcc481'
+
+initEnv=''
+initEnv+='cd ' + CMSSWDIR + ';'
+initEnv+='source /afs/cern.ch/cms/cmsset_default.sh' + ';'
+initEnv+='eval `scramv1 runtime -sh`' + ';'
+initEnv+='cd ' + RUNDIR + ';'   
+print initEnv;
+
+#If the script is run without argument, check what was the last run, and check on DBS what are the new runs (with at least 1K events) and run on them
+#If on the contrary a runnumber is provided as argument, request the list of input files from DBS in order to run on them and produce calibration trees
+
+
+if(len(sys.argv)<2): 
+   #### FETCH DBS TO FIND ALL RUNS TO PROCCESS
+
+   #PRODUCE A LIST OF FILE WITH ALL NEW RUNS TO BE ANALYZED
+   LASTRUN=int(commands.getstatusoutput("tail -n 1 LastRun.txt")[1])
+   print('Last analyzed Run: %i' % LASTRUN)
+
+   runs = []
+   print ("das_client.py  --limit=9999 --query='run dataset="+DATASET+"  | grep run.run_number>"+str(LASTRUN)+"'")
+   results = commands.getstatusoutput(initEnv+"das_client.py  --limit=9999 --query='run dataset="+DATASET+"'")[1].splitlines()
+
+#   print ('dbs search --query="find run,sum(block.numevents) where dataset=/StreamExpressCosmics/Commissioning2013-SiStripCalZeroBias-Express-v1/ALCARECO and run>'+str(LASTRUN)+'"')
+#   results = commands.getstatusoutput(initEnv+'dbs search --query="find run,sum(block.numevents) where dataset=/StreamExpressCosmics/Commissioning2013-SiStripCalZeroBias-Express-v1/ALCARECO and run>'+str(LASTRUN)+'"')[1].splitlines()
+#   print ('dbs search --query="find run,sum(block.numevents) where dataset=/MinimumBias/*Run*-SiStripCalMinBias-v*/ALCARECO and run>'+str(LASTRUN)+'"')
+#   results = commands.getstatusoutput(initEnv+'dbs search --query="find run,sum(block.numevents) where dataset=/MinimumBias/*Run*-SiStripCalMinBias-v*/ALCARECO and run>'+str(LASTRUN)+'"')[1].splitlines()
+   results.sort()
+   for line in results:
+      if(line.startswith('Showing')):continue
+      if(len(line)<=0):continue
+      linesplit = line.split('   ')
+#      if(len(linesplit)<2):continue
+      run     = int(line.split('   ')[0])
+      if(run<=LASTRUN): continue
+#      NEvents = int(line.split('   ')[1])
+#      if(NEvents<1000):continue
+      runs.append(run)
+   ####
+   print runs
+
+   #APPENDS TO THE LIST OF RUNS FOR WHICH THE PROCESSING FAILLED IN THE PAST
+   FAILLEDRUN=commands.getstatusoutput("cat FailledRun.txt")[1]
+   os.system('echo ' + '"   "' + ' > FailledRun.txt') #remove the file since these jobs will be resubmitted now
+   for line in FAILLEDRUN.splitlines():
+      try:
+         run = int(line)
+         runs.append(run)
+         print "Job running on run " + str(run) + " failed in the past... Resubmitting"
+      except:
+         continue   
+   ####
+
+   #SUBMIT JOB FOR EACH RUN IN THE LIST (see the second part of the script)
+   runs.sort()
+   runs = list(set(runs)) #remove duplicates
+   for run in runs:
+      print 'Submitting Run ' + str(run)
+      os.system('bsub -q 2nd -J calibTree_' + str(run) +  ' -R "type == SLC6_64 && pool > 30000" ' + ' "python '+RUNDIR+'/SubmitJobs.py '+str(run)+'"' )
+      if(run>LASTRUN):os.system('echo ' + str(run) + ' > LastRun.txt')
+   ####
+
+
+elif(sys.argv[1].isdigit()):
+   #### RUN ON ONE PARTICULAR RUN (GIVEN IN ARGUMENT)
+   PWDDIR  =os.getcwd() #Current Dir
+   os.chdir(RUNDIR);
+
+   run = int(sys.argv[1])
+   globaltag = 'GR_E_V42' # used in 2015 CRUZET
+#   globaltag = 'GR_E_V33A' # used for GRIN
+#   globaltag = 'GR_P_V40' # used for 2012
+   outfile = 'calibTree_'+str(run)+'.root'
+
+   #reinitialize the afs token, to make sure that the job isn't kill after a few hours of running
+   os.system('/usr/sue/bin/kinit -R')
+
+   #GET THE LIST OF FILE FROM THE DATABASE
+   files = ''
+#   print("das_client.py  --limit=9999 --query='file dataset="+DATASET+" run="+str(run)+"'")
+   results = commands.getstatusoutput(initEnv+"das_client.py  --limit=9999 --query='file dataset="+DATASET+" run="+str(run)+"'")
+#   print('dbs search --query="find file where dataset=/StreamExpressCosmics/Commissioning2013-SiStripCalZeroBias-Express-v1/ALCARECO and run='+str(run)+'"')
+#   results = commands.getstatusoutput(initEnv+'dbs search --query="find file where dataset=/StreamExpressCosmics/Commissioning2013-SiStripCalZeroBias-Express-v1/ALCARECO and run='+str(run)+'"')
+#   print('dbs search --query="find file where dataset=/MinimumBias/Run*-SiStripCalMinBias-v*/ALCARECO and run='+str(run)+'"')
+#   results = commands.getstatusoutput(initEnv+'dbs search --query="find file where dataset=/MinimumBias/Run*-SiStripCalMinBias-v*/ALCARECO and run='+str(run)+'"')
+   if(int(results[0])!=0 or results[1].find('Error:')>=0):
+      print results
+      os.system('echo ' + str(run) + ' >> FailledRun.txt')
+      sys.exit(1)
+   filesList = results[1].splitlines();
+   for f in filesList:
+      if(not f.startswith('/')):continue
+      files+="'"+f+"',"
+   if(files==''):
+      print('no files to process for run '+ str(run))
+      sys.exit(0)
+   ###
+
+
+   #BUILD CMSSW CONFIG, START CMSRUN, COPY THE OUTPUT AND CLEAN THE PROJECT
+#   os.system('sed -e "s@OUTFILE@/tmp/'+outfile+'@g" -e "s@GLOBALTAG@'+globaltag+'@g" -e "s@FILES@'+files+'@g" '+RUNDIR+'/produceCalibrationTree_template_cfg.py > ConfigFile_'+str(run)+'_cfg.py')
+   os.system('sed -e "s@OUTFILE@'+PWDDIR+'/'+outfile+'@g" -e "s@GLOBALTAG@'+globaltag+'@g" -e "s@FILES@'+files+'@g" '+RUNDIR+'/produceCalibrationTree_template_cfg.py > ConfigFile_'+str(run)+'_cfg.py')
+   exit_code = os.system(initEnv+'cmsRun ConfigFile_'+str(run)+'_cfg.py')
+   if(int(exit_code)!=0):
+      print("Job Failed with ExitCode "+str(exit_code))
+      os.system('echo ' + str(run) + ' >> FailledRun.txt')
+   else:
+      os.system('cmsRm ' + CASTORDIR+'/'+outfile) #make sure that the file is overwritten
+      FileSizeInKBytes =commands.getstatusoutput('ls  -lth --block-size=1024 '+PWDDIR+'/'+outfile)[1].split()[4]
+      if(int(FileSizeInKBytes)>50): 
+         print("Preparing for stageout of " + PWDDIR+'/'+outfile + ' on ' + CASTORDIR+'/'+outfile + '.  The file size is %d KB' % int(FileSizeInKBytes))
+         os.system('cmsStageOut '+PWDDIR+'/'+outfile + ' ' + CASTORDIR+'/'+outfile)
+         os.system('cmsLs ' + CASTORDIR+'/'+outfile)
+      else:
+         print('File size is %d KB, this is under the threshold --> the file will not be transfered on EOS' % int(FileSizeInKBytes))
+   os.system('ls -lth '+PWDDIR+'/'+outfile)
+   os.system('rm -f '+PWDDIR+'/'+outfile)
+   os.system('rm -f ConfigFile_'+str(run)+'_cfg.py')
+   os.system('cd /afs/cern.ch/user/c/cmstkmtc')
+   os.system('rm -rf LSFJOB_${LSB_JOBID}')
+   ###
+
+elif(sys.argv[1]=="--corrupted"):
+   #### FIND ALL CORRUPTED FILES ON CASTOR AND MARK THEM AS FAILLED RUN
+
+   calibTreeList = ""
+   print("Get the list of calibTree from" + CASTORDIR + ")")
+   calibTreeInfo = commands.getstatusoutput("cmsLs " + CASTORDIR)[1].split('\n');
+   NTotalEvents = 0;
+   run = 0
+   for info in calibTreeInfo:
+      subParts = info.split();
+      run = int(subParts[8].replace("calibTree_","").replace(".root",""))
+      file = "root://eoscms//eos/cms"+CASTORDIR+"/"+subParts[5] 
+      print("Checking " + file)
+      results = commands.getstatusoutput(initEnv+'root -l -b -q ' + file)
+      if(len(results[1].splitlines())>2):
+         print(results[1]);
+         print("add " + str(run) + " to the list of failled runs")
+         os.system('echo ' + str(run) + ' >> FailledRun.txt')
+
+else:
+   #### UNKNOWN CASE
+   print "unknown argument: make sure you know what you are doing?"

CalibTracker/SiStripCommon/test/MakeCalibrationTrees/cronjob.txt

@@ -0,0 +1,9 @@
+0 13 * * * lxplus5.cern.ch /afs/cern.ch/cms/tracker/sistrvalidation/Calibration/CalibrationTree/config/SubmitBatchJobs.sh > /afs/cern.ch/cms/tracker/sistrvalidation/Calibration/CalibrationTree/cronlog/cronjob_`date +\%Y-\%m-\%d_\%H-\%M-\%S`.log 2>&1
+55 0 * * 0 lxplus5.cern.ch /afs/cern.ch/cms/tracker/sistrvalidation/Calibration/CalibrationTree/config/cleanLogFiles.sh
+
+
+#SiStrip CalibTree Production --> Done everyday
+0 13 * * * lxplus6.cern.ch "python /afs/cern.ch/cms/tracker/sistrvalidation/Calibration/CalibrationTree/confignew/SubmitJobs.py" > /afs/cern.ch/cms/tracker/sistrvalidation/Calibration/CalibrationTree/cronlog/cronjob_`date+\%Y-\%m-\%d_\%H-\%M-\%S`.log 2>&1
+#SiStrip CalibTree (check that files are not corrupted, otherwise resprocess these jobs) --> Done once a week on sunday
+0 22 * * 0 lxplus6.cern.ch "python /afs/cern.ch/cms/tracker/sistrvalidation/Calibration/CalibrationTree/confignew/SubmitJobs.py --corrupted" > /afs/cern.ch/cms/tracker/sistrvalidation/Calibration/CalibrationTree/cronlog/cronjob_`date+\%Y-\%m-\%d_\%H-\%M-\%S`.log 2>&1
+