From f14b686058de5d0ea3e83457ab39622919dc72d6 Mon Sep 17 00:00:00 2001 From: Kirill Skovpen Date: Fri, 24 Sep 2021 21:47:10 +0200 Subject: [PATCH 1/6] add relval nano wfs --- Configuration/PyReleaseValidation/python/relval_steps.py | 9 +++++++++ .../python/upgradeWorkflowComponents.py | 8 ++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/Configuration/PyReleaseValidation/python/relval_steps.py b/Configuration/PyReleaseValidation/python/relval_steps.py index f4e71afd8fc11..bac98db263d0f 100644 --- a/Configuration/PyReleaseValidation/python/relval_steps.py +++ b/Configuration/PyReleaseValidation/python/relval_steps.py @@ -3504,6 +3504,15 @@ def gen2021HiMix(fragment,howMuch): '--geometry' : geom } + upgradeStepDict['NanoDQM'][k] = {'-s':'NANO,DQM:@nanoAODDQM', + '--conditions':gt, + '--datatier':'NANOAODSIM,DQMIO', + '-n':'10', + '--eventcontent':'NANOEDMAODSIM,DQM', + '--filein':'file:step3_inMINIAODSIM.root', + '--geometry' : geom + } + # setup baseline and variations for specialType,specialWF in upgradeWFs.items(): specialWF.setup(upgradeStepDict, k, upgradeProperties[year][k]) diff --git a/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py b/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py index f5ff92debab2c..001e34457bafa 100644 --- a/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py +++ b/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py @@ -174,6 +174,7 @@ def condition(self, fragment, stepList, key, hasHarvest): 'ALCA', 'Nano', 'MiniAOD', + 'NanoDQM', ], PU = [ 'DigiTrigger', @@ -187,6 +188,7 @@ def condition(self, fragment, stepList, key, hasHarvest): 'HARVESTGlobal', 'MiniAOD', 'Nano', + 'NanoDQM', ], suffix = '', offset = 0.0, @@ -675,6 +677,7 @@ def condition(self, fragment, stepList, key, hasHarvest): 'MiniAOD', 'ALCA', 'Nano', + 'NanoDQM', ], PU = [ 'GenSimHLBeamSpot14', @@ -687,6 +690,7 @@ def condition(self, fragment, stepList, key, hasHarvest): 'MiniAOD', 'ALCA', 'Nano', + 'NanoDQM', ], suffix = '_ProdLike', offset = 0.21, @@ -1208,7 +1212,7 @@ def condition(self, fragment, stepList, key, hasHarvest): 'HLTmenu': '@relval2021', 'Era' : 'Run3', 'BeamSpot': 'Run3RoundOptics25ns13TeVLowSigmaZ', - 'ScenToRun' : ['GenSim','Digi','Reco','HARVEST','ALCA'], + 'ScenToRun' : ['GenSim','Digi','Reco','HARVEST','ALCA','NanoDQM'], }, '2021Design' : { 'Geom' : 'DB:Extended', @@ -1241,7 +1245,7 @@ def condition(self, fragment, stepList, key, hasHarvest): upgradeProperties[2017][key+'PU'] = deepcopy(upgradeProperties[2017][key]) upgradeProperties[2017][key+'PU']['ScenToRun'] = ['GenSim','DigiPU'] + \ (['RecoPU','HARVESTPU'] if '202' in key else ['RecoFakeHLTPU','HARVESTFakeHLTPU']) + \ - (['Nano'] if 'Design' not in key else []) + (['NanoDQM'] if 'Design' not in key else []) upgradeProperties[2026] = { '2026D49' : { From a38f59479698833c04405a877aafe67c5c2b344a Mon Sep 17 00:00:00 2001 From: Kirill Skovpen Date: Sat, 25 Sep 2021 23:26:00 +0200 Subject: [PATCH 2/6] update --- .../PyReleaseValidation/python/relval_steps.py | 15 +++------------ .../python/upgradeWorkflowComponents.py | 8 ++------ 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/Configuration/PyReleaseValidation/python/relval_steps.py b/Configuration/PyReleaseValidation/python/relval_steps.py index bac98db263d0f..b0ed2cd58b2d1 100644 --- a/Configuration/PyReleaseValidation/python/relval_steps.py +++ b/Configuration/PyReleaseValidation/python/relval_steps.py @@ -3495,23 +3495,14 @@ def gen2021HiMix(fragment,howMuch): '--scenario' : 'pp' } - upgradeStepDict['Nano'][k] = {'-s':'NANO', + upgradeStepDict['Nano'][k] = {'-s':'NANO,DQM:@nanoAODDQM', '--conditions':gt, - '--datatier':'NANOAODSIM', + '--datatier':'NANOAODSIM,DQMIO', '-n':'10', - '--eventcontent':'NANOEDMAODSIM', + '--eventcontent':'NANOEDMAODSIM,DQM', '--filein':'file:step3_inMINIAODSIM.root', '--geometry' : geom } - - upgradeStepDict['NanoDQM'][k] = {'-s':'NANO,DQM:@nanoAODDQM', - '--conditions':gt, - '--datatier':'NANOAODSIM,DQMIO', - '-n':'10', - '--eventcontent':'NANOEDMAODSIM,DQM', - '--filein':'file:step3_inMINIAODSIM.root', - '--geometry' : geom - } # setup baseline and variations for specialType,specialWF in upgradeWFs.items(): diff --git a/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py b/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py index 001e34457bafa..29baf5833b068 100644 --- a/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py +++ b/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py @@ -174,7 +174,6 @@ def condition(self, fragment, stepList, key, hasHarvest): 'ALCA', 'Nano', 'MiniAOD', - 'NanoDQM', ], PU = [ 'DigiTrigger', @@ -188,7 +187,6 @@ def condition(self, fragment, stepList, key, hasHarvest): 'HARVESTGlobal', 'MiniAOD', 'Nano', - 'NanoDQM', ], suffix = '', offset = 0.0, @@ -677,7 +675,6 @@ def condition(self, fragment, stepList, key, hasHarvest): 'MiniAOD', 'ALCA', 'Nano', - 'NanoDQM', ], PU = [ 'GenSimHLBeamSpot14', @@ -690,7 +687,6 @@ def condition(self, fragment, stepList, key, hasHarvest): 'MiniAOD', 'ALCA', 'Nano', - 'NanoDQM', ], suffix = '_ProdLike', offset = 0.21, @@ -1212,7 +1208,7 @@ def condition(self, fragment, stepList, key, hasHarvest): 'HLTmenu': '@relval2021', 'Era' : 'Run3', 'BeamSpot': 'Run3RoundOptics25ns13TeVLowSigmaZ', - 'ScenToRun' : ['GenSim','Digi','Reco','HARVEST','ALCA','NanoDQM'], + 'ScenToRun' : ['GenSim','Digi','Reco','HARVEST','ALCA','Nano'], }, '2021Design' : { 'Geom' : 'DB:Extended', @@ -1245,7 +1241,7 @@ def condition(self, fragment, stepList, key, hasHarvest): upgradeProperties[2017][key+'PU'] = deepcopy(upgradeProperties[2017][key]) upgradeProperties[2017][key+'PU']['ScenToRun'] = ['GenSim','DigiPU'] + \ (['RecoPU','HARVESTPU'] if '202' in key else ['RecoFakeHLTPU','HARVESTFakeHLTPU']) + \ - (['NanoDQM'] if 'Design' not in key else []) + (['Nano'] if 'Design' not in key else []) upgradeProperties[2026] = { '2026D49' : { From 8a11a9776a8a0a684645d2536e4856e02c99f551 Mon Sep 17 00:00:00 2001 From: Kirill Skovpen Date: Tue, 28 Sep 2021 12:26:01 +0200 Subject: [PATCH 3/6] run reco and nano in one step --- .../PyReleaseValidation/python/relval_steps.py | 16 ++++++++++++++++ .../python/upgradeWorkflowComponents.py | 8 ++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/Configuration/PyReleaseValidation/python/relval_steps.py b/Configuration/PyReleaseValidation/python/relval_steps.py index b0ed2cd58b2d1..4d97597a9fb99 100644 --- a/Configuration/PyReleaseValidation/python/relval_steps.py +++ b/Configuration/PyReleaseValidation/python/relval_steps.py @@ -3460,6 +3460,14 @@ def gen2021HiMix(fragment,howMuch): '--scenario' : 'pp', '--filetype':'DQM', } + + upgradeStepDict['HARVESTRecNan'][k]={'-s':'HARVESTING:@standardValidation+@standardDQM+@ExtraHLT+@miniAODValidation+@miniAODDQM+@nanoAODDQM', + '--conditions':gt, + '--mc':'', + '--geometry' : geom, + '--scenario' : 'pp', + '--filetype':'DQM', + } upgradeStepDict['HARVESTFakeHLT'][k]={'-s':'HARVESTING:@standardValidationNoHLT+@standardDQMFakeHLT+@miniAODValidation+@miniAODDQM', '--conditions':gt, @@ -3503,6 +3511,14 @@ def gen2021HiMix(fragment,howMuch): '--filein':'file:step3_inMINIAODSIM.root', '--geometry' : geom } + + upgradeStepDict['RecNan'][k] = {'-s':'RAW2DIGI,L1Reco,RECO,RECOSIM,EI,PAT,NANO,VALIDATION:@standardValidation+@miniAODValidation,DQM:@standardDQM+@ExtraHLT+@miniAODDQM+@nanoAODDQM', + '--conditions':gt, + '--datatier':'GEN-SIM-RECO,MINIAODSIM,NANOAODSIM,DQMIO', + '-n':'10', + '--eventcontent':'RECOSIM,MINIAODSIM,NANOEDMAODSIM,DQM', + '--geometry' : geom + } # setup baseline and variations for specialType,specialWF in upgradeWFs.items(): diff --git a/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py b/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py index 29baf5833b068..702dca946cdd6 100644 --- a/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py +++ b/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py @@ -168,12 +168,14 @@ def condition(self, fragment, stepList, key, hasHarvest): 'RecoGlobal', 'HARVEST', 'HARVESTFakeHLT', + 'HARVESTRecNan', 'FastSim', 'HARVESTFast', 'HARVESTGlobal', 'ALCA', 'Nano', 'MiniAOD', + 'RecNan', ], PU = [ 'DigiTrigger', @@ -184,9 +186,11 @@ def condition(self, fragment, stepList, key, hasHarvest): 'RecoFakeHLT', 'HARVEST', 'HARVESTFakeHLT', + 'HARVESTRecNan', 'HARVESTGlobal', 'MiniAOD', 'Nano', + 'RecNan', ], suffix = '', offset = 0.0, @@ -1206,9 +1210,9 @@ def condition(self, fragment, stepList, key, hasHarvest): 'Geom' : 'DB:Extended', 'GT' : 'auto:phase1_2021_realistic', 'HLTmenu': '@relval2021', - 'Era' : 'Run3', + 'Era' : 'Run3,run3_nanoAOD_devel', 'BeamSpot': 'Run3RoundOptics25ns13TeVLowSigmaZ', - 'ScenToRun' : ['GenSim','Digi','Reco','HARVEST','ALCA','Nano'], + 'ScenToRun' : ['GenSim','Digi','RecNan','HARVESTRecNan','ALCA'], }, '2021Design' : { 'Geom' : 'DB:Extended', From dd33215442fb10fa673a3c29c73c7000ea55406d Mon Sep 17 00:00:00 2001 From: Kirill Skovpen Date: Sat, 2 Oct 2021 13:04:00 +0200 Subject: [PATCH 4/6] fix wfs --- .../PyReleaseValidation/python/relval_steps.py | 13 ++++++++++--- .../PyReleaseValidation/python/relval_upgrade.py | 6 +++++- .../python/upgradeWorkflowComponents.py | 8 +++++--- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/Configuration/PyReleaseValidation/python/relval_steps.py b/Configuration/PyReleaseValidation/python/relval_steps.py index 4d97597a9fb99..1a88f0104ffdc 100644 --- a/Configuration/PyReleaseValidation/python/relval_steps.py +++ b/Configuration/PyReleaseValidation/python/relval_steps.py @@ -2864,7 +2864,6 @@ def gen2021HiMix(fragment,howMuch): '--filetype':'DQM', '--scenario':'pp'} - steps['HARVESTCOS']={'-s':'HARVESTING:dqmHarvestingFakeHLT', '--conditions':'auto:run1_mc', '--mc':'', @@ -3460,7 +3459,7 @@ def gen2021HiMix(fragment,howMuch): '--scenario' : 'pp', '--filetype':'DQM', } - + upgradeStepDict['HARVESTRecNan'][k]={'-s':'HARVESTING:@standardValidation+@standardDQM+@ExtraHLT+@miniAODValidation+@miniAODDQM+@nanoAODDQM', '--conditions':gt, '--mc':'', @@ -3468,7 +3467,7 @@ def gen2021HiMix(fragment,howMuch): '--scenario' : 'pp', '--filetype':'DQM', } - + upgradeStepDict['HARVESTFakeHLT'][k]={'-s':'HARVESTING:@standardValidationNoHLT+@standardDQMFakeHLT+@miniAODValidation+@miniAODDQM', '--conditions':gt, '--mc':'', @@ -3512,6 +3511,14 @@ def gen2021HiMix(fragment,howMuch): '--geometry' : geom } + upgradeStepDict['Reco'][k] = {'-s':'RAW2DIGI,L1Reco,RECO,RECOSIM,EI,PAT,VALIDATION:@standardValidation+@miniAODValidation,DQM:@standardDQM+@ExtraHLT+@miniAODDQM', + '--conditions':gt, + '--datatier':'GEN-SIM-RECO,MINIAODSIM,DQMIO', + '-n':'10', + '--eventcontent':'RECOSIM,MINIAODSIM,DQM', + '--geometry' : geom + } + upgradeStepDict['RecNan'][k] = {'-s':'RAW2DIGI,L1Reco,RECO,RECOSIM,EI,PAT,NANO,VALIDATION:@standardValidation+@miniAODValidation,DQM:@standardDQM+@ExtraHLT+@miniAODDQM+@nanoAODDQM', '--conditions':gt, '--datatier':'GEN-SIM-RECO,MINIAODSIM,NANOAODSIM,DQMIO', diff --git a/Configuration/PyReleaseValidation/python/relval_upgrade.py b/Configuration/PyReleaseValidation/python/relval_upgrade.py index 574bb2cff83a9..9d2ef0e1c3b4c 100644 --- a/Configuration/PyReleaseValidation/python/relval_upgrade.py +++ b/Configuration/PyReleaseValidation/python/relval_upgrade.py @@ -28,7 +28,7 @@ def makeStepName(key,frag,step,suffix): for specialType in upgradeWFs.keys(): stepList[specialType] = [] hasHarvest = False - for step in upgradeProperties[year][key]['ScenToRun']: + for step in upgradeProperties[year][key]['ScenToRun']: stepMaker = makeStepName if 'Sim' in step: if 'HLBeamSpot' in step: @@ -41,6 +41,10 @@ def makeStepName(key,frag,step,suffix): if 'HARVEST' in step: hasHarvest = True for specialType,specialWF in upgradeWFs.items(): + if specialType == 'baseline': + for ist, st in enumerate(stepList[specialType]): + if st.split('_')[0] == 'Reco': stepList[specialType][ist] = st.replace('Reco', 'RecNan') + elif st.split('_')[0] == 'HARVEST': stepList[specialType][ist] = st.replace('HARVEST', 'HARVESTRecNan') if (specialType != 'baseline') and ( ('PU' in step and step.replace('PU','') in specialWF.PU) or (step in specialWF.steps) ): stepList[specialType].append(stepMaker(key,frag[:-4],step,specialWF.suffix)) # hack to add an extra step diff --git a/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py b/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py index 702dca946cdd6..0c79289bec1cc 100644 --- a/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py +++ b/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py @@ -150,7 +150,9 @@ def setup_(self, step, stepName, stepDict, k, properties): era=properties.get('Era', None) modifier=properties.get('ProcessModifier',None) if cust is not None: stepDict[stepName][k]['--customise']=cust - if era is not None: stepDict[stepName][k]['--era']=era + if era is not None: + stepDict[stepName][k]['--era']=era + if 'RecNan' in stepName: stepDict[stepName][k]['--era'] += ',run3_nanoAOD_devel' if modifier is not None: stepDict[stepName][k]['--procModifier']=modifier def condition(self, fragment, stepList, key, hasHarvest): return True @@ -1210,9 +1212,9 @@ def condition(self, fragment, stepList, key, hasHarvest): 'Geom' : 'DB:Extended', 'GT' : 'auto:phase1_2021_realistic', 'HLTmenu': '@relval2021', - 'Era' : 'Run3,run3_nanoAOD_devel', + 'Era' : 'Run3', 'BeamSpot': 'Run3RoundOptics25ns13TeVLowSigmaZ', - 'ScenToRun' : ['GenSim','Digi','RecNan','HARVESTRecNan','ALCA'], + 'ScenToRun' : ['GenSim','Digi','Reco','HARVEST','ALCA'], }, '2021Design' : { 'Geom' : 'DB:Extended', From 84d062845748caecf4ec8564a87d3be5e52e25b5 Mon Sep 17 00:00:00 2001 From: Kirill Skovpen Date: Sat, 2 Oct 2021 17:40:09 +0200 Subject: [PATCH 5/6] fix typo --- Configuration/PyReleaseValidation/python/relval_steps.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/Configuration/PyReleaseValidation/python/relval_steps.py b/Configuration/PyReleaseValidation/python/relval_steps.py index 1a88f0104ffdc..1bd423c43dcfb 100644 --- a/Configuration/PyReleaseValidation/python/relval_steps.py +++ b/Configuration/PyReleaseValidation/python/relval_steps.py @@ -3510,14 +3510,6 @@ def gen2021HiMix(fragment,howMuch): '--filein':'file:step3_inMINIAODSIM.root', '--geometry' : geom } - - upgradeStepDict['Reco'][k] = {'-s':'RAW2DIGI,L1Reco,RECO,RECOSIM,EI,PAT,VALIDATION:@standardValidation+@miniAODValidation,DQM:@standardDQM+@ExtraHLT+@miniAODDQM', - '--conditions':gt, - '--datatier':'GEN-SIM-RECO,MINIAODSIM,DQMIO', - '-n':'10', - '--eventcontent':'RECOSIM,MINIAODSIM,DQM', - '--geometry' : geom - } upgradeStepDict['RecNan'][k] = {'-s':'RAW2DIGI,L1Reco,RECO,RECOSIM,EI,PAT,NANO,VALIDATION:@standardValidation+@miniAODValidation,DQM:@standardDQM+@ExtraHLT+@miniAODDQM+@nanoAODDQM', '--conditions':gt, From 63cafe02829da0395684ca0d32e6ccafae89971d Mon Sep 17 00:00:00 2001 From: Kirill Skovpen Date: Tue, 5 Oct 2021 13:40:19 +0200 Subject: [PATCH 6/6] add nano to prodlike wfs --- .../PyReleaseValidation/python/relval_steps.py | 2 +- .../PyReleaseValidation/python/relval_upgrade.py | 10 ++++++---- .../python/upgradeWorkflowComponents.py | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Configuration/PyReleaseValidation/python/relval_steps.py b/Configuration/PyReleaseValidation/python/relval_steps.py index 1bd423c43dcfb..5586e1657abfd 100644 --- a/Configuration/PyReleaseValidation/python/relval_steps.py +++ b/Configuration/PyReleaseValidation/python/relval_steps.py @@ -3279,7 +3279,7 @@ def gen2021HiMix(fragment,howMuch): '--datatier':'NANOAODSIM', '--eventcontent':'NANOEDMAODSIM', '--filein':'file:step3_inMINIAODSIM.root'} - + ################################################################################# ####From this line till the end of the file : ####UPGRADE WORKFLOWS IN PREPARATION - Gaelle's sandbox - diff --git a/Configuration/PyReleaseValidation/python/relval_upgrade.py b/Configuration/PyReleaseValidation/python/relval_upgrade.py index 9d2ef0e1c3b4c..341102308fc28 100644 --- a/Configuration/PyReleaseValidation/python/relval_upgrade.py +++ b/Configuration/PyReleaseValidation/python/relval_upgrade.py @@ -41,16 +41,13 @@ def makeStepName(key,frag,step,suffix): if 'HARVEST' in step: hasHarvest = True for specialType,specialWF in upgradeWFs.items(): - if specialType == 'baseline': - for ist, st in enumerate(stepList[specialType]): - if st.split('_')[0] == 'Reco': stepList[specialType][ist] = st.replace('Reco', 'RecNan') - elif st.split('_')[0] == 'HARVEST': stepList[specialType][ist] = st.replace('HARVEST', 'HARVESTRecNan') if (specialType != 'baseline') and ( ('PU' in step and step.replace('PU','') in specialWF.PU) or (step in specialWF.steps) ): stepList[specialType].append(stepMaker(key,frag[:-4],step,specialWF.suffix)) # hack to add an extra step if 'ProdLike' in specialType: if 'Reco' in step: # handles both Reco and RecoGlobal stepList[specialType].append(stepMaker(key,frag[:-4],step.replace('RecoGlobal','MiniAOD').replace('Reco','MiniAOD'),specialWF.suffix)) + stepList[specialType].append(stepMaker(key,frag[:-4],step.replace('RecoGlobal','Nano').replace('Reco','Nano'),specialWF.suffix)) # similar hacks for premixing if 'PMX' in specialType: if 'GenSim' in step: @@ -63,6 +60,11 @@ def makeStepName(key,frag,step,suffix): else: stepList[specialType][-1] = stepMade else: stepList[specialType].append(stepMaker(key,frag[:-4],step,'')) + + if specialType in ['baseline']: + for ist, st in enumerate(stepList[specialType]): + if st.split('_')[0] == 'Reco': stepList[specialType][ist] = st.replace('Reco', 'RecNan') + elif st.split('_')[0] == 'HARVEST': stepList[specialType][ist] = st.replace('HARVEST', 'HARVESTRecNan') for specialType,specialWF in upgradeWFs.items(): # remove other steps for premixS1 diff --git a/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py b/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py index 0c79289bec1cc..2ef67714ca7ff 100644 --- a/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py +++ b/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py @@ -666,7 +666,7 @@ def setup_(self, step, stepName, stepDict, k, properties): # remove step stepDict[stepName][k] = None if 'Nano' in step: - stepDict[stepName][k] = merge([{'--filein':'file:step4.root'}, stepDict[step][k]]) + stepDict[stepName][k] = merge([{'--filein':'file:step4.root','-s':'NANO','--datatier':'NANOAODSIM','--eventcontent':'NANOEDMAODSIM'}, stepDict[step][k]]) def condition(self, fragment, stepList, key, hasHarvest): return fragment=="TTbar_14TeV" and ('2026' in key or '2021' in key) upgradeWFs['ProdLike'] = UpgradeWorkflow_ProdLike(