From f1766945fc18b60dd859c7e833db764d68b74b57 Mon Sep 17 00:00:00 2001 From: Marino Missiroli Date: Wed, 11 Jan 2023 16:36:08 +0100 Subject: [PATCH 1/2] avoid use of auto:run*_data_relval GTs in non-reHLT wfs --- .../python/relval_standard.py | 36 ++++++++--------- .../python/relval_steps.py | 39 ++++++++++--------- 2 files changed, 39 insertions(+), 36 deletions(-) diff --git a/Configuration/PyReleaseValidation/python/relval_standard.py b/Configuration/PyReleaseValidation/python/relval_standard.py index 212fda82d1ec6..d1d0714046c2c 100644 --- a/Configuration/PyReleaseValidation/python/relval_standard.py +++ b/Configuration/PyReleaseValidation/python/relval_standard.py @@ -443,10 +443,10 @@ workflows[136.9] = ['', ['RunDoubleMuon2016C', 'RAWRECOTE16', 'RAWRECOLHECLEANTE16', 'EMBEDHADTE16', 'EMBEDMINIAOD16']] ### run 2021 collisions ### -workflows[139.001] = ['RunMinimumBias2021',['RunMinimumBias2021','HLTDR3_2022','RECODR3_reHLT_MinBiasOffline','HARVESTD2021MB']] -workflows[139.002] = ['',['RunZeroBias2021','HLTDR3_2022','RECODR3_reHLT_ZBOffline','HARVESTD2021ZB']] -workflows[139.003] = ['',['RunHLTPhy2021','HLTDR3_2022','RECODR3_reHLT_HLTPhysics_Offline','HARVESTD2021HLTPhy']] -workflows[139.004] = ['',['RunNoBPTX2021','HLTDR3_2022','RECODR3_reHLT_AlCaTkCosmics_Offline','HARVESTDR3']] +workflows[139.001] = ['RunMinimumBias2021',['RunMinimumBias2021','HLTDR3_2022','RECODR3_reHLT_MinBiasOffline','HARVESTD2021MB_reHLT']] +workflows[139.002] = ['',['RunZeroBias2021','HLTDR3_2022','RECODR3_reHLT_ZBOffline','HARVESTD2021ZB_reHLT']] +workflows[139.003] = ['',['RunHLTPhy2021','HLTDR3_2022','RECODR3_reHLT_HLTPhysics_Offline','HARVESTD2021HLTPhy_reHLT']] +workflows[139.004] = ['',['RunNoBPTX2021','HLTDR3_2022','RECODR3_reHLT_AlCaTkCosmics_Offline','HARVESTDR3_reHLT']] workflows[139.005] = ['',['AlCaPhiSym2021','RECOALCAECALPHISYMDR3','ALCAECALPHISYM']] ### run3 (2022) ### @@ -495,20 +495,20 @@ workflows[140.069] = ['',['RunMuonEG2022D','HLTDR3_2022','RECONANORUN3_reHLT','HARVESTRUN3']] ### run3 (2022) skims ### -workflows[140.104] = ['',['RunZeroBias2022B','HLTDR3_2022','RECONANORUN3_reHLT','SKIMZEROBIASRUN3']] -workflows[140.105] = ['',['RunBTagMu2022B','HLTDR3_2022','RECONANORUN3_reHLT','SKIMBTAGMURUN3']] -workflows[140.106] = ['',['RunJetHT2022B','HLTDR3_2022','RECONANORUN3_reHLT','SKIMJETMETRUN3']] -workflows[140.107] = ['',['RunDisplacedJet2022B','HLTDR3_2022','RECONANORUN3_reHLT','SKIMDISPLACEDJETRUN3']] -workflows[140.109] = ['',['RunEGamma2022B','HLTDR3_2022','RECONANORUN3_reHLT','SKIMEGAMMARUN3']] -workflows[140.110] = ['',['RunTau2022B','HLTDR3_2022','RECONANORUN3_reHLT','SKIMTAURUN3']] -workflows[140.111] = ['',['RunDoubleMuon2022B','HLTDR3_2022','RECONANORUN3_reHLT','SKIMMUONRUN3']] -workflows[140.112] = ['',['RunMuonEG2022B','HLTDR3_2022','RECONANORUN3_reHLT','SKIMMUONEGRUN3']] -workflows[140.113] = ['',['RunNoBPTX2022B','HLTDR3_2022','RECONANORUN3_reHLT','SKIMNOBPTXRUN3']] -workflows[140.114] = ['',['RunHcalNZS2022B','HLTDR3_2022','RECONANORUN3_reHLT','SKIMHCALNZSRUN3']] -workflows[140.115] = ['',['RunHLTPhysics2022B','HLTDR3_2022','RECONANORUN3_reHLT','SKIMHLTPHYSICSRUN3']] -workflows[140.116] = ['',['RunCommissioning2022B','HLTDR3_2022','RECONANORUN3_reHLT','SKIMCOMMISSIONINGRUN3']] -workflows[140.117] = ['',['RunCosmics2022B','HLTDR3_2022','RECOCOSMRUN3_reHLT','SKIMCOSMICSRUN3']] -#workflows[140.118] = ['',['RunParkingBPH2022B','HLTDR3_2022','RECONANORUN3_reHLT','SKIMPARKINGBPHRUN3']] +workflows[140.104] = ['',['RunZeroBias2022B','HLTDR3_2022','RECONANORUN3_reHLT','SKIMZEROBIASRUN3_reHLT']] +workflows[140.105] = ['',['RunBTagMu2022B','HLTDR3_2022','RECONANORUN3_reHLT','SKIMBTAGMURUN3_reHLT']] +workflows[140.106] = ['',['RunJetHT2022B','HLTDR3_2022','RECONANORUN3_reHLT','SKIMJETMETRUN3_reHLT']] +workflows[140.107] = ['',['RunDisplacedJet2022B','HLTDR3_2022','RECONANORUN3_reHLT','SKIMDISPLACEDJETRUN3_reHLT']] +workflows[140.109] = ['',['RunEGamma2022B','HLTDR3_2022','RECONANORUN3_reHLT','SKIMEGAMMARUN3_reHLT']] +workflows[140.110] = ['',['RunTau2022B','HLTDR3_2022','RECONANORUN3_reHLT','SKIMTAURUN3_reHLT']] +workflows[140.111] = ['',['RunDoubleMuon2022B','HLTDR3_2022','RECONANORUN3_reHLT','SKIMMUONRUN3_reHLT']] +workflows[140.112] = ['',['RunMuonEG2022B','HLTDR3_2022','RECONANORUN3_reHLT','SKIMMUONEGRUN3_reHLT']] +workflows[140.113] = ['',['RunNoBPTX2022B','HLTDR3_2022','RECONANORUN3_reHLT','SKIMNOBPTXRUN3_reHLT']] +workflows[140.114] = ['',['RunHcalNZS2022B','HLTDR3_2022','RECONANORUN3_reHLT','SKIMHCALNZSRUN3_reHLT']] +workflows[140.115] = ['',['RunHLTPhysics2022B','HLTDR3_2022','RECONANORUN3_reHLT','SKIMHLTPHYSICSRUN3_reHLT']] +workflows[140.116] = ['',['RunCommissioning2022B','HLTDR3_2022','RECONANORUN3_reHLT','SKIMCOMMISSIONINGRUN3_reHLT']] +workflows[140.117] = ['',['RunCosmics2022B','HLTDR3_2022','RECOCOSMRUN3_reHLT','SKIMCOSMICSRUN3_reHLT']] +#workflows[140.118] = ['',['RunParkingBPH2022B','HLTDR3_2022','RECONANORUN3_reHLT','SKIMPARKINGBPHRUN3_reHLT']] ### fastsim ### workflows[5.1] = ['TTbarFS', ['TTbarFS','HARVESTFS']] diff --git a/Configuration/PyReleaseValidation/python/relval_steps.py b/Configuration/PyReleaseValidation/python/relval_steps.py index 15e228724a5f2..5f38e9b2bdbc2 100644 --- a/Configuration/PyReleaseValidation/python/relval_steps.py +++ b/Configuration/PyReleaseValidation/python/relval_steps.py @@ -2477,11 +2477,11 @@ def lhegensim2018ml(fragment,howMuch): # Run 3 steps['RECODR3']=merge([{'--scenario':'pp', '-s':'RAW2DIGI,L1Reco,RECO,DQM', - '--conditions':'auto:run3_data_relval', + '--conditions':'auto:run3_data', '--era':'Run3', '--customise':'Configuration/DataProcessing/RecoTLR.customisePostEra_Run3'},dataReco]) -steps['RECODR3_reHLT']=merge([{'--hltProcess':'reHLT'},steps['RECODR3']]) +steps['RECODR3_reHLT']=merge([{'--conditions':'auto:run3_data_relval', '--hltProcess':'reHLT'},steps['RECODR3']]) steps['RECODR3Splash']=merge([{'-n': 2, '-s': 'RAW2DIGI,L1Reco,RECO,PAT,ALCA:SiStripCalZeroBias+SiStripCalMinBias+TkAlMinBias+EcalESAlign,DQM:@standardDQMFakeHLT+@miniAODDQM' @@ -2510,12 +2510,14 @@ def lhegensim2018ml(fragment,howMuch): # Run-3 skims for s in autoSkim.keys(): - steps['SKIM'+s.upper()+'RUN3']={'-s':'SKIM:%s'%(autoSkim[s]), - '--data':'', - '--era':'Run3', - '--scenario':'pp', - '--conditions':'auto:run3_data_relval'} - if 'Cosmics' in s: steps['SKIM'+s.upper()+'RUN3']['--scenario'] = 'cosmics' + steps['SKIM'+s.upper()+'RUN3_reHLT'] = {'-s':'SKIM:%s'%(autoSkim[s]), + '--data':'', + '--era':'Run3', + '--scenario':'pp', + '--conditions':'auto:run3_data_relval', + '--hltProcess':'reHLT'} + if 'Cosmics' in s: + steps['SKIM'+s.upper()+'RUN3_reHLT']['--scenario'] = 'cosmics' # step1 gensim for HI mixing step1Up2018HiMixDefaults = merge ([{'--beamspot':'MatchHI', '--pileup':'HiMixGEN', '--scenario':'HeavyIons'},hiDefaults2018_ppReco,PUHI,step1Up2018HiProdDefaults]) @@ -3014,7 +3016,7 @@ def gen2021HiMix(fragment,howMuch): steps['ALCAEXPHI']=merge([{'-s':'ALCA:PromptCalibProd+PromptCalibProdSiStrip+PromptCalibProdSiStripGains+PromptCalibProdSiStripGainsAAG', '--scenario':'HeavyIons'},steps['ALCAEXP']]) steps['ALCAEXPTE']={'-s':'ALCA:PromptCalibProdEcalPedestals', - '--conditions':'auto:run2_data_relval', + '--conditions':'auto:run2_data', '--datatier':'ALCARECO', '--eventcontent':'ALCARECO', '--triggerResultsProcess': 'RECO'} @@ -3188,8 +3190,8 @@ def gen2021HiMix(fragment,howMuch): '--filetype':'DQM', '--scenario':'pp'} -steps['HARVESTDreHLT'] = merge([ {'--conditions':'auto:run1_data_%s'%menu}, steps['HARVESTD'] ]) -steps['HARVESTDR1reHLT'] = merge([ {'--conditions':'auto:run1_data_%s'%menu}, steps['HARVESTDR1'] ]) +steps['HARVESTDreHLT'] = merge([ {'--conditions':'auto:run1_data_%s'%menu,'--hltProcess':'reHLT'}, steps['HARVESTD'] ]) +steps['HARVESTDR1reHLT'] = merge([ {'--conditions':'auto:run1_data_%s'%menu,'--hltProcess':'reHLT'}, steps['HARVESTDR1'] ]) steps['HARVESTDR2'] = merge([ {'--conditions':'auto:run2_data_relval'}, steps['HARVESTD'] ]) steps['HARVESTDR2_skimDoubleEG'] = merge([ {'-s':'HARVESTING:@commonFakeHLT+@ecal+@egamma'}, steps['HARVESTDR2'] ]) steps['HARVESTDR2_skimJetHT'] = merge([ {'-s':'HARVESTING:@commonFakeHLT+@jetmet+@hcalOnly'}, steps['HARVESTDR2'] ]) @@ -3302,10 +3304,11 @@ def gen2021HiMix(fragment,howMuch): steps['HARVESTDCRUN2']=merge([{'--conditions':'auto:run2_data','--era':'Run2_2016'},steps['HARVESTDC']]) -steps['HARVESTDR3'] = merge([{'--conditions':'auto:run3_data_relval','--era':'Run3'}, steps['HARVESTD']]) -steps['HARVESTD2021MB'] = merge([{'-s':'HARVESTING:@commonSiStripZeroBias+@ExtraHLT+@miniAODDQM'}, steps['HARVESTDR3'] ]) -steps['HARVESTD2021ZB'] = merge([{'-s':'HARVESTING:@rerecoZeroBias+@ExtraHLT+@miniAODDQM'}, steps['HARVESTDR3'] ]) -steps['HARVESTD2021HLTPhy'] = merge([{'-s':'HARVESTING:@commonReduced+@miniAODDQM'}, steps['HARVESTDR3'] ]) +steps['HARVESTDR3'] = merge([{'--conditions':'auto:run3_data','--era':'Run3'}, steps['HARVESTD']]) +steps['HARVESTDR3_reHLT'] = merge([{'--conditions':'auto:run3_data_relval','--hltProcess':'reHLT'}, steps['HARVESTDR3']]) +steps['HARVESTD2021MB_reHLT'] = merge([{'-s':'HARVESTING:@commonSiStripZeroBias+@ExtraHLT+@miniAODDQM'}, steps['HARVESTDR3_reHLT'] ]) +steps['HARVESTD2021ZB_reHLT'] = merge([{'-s':'HARVESTING:@rerecoZeroBias+@ExtraHLT+@miniAODDQM'}, steps['HARVESTDR3_reHLT'] ]) +steps['HARVESTD2021HLTPhy_reHLT'] = merge([{'-s':'HARVESTING:@commonReduced+@miniAODDQM'}, steps['HARVESTDR3_reHLT'] ]) steps['HARVESTDPROMPTR3']=merge([{'--conditions':'auto:run3_data_prompt','-s':'HARVESTING:dqmHarvestingFakeHLT'},steps['HARVESTDR3']]) steps['HARVESTDEXPR3']=merge([{'--conditions':'auto:run3_data_express','--filein':'file:step2_inDQM.root'},steps['HARVESTDR3']]) @@ -3582,7 +3585,7 @@ def gen2021HiMix(fragment,howMuch): '--filein':'file:step2.root', '--secondfilein':'filelist:step1_dasquery.log'} -steps['SKIMDreHLT'] = merge([ {'--conditions':'auto:run1_data_%s'%menu,'--filein':'file:step3.root'}, steps['SKIMD'] ]) +steps['SKIMDreHLT'] = merge([ {'--conditions':'auto:run1_data_%s'%menu, '--hltProcess':'reHLT', '--filein':'file:step3.root'}, steps['SKIMD'] ]) steps['SKIMCOSD']={'-s':'SKIM:all', '--conditions':'auto:run1_data', @@ -3624,7 +3627,7 @@ def gen2021HiMix(fragment,howMuch): steps['REMINIAOD_data2016'] = merge([{'-s' : 'PAT,DQM:@miniAODDQM', '--process' : 'PAT', '--era' : 'Run2_2016,run2_miniAOD_80XLegacy', - '--conditions' : 'auto:run2_data_relval', + '--conditions' : 'auto:run2_data', '--data' : '', '--scenario' : 'pp', '--eventcontent' : 'MINIAOD,DQM', @@ -3634,7 +3637,7 @@ def gen2021HiMix(fragment,howMuch): steps['REMININANO_data2016'] = merge([{'-s' : 'PAT,NANO,DQM:@miniAODDQM+@nanoAODDQM', '--process' : 'PATnano', '--era' : 'Run2_2016', - '--conditions' : 'auto:run2_data_relval', + '--conditions' : 'auto:run2_data', '--data' : '', '--scenario' : 'pp', '--eventcontent' : 'MINIAOD,NANOEDMAOD,DQM', From 9ec7697c9f80efd12168a2ef827a67592e508d1c Mon Sep 17 00:00:00 2001 From: Marino Missiroli Date: Fri, 13 Jan 2023 11:50:40 +0100 Subject: [PATCH 2/2] add support for "hltProcess" option in SKIM step --- .../Applications/python/ConfigBuilder.py | 59 +++++++++++-------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/Configuration/Applications/python/ConfigBuilder.py b/Configuration/Applications/python/ConfigBuilder.py index 9b3adf64f7704..3c52be2071cb8 100644 --- a/Configuration/Applications/python/ConfigBuilder.py +++ b/Configuration/Applications/python/ConfigBuilder.py @@ -868,7 +868,7 @@ def addCustomise(self,unsch=0): if opt.count('.')>1: raise Exception("more than . in the specification:"+opt) fileName=opt.split('.')[0] - if opt.count('.')==0: rest='customise' + if opt.count('.')==0: rest='customise' else: rest=opt.split('.')[1] if rest=='py': rest='customise' #catch the case of --customise file.py @@ -1750,15 +1750,21 @@ def prepare_SKIM(self, stepSpec = "all"): ## support @Mu+DiJet+@Electron configuration via autoSkim.py from Configuration.Skimming.autoSkim import autoSkim self.expandMapping(skimlist,autoSkim) - - #print "dictionnary for skims:",skimConfig.__dict__ + #print("dictionary for skims:", skimConfig.__dict__) for skim in skimConfig.__dict__: - skimstream = getattr(skimConfig,skim) - if isinstance(skimstream,cms.Path): - #black list the alca path so that they do not appear in the cfg + skimstream = getattr(skimConfig, skim) + + # blacklist AlCa paths so that they do not appear in the cfg + if isinstance(skimstream, cms.Path): self.blacklist_paths.append(skimstream) - if (not isinstance(skimstream,cms.FilteredStream)): + # if enabled, apply "hltProcess" renaming to Sequences + elif isinstance(skimstream, cms.Sequence): + if self._options.hltProcess or ('HLT' in self.stepMap): + self.renameHLTprocessInSequence(skim) + + if not isinstance(skimstream, cms.FilteredStream): continue + shortname = skim.replace('SKIMStream','') if (sequence=="all"): self.addExtraStream(skim,skimstream) @@ -1779,12 +1785,11 @@ def prepare_SKIM(self, stepSpec = "all"): for i in range(skimlist.count(shortname)): skimlist.remove(shortname) - - if (skimlist.__len__()!=0 and sequence!="all"): print('WARNING, possible typo with SKIM:'+'+'.join(skimlist)) raise Exception('WARNING, possible typo with SKIM:'+'+'.join(skimlist)) + def prepare_USER(self, stepSpec = None): ''' Enrich the schedule with a user defined sequence ''' _,_userSeq,_ = self.loadDefaultOrSpecifiedCFF(stepSpec,self.USERDefaultCFF) @@ -1877,7 +1882,7 @@ def __init__(self, paramSearch, paramReplace, verbose=False, whitelist=()): self._verbose = verbose self._whitelist = whitelist - def doIt(self,pset,base): + def doIt(self, pset, base): if isinstance(pset, cms._Parameterizable): for name in pset.parameters_().keys(): # skip whitelisted parameters @@ -1885,17 +1890,17 @@ def doIt(self,pset,base): continue # if I use pset.parameters_().items() I get copies of the parameter values # so I can't modify the nested pset - value = getattr(pset,name) - type = value.pythonTypeName() - if type in ('cms.PSet', 'cms.untracked.PSet'): + value = getattr(pset, name) + valueType = type(value) + if valueType in [cms.PSet, cms.untracked.PSet, cms.EDProducer]: self.doIt(value,base+"."+name) - elif type in ('cms.VPSet', 'cms.untracked.VPSet'): + elif valueType in [cms.VPSet, cms.untracked.VPSet]: for (i,ps) in enumerate(value): self.doIt(ps, "%s.%s[%d]"%(base,name,i) ) - elif type in ('cms.string', 'cms.untracked.string'): + elif valueType in [cms.string, cms.untracked.string]: if value.value() == self._paramSearch: if self._verbose: print("set string process name %s.%s %s ==> %s"% (base, name, value, self._paramReplace)) setattr(pset, name,self._paramReplace) - elif type in ('cms.VInputTag', 'cms.untracked.VInputTag'): + elif valueType in [cms.VInputTag, cms.untracked.VInputTag]: for (i,n) in enumerate(value): if not isinstance(n, cms.InputTag): n=cms.InputTag(n) @@ -1904,11 +1909,11 @@ def doIt(self,pset,base): if self._verbose:print("set process name %s.%s[%d] %s ==> %s " % (base, name, i, n, self._paramReplace)) setattr(n,"processName",self._paramReplace) value[i]=n - elif type in ('cms.vstring', 'cms.untracked.vstring'): + elif valueType in [cms.vstring, cms.untracked.vstring]: for (i,n) in enumerate(value): if n==self._paramSearch: getattr(pset,name)[i]=self._paramReplace - elif type in ('cms.InputTag', 'cms.untracked.InputTag'): + elif valueType in [cms.InputTag, cms.untracked.InputTag]: if value.processName == self._paramSearch: if self._verbose: print("set process name %s.%s %s ==> %s " % (base, name, value, self._paramReplace)) setattr(getattr(pset, name),"processName",self._paramReplace) @@ -1938,13 +1943,11 @@ def renameInputTagsInSequence(self,sequence,oldT="rawDataCollector",newT="rawDat #change the process name used to address HLT results in any sequence def renameHLTprocessInSequence(self,sequence,proc=None,HLTprocess='HLT'): - if self._options.hltProcess: - proc=self._options.hltProcess - else: - proc=self.process.name_() - if proc==HLTprocess: return - # look up all module in dqm sequence - print("replacing %s process name - sequence %s will use '%s'" % (HLTprocess,sequence, proc)) + proc = self._options.hltProcess if self._options.hltProcess else self.process.name_() + if proc == HLTprocess: + return + # look up all module in sequence + print("replacing %s process name - sequence %s will use '%s'" % (HLTprocess, sequence, proc)) getattr(self.process,sequence).visit(ConfigBuilder.MassSearchReplaceProcessNameVisitor(HLTprocess,proc,whitelist = ("subSystemFolder",))) if 'from Configuration.Applications.ConfigBuilder import ConfigBuilder' not in self.additionalCommands: self.additionalCommands.append('from Configuration.Applications.ConfigBuilder import ConfigBuilder') @@ -2004,7 +2007,7 @@ def prepare_DQM(self, stepSpec = 'DQMOffline'): pathName='dqmofflineOnPAT_step' for (i,_sequence) in enumerate(postSequenceList): - #Fix needed to avoid duplication of sequences not defined in autoDQM or without a PostDQM + #Fix needed to avoid duplication of sequences not defined in autoDQM or without a PostDQM if (sequenceList[i]==postSequenceList[i]): continue if (i!=0): @@ -2047,6 +2050,10 @@ def prepare_HARVESTING(self, stepSpec = None): setattr(self.process,name+"_step",cms.Path(harvestingstream)) self.schedule.append(getattr(self.process,name+"_step")) + # # NOTE: the "hltProcess" option currently does nothing in the HARVEST step + # if self._options.hltProcess or ('HLT' in self.stepMap): + # pass + self.scheduleSequence('DQMSaver','dqmsave_step') return