From 0641130b52d78e40a040817fa93b2516f39eaaaf Mon Sep 17 00:00:00 2001 From: Matti Kortelainen Date: Fri, 25 Nov 2016 10:54:48 +0100 Subject: [PATCH 1/4] Improve maintainability of 'early delete' in ConfigBuilder --- .../Applications/python/ConfigBuilder.py | 11 ++-- .../python/earlyDeleteSettings_cff.py | 53 ++++++++++++++++++- .../python/customiseEarlyDeleteForSeeding.py | 49 ++--------------- 3 files changed, 61 insertions(+), 52 deletions(-) diff --git a/Configuration/Applications/python/ConfigBuilder.py b/Configuration/Applications/python/ConfigBuilder.py index 4afa68680bc44..8be2c2057fa15 100644 --- a/Configuration/Applications/python/ConfigBuilder.py +++ b/Configuration/Applications/python/ConfigBuilder.py @@ -2221,11 +2221,12 @@ def prepare(self, doChecking = False): # everything else # # FIXME: remove when no longer needed - if "RECO" in self.stepMap or "RAW2RECO" in self.stepMap: - self.pythonCfgCode += "\n# Add early deletion of temporary data products to reduce peak memory need\n" - self.pythonCfgCode += "from Configuration.StandardSequences.earlyDeleteSettings_cff import customiseEarlyDeleteForRECO\n" - self.pythonCfgCode += "process = customiseEarlyDeleteForRECO(process)\n" - self.pythonCfgCode += "# End adding early deletion\n" + self.pythonCfgCode += "\n# Add early deletion of temporary data products to reduce peak memory need\n" + self.pythonCfgCode += "from Configuration.StandardSequences.earlyDeleteSettings_cff import customiseEarlyDeleteForRECO\n" + self.pythonCfgCode += "process = customiseEarlyDeleteForRECO(process)\n" + self.pythonCfgCode += "# End adding early deletion\n" + from Configuration.StandardSequences.earlyDeleteSettings_cff import customiseEarlyDeleteForRECO + self.process = customiseEarlyDeleteForRECO(self.process) # make the .io file diff --git a/Configuration/StandardSequences/python/earlyDeleteSettings_cff.py b/Configuration/StandardSequences/python/earlyDeleteSettings_cff.py index 1c6ff931d9f40..89c5161879699 100644 --- a/Configuration/StandardSequences/python/earlyDeleteSettings_cff.py +++ b/Configuration/StandardSequences/python/earlyDeleteSettings_cff.py @@ -1,7 +1,58 @@ # Abstract all early deletion settings here +import collections + +import FWCore.ParameterSet.Config as cms + from RecoTracker.Configuration.customiseEarlyDeleteForSeeding import customiseEarlyDeleteForSeeding +def _hasInputTagModuleLabel(pset, moduleLabel): + for name in pset.parameterNames_(): + value = getattr(pset,name) + if isinstance(value, cms.PSet): + if _hasInputTagModuleLabel(value, moduleLabel): + return True + elif isinstance(value, cms.VPSet): + for ps in value: + if _hasInputTagModuleLabel(ps, moduleLabel): + return True + elif isinstance(value, cms.VInputTag): + for t in value: + t2 = t + if not isinstance(t, cms.InputTag): + t2 = cms.InputTag(t2) + if t2.getModuleLabel() == moduleLabel: + return True + elif isinstance(value, cms.InputTag): + if value.getModuleLabel() == moduleLabel: + return True + return False + def customiseEarlyDeleteForRECO(process): - process = customiseEarlyDeleteForSeeding(process) + # Mapping label -> [branches] + # for the producers whose products are to be deleted early + products = collections.defaultdict(list) + + products = customiseEarlyDeleteForSeeding(process, products) + + # Set process.options.canDeleteEarly + if not hasattr(process.options, "canDeleteEarly"): + process.options.canDeleteEarly = cms.untracked.vstring() + + branchSet = set() + for branches in products.itervalues(): + for branch in branches: + branchSet.add(branch) + process.options.canDeleteEarly.extend(list(branchSet)) + + # Find the consumers + for moduleType in [process.producers_(), process.filters_(), process.analyzers_()]: + for name, module in moduleType.iteritems(): + for producer, branches in products.iteritems(): + if _hasInputTagModuleLabel(module, producer): + #print "Module %s mightGet %s" % (name, str(branches)) + if hasattr(module, "mightGet"): + module.mightGet.extend(branches) + else: + module.mightGet = cms.untracked.vstring(branches) return process diff --git a/RecoTracker/Configuration/python/customiseEarlyDeleteForSeeding.py b/RecoTracker/Configuration/python/customiseEarlyDeleteForSeeding.py index 40cf9a502df5a..9ebaa35fdb3d6 100644 --- a/RecoTracker/Configuration/python/customiseEarlyDeleteForSeeding.py +++ b/RecoTracker/Configuration/python/customiseEarlyDeleteForSeeding.py @@ -2,32 +2,8 @@ import collections -def _hasInputTagModuleLabel(pset, moduleLabel): - for name in pset.parameterNames_(): - value = getattr(pset,name) - if isinstance(value, cms.PSet): - if _hasInputTagModuleLabel(value, moduleLabel): - return True - elif isinstance(value, cms.VPSet): - for ps in value: - if _hasInputTagModuleLabel(ps, moduleLabel): - return True - elif isinstance(value, cms.VInputTag): - for t in value: - t2 = t - if not isinstance(t, cms.InputTag): - t2 = cms.InputTag(t2) - if t2.getModuleLabel() == moduleLabel: - return True - elif isinstance(value, cms.InputTag): - if value.getModuleLabel() == moduleLabel: - return True - return False - - -def customiseEarlyDeleteForSeeding(process): +def customiseEarlyDeleteForSeeding(process, products): # Find the producers - products = collections.defaultdict(list) depends = collections.defaultdict(list) def _branchName(productType, moduleLabel, instanceLabel=""): @@ -60,15 +36,7 @@ def _branchName(productType, moduleLabel, instanceLabel=""): ]) if len(products) == 0: - return process - - # Set process.options - if not hasattr(process, "options"): - process.options = cms.untracked.PSet() - if not hasattr(process.options, "canDeleteEarly"): - process.options.canDeleteEarly = cms.untracked.vstring() - for branches in products.itervalues(): - process.options.canDeleteEarly.extend(branches) + return products # Resolve data dependencies # @@ -86,15 +54,4 @@ def _resolve(keys, name): name = keys.pop() _resolve(keys, name) - # Find the consumers - for moduleType in [process.producers_(), process.filters_(), process.analyzers_()]: - for name, module in moduleType.iteritems(): - for producer, branches in products.iteritems(): - if _hasInputTagModuleLabel(module, producer): - #print "Module %s mightGet %s" % (name, str(branches)) - if hasattr(module, "mightGet"): - module.mightGet.extend(branches) - else: - module.mightGet = cms.untracked.vstring(branches) - - return process + return products From 9302986e4a53abde9c97829a280363c109c32028 Mon Sep 17 00:00:00 2001 From: Matti Kortelainen Date: Mon, 28 Nov 2016 13:43:47 +0100 Subject: [PATCH 2/4] Add support for refToPSet_ and add a unit test --- .../python/earlyDeleteSettings_cff.py | 48 +++++++++++++++++-- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/Configuration/StandardSequences/python/earlyDeleteSettings_cff.py b/Configuration/StandardSequences/python/earlyDeleteSettings_cff.py index 89c5161879699..b5da4756a0ab7 100644 --- a/Configuration/StandardSequences/python/earlyDeleteSettings_cff.py +++ b/Configuration/StandardSequences/python/earlyDeleteSettings_cff.py @@ -6,15 +6,15 @@ from RecoTracker.Configuration.customiseEarlyDeleteForSeeding import customiseEarlyDeleteForSeeding -def _hasInputTagModuleLabel(pset, moduleLabel): +def _hasInputTagModuleLabel(process, pset, moduleLabel): for name in pset.parameterNames_(): value = getattr(pset,name) if isinstance(value, cms.PSet): - if _hasInputTagModuleLabel(value, moduleLabel): + if _hasInputTagModuleLabel(process, value, moduleLabel): return True elif isinstance(value, cms.VPSet): for ps in value: - if _hasInputTagModuleLabel(ps, moduleLabel): + if _hasInputTagModuleLabel(process, ps, moduleLabel): return True elif isinstance(value, cms.VInputTag): for t in value: @@ -26,6 +26,8 @@ def _hasInputTagModuleLabel(pset, moduleLabel): elif isinstance(value, cms.InputTag): if value.getModuleLabel() == moduleLabel: return True + if isinstance(value, cms.string) and name == "refToPSet_": + return _hasInputTagModuleLabel(process, getattr(process, value.value()), moduleLabel) return False def customiseEarlyDeleteForRECO(process): @@ -49,10 +51,48 @@ def customiseEarlyDeleteForRECO(process): for moduleType in [process.producers_(), process.filters_(), process.analyzers_()]: for name, module in moduleType.iteritems(): for producer, branches in products.iteritems(): - if _hasInputTagModuleLabel(module, producer): + if _hasInputTagModuleLabel(process, module, producer): #print "Module %s mightGet %s" % (name, str(branches)) if hasattr(module, "mightGet"): module.mightGet.extend(branches) else: module.mightGet = cms.untracked.vstring(branches) return process + + +if __name__=="__main__": + import unittest + + class TestHasInputTagModuleLabel(unittest.TestCase): + def setUp(self): + """Nothing to do """ + None + def testHasInputTagModuleLabel(self): + p = cms.Process("A") + p.pset = cms.PSet(a=cms.InputTag("a")) + p.prod = cms.EDProducer("Producer", + foo = cms.InputTag("foo"), + foo2 = cms.InputTag("foo2", "instance"), + foo3 = cms.InputTag("foo3", "instance", "PROCESS"), + nested = cms.PSet( + bar = cms.InputTag("bar"), + ), + flintstones = cms.VPSet( + cms.PSet(fred=cms.InputTag("fred")), + cms.PSet(wilma=cms.InputTag("wilma")) + ), + ref = cms.PSet( + refToPSet_ = cms.string("pset") + ) + ) + + self.assert_(_hasInputTagModuleLabel(p, p.prod, "foo")) + self.assert_(_hasInputTagModuleLabel(p, p.prod, "foo2")) + self.assert_(_hasInputTagModuleLabel(p, p.prod, "foo3")) + self.assert_(_hasInputTagModuleLabel(p, p.prod, "bar")) + self.assert_(_hasInputTagModuleLabel(p, p.prod, "fred")) + self.assert_(_hasInputTagModuleLabel(p, p.prod, "wilma")) + self.assert_(_hasInputTagModuleLabel(p, p.prod, "a")) + self.assert_(not _hasInputTagModuleLabel(p, p.prod, "joe")) + + unittest.main() From 006cb19e22e1454aa9aa076e627120e2ac8aa868 Mon Sep 17 00:00:00 2001 From: Matti Kortelainen Date: Mon, 28 Nov 2016 14:24:30 +0100 Subject: [PATCH 3/4] Extend unit test to cover untracked.InputTag --- .../python/earlyDeleteSettings_cff.py | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/Configuration/StandardSequences/python/earlyDeleteSettings_cff.py b/Configuration/StandardSequences/python/earlyDeleteSettings_cff.py index b5da4756a0ab7..9b0fbf357ec20 100644 --- a/Configuration/StandardSequences/python/earlyDeleteSettings_cff.py +++ b/Configuration/StandardSequences/python/earlyDeleteSettings_cff.py @@ -69,21 +69,33 @@ def setUp(self): None def testHasInputTagModuleLabel(self): p = cms.Process("A") - p.pset = cms.PSet(a=cms.InputTag("a")) + p.pset = cms.PSet(a=cms.InputTag("a"),a2=cms.untracked.InputTag("a2")) p.prod = cms.EDProducer("Producer", foo = cms.InputTag("foo"), foo2 = cms.InputTag("foo2", "instance"), foo3 = cms.InputTag("foo3", "instance", "PROCESS"), + foo4 = cms.untracked.InputTag("foo4"), nested = cms.PSet( bar = cms.InputTag("bar"), + bar2 = cms.untracked.InputTag("bar2"), + ), + nested2 = cms.untracked.PSet( + bar3 = cms.untracked.InputTag("bar3"), ), flintstones = cms.VPSet( cms.PSet(fred=cms.InputTag("fred")), cms.PSet(wilma=cms.InputTag("wilma")) ), + flintstones2 = cms.VPSet( + cms.PSet(fred2=cms.untracked.InputTag("fred2")), + cms.PSet(wilma2=cms.InputTag("wilma2")) + ), ref = cms.PSet( refToPSet_ = cms.string("pset") - ) + ), + ref2 = cms.untracked.PSet( + refToPSet_ = cms.string("pset") + ), ) self.assert_(_hasInputTagModuleLabel(p, p.prod, "foo")) @@ -93,6 +105,12 @@ def testHasInputTagModuleLabel(self): self.assert_(_hasInputTagModuleLabel(p, p.prod, "fred")) self.assert_(_hasInputTagModuleLabel(p, p.prod, "wilma")) self.assert_(_hasInputTagModuleLabel(p, p.prod, "a")) + self.assert_(_hasInputTagModuleLabel(p, p.prod, "foo4")) + self.assert_(_hasInputTagModuleLabel(p, p.prod, "bar2")) + self.assert_(_hasInputTagModuleLabel(p, p.prod, "bar3")) + self.assert_(_hasInputTagModuleLabel(p, p.prod, "fred2")) + self.assert_(_hasInputTagModuleLabel(p, p.prod, "wilma2")) + self.assert_(_hasInputTagModuleLabel(p, p.prod, "a2")) self.assert_(not _hasInputTagModuleLabel(p, p.prod, "joe")) unittest.main() From 880861406f0fa32c1c8d5cc66b720de33d950a9d Mon Sep 17 00:00:00 2001 From: Matti Kortelainen Date: Tue, 29 Nov 2016 15:56:11 +0100 Subject: [PATCH 4/4] Remove 'RECO' from the customize function name As the customize is now applied for all steps. --- Configuration/Applications/python/ConfigBuilder.py | 8 ++++---- .../StandardSequences/python/earlyDeleteSettings_cff.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Configuration/Applications/python/ConfigBuilder.py b/Configuration/Applications/python/ConfigBuilder.py index 8be2c2057fa15..572b77d57dde2 100644 --- a/Configuration/Applications/python/ConfigBuilder.py +++ b/Configuration/Applications/python/ConfigBuilder.py @@ -2222,11 +2222,11 @@ def prepare(self, doChecking = False): # # FIXME: remove when no longer needed self.pythonCfgCode += "\n# Add early deletion of temporary data products to reduce peak memory need\n" - self.pythonCfgCode += "from Configuration.StandardSequences.earlyDeleteSettings_cff import customiseEarlyDeleteForRECO\n" - self.pythonCfgCode += "process = customiseEarlyDeleteForRECO(process)\n" + self.pythonCfgCode += "from Configuration.StandardSequences.earlyDeleteSettings_cff import customiseEarlyDelete\n" + self.pythonCfgCode += "process = customiseEarlyDelete(process)\n" self.pythonCfgCode += "# End adding early deletion\n" - from Configuration.StandardSequences.earlyDeleteSettings_cff import customiseEarlyDeleteForRECO - self.process = customiseEarlyDeleteForRECO(self.process) + from Configuration.StandardSequences.earlyDeleteSettings_cff import customiseEarlyDelete + self.process = customiseEarlyDelete(self.process) # make the .io file diff --git a/Configuration/StandardSequences/python/earlyDeleteSettings_cff.py b/Configuration/StandardSequences/python/earlyDeleteSettings_cff.py index 9b0fbf357ec20..4fb52a891d99e 100644 --- a/Configuration/StandardSequences/python/earlyDeleteSettings_cff.py +++ b/Configuration/StandardSequences/python/earlyDeleteSettings_cff.py @@ -30,7 +30,7 @@ def _hasInputTagModuleLabel(process, pset, moduleLabel): return _hasInputTagModuleLabel(process, getattr(process, value.value()), moduleLabel) return False -def customiseEarlyDeleteForRECO(process): +def customiseEarlyDelete(process): # Mapping label -> [branches] # for the producers whose products are to be deleted early products = collections.defaultdict(list)