Simple configuration testing

FWCore/Framework/test/stubs/ToyAnalyzers.cc

@@ -71,6 +71,36 @@ namespace edmtest {
     edm::InputTag moduleLabel_;
   };
 
+  //--------------------------------------------------------------------
+  //
+  class MultipleIntsAnalyzer : public edm::global::EDAnalyzer<> {
+  public:
+    MultipleIntsAnalyzer(edm::ParameterSet const& iPSet)
+    {
+      auto const& tags = iPSet.getUntrackedParameter<std::vector<edm::InputTag>>("getFromModules");
+      for(auto const& tag: tags) {
+        m_tokens.emplace_back(consumes<IntProduct>(tag));
+      }
+    }
+
+    void analyze(edm::StreamID, edm::Event const& iEvent, edm::EventSetup const&) const override {
+      edm::Handle<IntProduct> h;
+      for(auto const& token: m_tokens) {
+        iEvent.getByToken(token,h);
+      }
+    }
+
+    static void fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
+      edm::ParameterSetDescription desc;
+      desc.addUntracked<std::vector<edm::InputTag>>("getFromModules");
+      descriptions.addDefault(desc);
+
+    }
+  private:
+    std::vector<edm::EDGetTokenT<IntProduct>> m_tokens;
+
+  };
+
   //--------------------------------------------------------------------
   //
   class IntConsumingAnalyzer : public edm::global::EDAnalyzer<> {
@@ -270,8 +300,10 @@ using edmtest::ConsumingStreamAnalyzer;
 using edmtest::ConsumingOneSharedResourceAnalyzer;
 using edmtest::SCSimpleAnalyzer;
 using edmtest::DSVAnalyzer;
+using edmtest::MultipleIntsAnalyzer;
 DEFINE_FWK_MODULE(NonAnalyzer);
 DEFINE_FWK_MODULE(IntTestAnalyzer);
+DEFINE_FWK_MODULE(MultipleIntsAnalyzer);
 DEFINE_FWK_MODULE(IntConsumingAnalyzer);
 DEFINE_FWK_MODULE(edmtest::IntFromRunConsumingAnalyzer);
 DEFINE_FWK_MODULE(ConsumingStreamAnalyzer);

FWCore/Framework/test/stubs/ToyIntProducers.cc

@@ -391,19 +391,19 @@ namespace edmtest {
   class AddIntsProducer : public edm::global::EDProducer<> {
   public:
     explicit AddIntsProducer(edm::ParameterSet const& p) :
-      labels_(p.getParameter<std::vector<std::string> >("labels")),
       onlyGetOnEvent_(p.getUntrackedParameter<unsigned int>("onlyGetOnEvent", 0u)) {
 
       produces<IntProduct>();
       produces<IntProduct>("other");
 
-      for( auto const& label: labels_) {
-        consumes<IntProduct>(edm::InputTag{label});
+      auto const& labels = p.getParameter<std::vector<std::string> >("labels");
+      for( auto const& label: labels) {
+        tokens_.emplace_back(consumes<IntProduct>(edm::InputTag{label}));
       }
     }
     virtual void produce(edm::StreamID, edm::Event& e, edm::EventSetup const& c) const override;
   private:
-    std::vector<std::string> labels_;
+    std::vector<edm::EDGetTokenT<IntProduct>> tokens_;
     unsigned int onlyGetOnEvent_;
   };
 
@@ -413,9 +413,9 @@ namespace edmtest {
     int value = 0;
 
     if (onlyGetOnEvent_ == 0u || e.eventAuxiliary().event() == onlyGetOnEvent_) {
-      for(auto const& label: labels_) {
+      for(auto const& token: tokens_) {
         edm::Handle<IntProduct> anInt;
-        e.getByLabel(label, anInt);
+        e.getByToken(token, anInt);
         value += anInt->value;
       }
     }

FWCore/Services/bin/edmTracerLogToSimpleConfig.py

@@ -0,0 +1,164 @@
+#==============================
+#
+# First argument is a log file from cmsRun
+# containing output from Tracer service with
+# the configuration containing
+#     dumpPathsAndConsumes = cms.untracked.bool(True)
+#
+# A new configuration will be created with the same
+# topology as the original configuration but with
+# the modules replaced with 'trivial' versions.
+# This allows studying the cost of the framework infrastructure
+# on realistic module topologies.
+#==============================
+
+import sys
+
+f = open(sys.argv[1])
+
+
+def fixName(name):
+    return name.replace("_","IoI")
+
+class PathParser(object):
+    def __init__(self):
+        self._pathToModules = dict()
+        self._isEndPath = set()
+        self._presentPath = []
+        self._presentPathName = None
+        self.__preamble = 'modules on '
+    def parse(self,line):
+        if line[:len(self.__preamble)] == self.__preamble:
+            if self._presentPathName:
+                self._pathToModules[self._presentPathName] = self._presentPath
+            self._presentPathName = line.split(" ")[3][:-2]
+            if -1 != line.find('end path'):
+                self._isEndPath.add(self._presentPathName)
+            self._presentPath = []
+        else:
+            self._presentPath.append( fixName(line.strip()) )
+    def finish(self):
+        if self._presentPathName:
+            self._pathToModules[self._presentPathName] = self._presentPath
+
+class ConsumesParser(object):
+    def __init__(self):
+        self._consumesForModule = dict()
+        self._isAnalyzer = set()
+        self._presentConsumes = []
+        self._presentModuleName = None
+        self.__preramble = '    '
+    def parse(self,line):
+        if line[:len(self.__preramble)] != self.__preramble:
+            if self._presentModuleName:
+                self._consumesForModule[self._presentModuleName] = self._presentConsumes
+            start = line.find("'")+1
+            length = line[start:].find("'")
+            self._presentModuleName = fixName(line[start:length+start])
+            self._presentConsumes = []
+            if -1 != l.find("Analyzer"):
+                self._isAnalyzer.add(self._presentModuleName)
+        else:
+            self._presentConsumes.append( fixName(line[line.find("'")+1:-2]) )
+    def finish(self):
+        if self._presentModuleName:
+            self._consumesForModule[self._presentModuleName] = self._presentConsumes
+
+pathParser = PathParser()
+consumesParser = ConsumesParser()
+
+parser = pathParser
+
+foundPaths = False
+pathStartsWith = "modules on "
+
+startOfConsumes = "All modules and modules in the current process whose products they consume:"
+skipLineAfterConsumes = False
+doneWithPaths = False
+
+endOfConsumes = "All modules (listed by class and label) and all their consumed products."
+for l in f.readlines():
+    if not foundPaths:
+        if l[:len(pathStartsWith)] == pathStartsWith:
+            foundPaths = True
+        else:
+            #skip lines till find paths
+            continue
+    if not doneWithPaths:
+        if l[:len(startOfConsumes)] == startOfConsumes:
+            skipLineAfterConsumes = True
+            doneWithPaths = True
+            pathParser.finish()
+            parser = consumesParser
+            continue
+    if skipLineAfterConsumes:
+        skipLineAfterConsumes = False
+        continue
+    if l[:len(endOfConsumes)] == endOfConsumes:
+        break
+    parser.parse(l)
+
+parser.finish()
+
+print "import FWCore.ParameterSet.Config as cms"
+print "process = cms.Process('RECO')"
+
+print """process.maxEvents = cms.untracked.PSet(input = cms.untracked.int32(2000))
+process.options = cms.untracked.PSet(
+#    numberOfThreads = cms.untracked.uint32(8),
+    numberOfThreads = cms.untracked.uint32(1),
+    numberOfStreams = cms.untracked.uint32(0),
+#    wantSummary = cms.untracked.bool(True)
+)
+
+process.add_(cms.Service("Timing", summaryOnly = cms.untracked.bool(True)))
+
+# The following two lines reduce the clutter of repeated printouts
+# of the same exception message.
+process.load("FWCore.MessageLogger.MessageLogger_cfi")
+process.MessageLogger.destinations = ['cerr']
+process.MessageLogger.statistics = []
+process.MessageLogger.fwkJobReports = []
+process.MessageLogger.cerr.FwkReport.reportEvery = 50000
+process.MessageLogger.cerr.threshold = 'WARNING'
+"""
+
+print "process.source = cms.Source('EmptySource')"
+
+allModules = set()
+modulesWithConsumes = set()
+for m,c in consumesParser._consumesForModule.iteritems():
+    if m in consumesParser._isAnalyzer:
+        print "process.%s = cms.EDAnalyzer('MultipleIntsAnalyzer', getFromModules = cms.untracked.VInputTag(*[%s]))"%(m,",".join(["cms.InputTag('%s')"%i for i in c]))
+    elif not c:
+        print "process.%s = cms.EDProducer('IntProducer', ivalue = cms.int32(1))"%m
+    else:
+        print "process.%s = cms.EDProducer('AddIntsProducer', labels = cms.vstring(*[%s]))"%(m,",".join(["'%s'"%i for i in c]))
+    allModules.add(m)
+    for o  in c:
+        allModules.add(o)
+    modulesWithConsumes.add(m)
+
+for m in pathParser._pathToModules.itervalues():
+    for i in m:
+        allModules.add(i)
+
+for m in allModules.difference(modulesWithConsumes):
+    print "process.%s = cms.EDProducer('IntProducer', ivalue = cms.int32(1))"%(m)
+
+
+print 't = cms.Task(*[%s])'%(",".join(["process.%s"%i for i in allModules if i not in consumesParser._isAnalyzer]))
+for p,m in pathParser._pathToModules.iteritems():
+    if p in pathParser._isEndPath:
+        print "process.%s = cms.EndPath(%s)"%(p,"+".join(["process.%s"%i for i in m]))
+    else:
+        if m:
+            print "process.%s = cms.Path(%s,t)"%(p,"+".join(["process.%s"%i for i in m]))
+        else:
+            print "process.%s = cms.Path()"%(p)
+
+
+#print "paths = ",pathParser._pathToModules
+#print "modulesToConsumes =",consumesParser._consumesForModule
+
+