Merge pull request #5860 from dmitrijus/forwardport5638

Forward port of #5113 #5721 #5609 #5638 to 72x

Configuration/Applications/python/ConfigBuilder.py

@@ -75,6 +75,7 @@ class Options:
 defaultOptions.runsAndWeightsForMC = None
 defaultOptions.runsScenarioForMC = None
 defaultOptions.runUnscheduled = False
+defaultOptions.timeoutOutput = False
 
 # some helper routines
 def dumpPython(process,name):
@@ -396,6 +397,11 @@ def filesFromOption(self):
 		   self.process.source=cms.Source("DQMRootSource",
 						  fileNames = cms.untracked.vstring())
 		   filesFromOption(self)
+
+	   elif self._options.filetype == "DQMDAQ":
+		   # FIXME: how to configure it if there are no input files specified?
+		   self.process.source=cms.Source("DQMStreamerReader")
+
 
            if ('HARVESTING' in self.stepMap.keys() or 'ALCAHARVEST' in self.stepMap.keys()) and (not self._options.filetype == "DQM"):
                self.process.source.processingMode = cms.untracked.string("RunsAndLumis")
@@ -515,6 +521,8 @@ def anyOf(listOfKeys,dict,opt=None):
 				theFilterName='StreamALCACombined'
 
 			CppType='PoolOutputModule'
+			if self._options.timeoutOutput:
+				CppType='TimeoutPoolOutputModule'
 			if theStreamType=='DQM' and theTier=='DQMIO': CppType='DQMRootOutputModule'
 			output = cms.OutputModule(CppType,			
 						  theEventContent.clone(),
@@ -576,6 +584,8 @@ def doNotInlineEventContent(instance,label = "cms.untracked.vstring(process."+th
                         theFileName=self._options.outfile_name.replace('.root','_in'+streamType+'.root')
                         theFilterName=self._options.filtername
 		CppType='PoolOutputModule'
+		if self._options.timeoutOutput:
+			CppType='TimeoutPoolOutputModule'
 		if streamType=='DQM' and tier=='DQMIO': CppType='DQMRootOutputModule'
                 output = cms.OutputModule(CppType,
                                           theEventContent,

Configuration/Applications/python/Options.py

@@ -59,7 +59,7 @@
                   help="The type of the infile (EDM, LHE or MCDB).",
                   default=defaultOptions.filetype,
                   dest="filetype",
-                  choices=['EDM','DAT','LHE','MDCB','DQM']
+                  choices=['EDM','DAT','LHE','MDCB','DQM','DQMDAQ']
                   )
 
 parser.add_option("-n", "--number",
@@ -372,3 +372,9 @@
                           default=None,
                           dest='lumiToProcess'
                           )
+
+expertSettings.add_option("--timeoutOutput",
+                          help="use a TimeoutPoolOutputModule instead of a PoolOutputModule (needed for evt. display)",
+                          default=False,
+                          dest='timeoutOutput'
+                          )

Configuration/DataProcessing/python/Impl/cosmics.py

@@ -52,6 +52,20 @@ def expressProcessing(self, globalTag, **args):
         customiseCosmicData(process)  
         return process
 
+    def visualizationProcessing(self, globalTag, **args):
+        """
+        _visualizationProcessing_
+
+        Cosmic data taking visualization processing
+
+        """
+
+        process = Reco.visualizationProcessing(self,globalTag, **args)
+
+        customiseCosmicData(process)  
+        process.reconstructionCosmics.remove(process.lumiProducer)
+
+        return process
 
     def alcaHarvesting(self, globalTag, datasetName, **args):
         """

Configuration/DataProcessing/python/Impl/cosmicsRun2.py

@@ -55,6 +55,20 @@ def expressProcessing(self, globalTag, **args):
         customiseCosmicDataRun2(process)  
         return process
 
+    def visualizationProcessing(self, globalTag, **args):
+        """
+        _visualizationProcessing_
+
+        Cosmic data taking visualization processing
+
+        """
+
+        process = Reco.visualizationProcessing(self,globalTag, **args)
+
+        customiseCosmicDataRun2(process)  
+        process.reconstructionCosmics.remove(process.lumiProducer)
+
+        return process
 
     def alcaHarvesting(self, globalTag, datasetName, **args):
         """

Configuration/DataProcessing/python/Impl/pp.py

@@ -55,6 +55,18 @@ def expressProcessing(self, globalTag, **args):
 
         return process
 
+    def visualizationProcessing(self, globalTag, **args):
+        """
+        _visualizationProcessing_
+
+        Proton collision data taking visualization processing
+
+        """
+        process = Reco.visualizationProcessing(self,globalTag, **args)
+
+        customiseExpress(process)
+
+        return process
 
     def alcaHarvesting(self, globalTag, datasetName, **args):
         """

Configuration/DataProcessing/python/Impl/ppRun2.py

@@ -58,6 +58,18 @@ def expressProcessing(self, globalTag, **args):
 
         return process
 
+    def visualizationProcessing(self, globalTag, **args):
+        """
+        _visualizationProcessing_
+
+        Proton collision data taking visualization processing
+
+        """
+        process = Reco.visualizationProcessing(self,globalTag, **args)
+
+        customiseExpressRun2(process)
+
+        return process
 
     def alcaHarvesting(self, globalTag, datasetName, **args):
         """

Configuration/DataProcessing/python/Reco.py

@@ -90,6 +90,53 @@ def expressProcessing(self, globalTag, **args):
         return process
 
 
+    def visualizationProcessing(self, globalTag, **args):
+        """
+        _visualizationProcessing_
+
+        """
+
+        options = Options()
+        options.__dict__.update(defaultOptions.__dict__)
+        options.scenario = self.cbSc
+        # FIXME: do we need L1Reco here?
+        options.step =''
+        if 'preFilter' in args:
+            options.step +='FILTER:'+args['preFilter']+','
+
+        options.step += 'RAW2DIGI,L1Reco,RECO,ENDJOB'
+
+
+        dictIO(options,args)
+        options.conditions = globalTag
+        options.timeoutOutput = True
+        # FIXME: maybe can go...maybe not
+        options.filein = 'tobeoverwritten.xyz'
+
+        if 'inputSource' in args:
+            options.filetype = args['inputSource']
+        else:
+            # this is the default as this is what is needed on the OnlineCluster
+            options.filetype = 'DQMDAQ'
+
+        print "Using %s source"%options.filetype            
+
+        process = cms.Process('RECO')
+        cb = ConfigBuilder(options, process = process, with_output = True, with_input = True)
+
+        cb.prepare()
+
+
+
+
+        # FIXME: not sure abou this one...drop for the moment
+        # addMonitoring(process)
+
+        return process
+
+
+
+
     def alcaSkim(self, skims, **args):
         """
         _alcaSkim_

Configuration/DataProcessing/python/Scenario.py

@@ -70,6 +70,28 @@ def expressProcessing(self, globalTag, **options):
         raise NotImplementedError, msg
 
 
+
+    def visualizationProcessing(self, globalTag, **options):
+        """
+        _expressProcessing_
+
+        Build a configuration for the visualization processing for this scenario.
+
+        Visualization processing runs unpacking, and reco on 
+        streamer files and it is equipped to run on the online cluster
+        and writes RECO or FEVT files,
+        
+        writeTiers is list of tiers to write out.
+        
+
+        """
+        msg = "Scenario Implementation %s\n" % self.__class__.__name__
+        msg += "Does not contain an implementation for visualizationProcessing"
+        raise NotImplementedError, msg
+
+
+
+
     def alcaSkim(self, skims, **options):
         """
         _alcaSkim_

Configuration/DataProcessing/test/RunVisualizationProcessing.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python
+"""
+_RunVisualizationProcessing_
+
+Test wrapper to generate an express processing config and actually push
+it into cmsRun for testing with a few input files etc from the command line
+
+"""
+
+import sys
+import getopt
+
+from Configuration.DataProcessing.GetScenario import getScenario
+
+
+
+class RunVisualizationProcessing:
+
+    def __init__(self):
+        self.scenario = None
+        self.writeRaw = False
+        self.writeReco = False
+        self.writeFevt = False
+        self.writeAlca = False
+        self.writeDqm = False
+        self.noOutput = False
+        self.globalTag = None
+        self.inputLFN = None
+        self.preFilter = None
+
+
+ #FIXME: should add an option to specify an EDM input source?
+
+
+    def __call__(self):
+        if self.scenario == None:
+            msg = "No --scenario specified"
+            raise RuntimeError, msg
+        if self.globalTag == None:
+            msg = "No --global-tag specified"
+            raise RuntimeError, msg
+
+
+
+        try:
+            scenario = getScenario(self.scenario)
+        except Exception, ex:
+            msg = "Error getting Scenario implementation for %s\n" % (
+                self.scenario,)
+            msg += str(ex)
+            raise RuntimeError, msg
+
+        print "Retrieved Scenario: %s" % self.scenario
+        print "Using Global Tag: %s" % self.globalTag
+
+        dataTiers = []
+        if self.writeRaw:
+            dataTiers.append("RAW")
+            print "Configuring to Write out Raw..."
+        if self.writeReco:
+            dataTiers.append("RECO")
+            print "Configuring to Write out Reco..."
+        if self.writeFevt:
+            dataTiers.append("FEVT")
+            print "Configuring to Write out Fevt..."
+        if self.writeAlca:
+            dataTiers.append("ALCARECO")
+            print "Configuring to Write out Alca..."
+        if self.writeDqm:
+            dataTiers.append("DQM")
+            print "Configuring to Write out Dqm..."
+
+
+
+        try:
+            kwds = {}
+            if self.inputLFN != None:
+                kwds['inputSource'] = 'EDM'
+
+            if self.noOutput:
+                # get config without any output
+                kwds['writeTiers'] = []
+
+            elif len(dataTiers) > 0:
+                # get config with specified output
+                kwds['writeTiers'] = dataTiers
+
+            if self.preFilter:
+                kwds['preFilter'] = self.preFilter
+
+
+            # if none of the above use default output data tiers
+
+            process = scenario.visualizationProcessing(self.globalTag, **kwds)
+
+        except NotImplementedError, ex:
+            print "This scenario does not support Visualization Processing:\n"
+            return
+        except Exception, ex:
+            msg = "Error creating Visualization Processing config:\n"
+            msg += str(ex)
+            raise RuntimeError, msg
+
+        if self.inputLFN != None:
+            process.source.fileNames = [self.inputLFN]
+
+        import FWCore.ParameterSet.Config as cms
+
+        process.maxEvents = cms.untracked.PSet( input = cms.untracked.int32(10) )
+
+        psetFile = open("RunVisualizationProcessingCfg.py", "w")
+        psetFile.write(process.dumpPython())
+        psetFile.close()
+        cmsRun = "cmsRun -e RunVisualizationProcessingCfg.py"
+        print "Now do:\n%s" % cmsRun
+
+
+
+if __name__ == '__main__':
+    valid = ["scenario=", "reco", "fevt", "no-output",
+             "global-tag=", "lfn=",'preFilter=']
+    usage = \
+"""
+RunVisualizationProcessing.py <options>
+
+Where options are:
+ --scenario=ScenarioName
+ --reco (to enable RECO output)
+ --fevt (to enable FEVT output)
+ --no-output (create config with no output, overrides other settings)
+ --global-tag=GlobalTag
+ --lfn=/store/input/lfn
+ --preFilter=/sybsystem/package/filtername.sequence
+ 
+Example:
+python RunVisualizationProcessing.py --scenario cosmics --global-tag GLOBALTAG::ALL --lfn /store/whatever --reco
+
+"""
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "", valid)
+    except getopt.GetoptError, ex:
+        print usage
+        print str(ex)
+        sys.exit(1)
+
+
+    visualizator = RunVisualizationProcessing()
+
+    for opt, arg in opts:
+        if opt == "--scenario":
+            visualizator.scenario = arg
+        if opt == "--reco":
+            visualizator.writeReco = True
+        if opt == "--fevt":
+            visualizator.writeFevt = True
+        if opt == "--no-output":
+            visualizator.noOutput = True
+        if opt == "--global-tag":
+            visualizator.globalTag = arg
+        if opt == "--lfn" :
+            visualizator.inputLFN = arg
+        if opt == "--preFilter":
+            visualizator.preFilter = arg
+
+    visualizator()