Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Change profiler format for igprof #27338

Merged
merged 3 commits into from Jun 27, 2019
Merged

Change profiler format for igprof #27338

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
422e8fa
In ConfigBuilder.py use hashlib to shorten profilerFormat name
gudrutis Jun 21, 2019
d95c027
In ConfigBuilder.py shortened profilerFormat even more
gudrutis Jun 24, 2019
beb99b1
Commit formating
gudrutis Jun 25, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
49 changes: 25 additions & 24 deletions Configuration/Applications/python/ConfigBuilder.py
View file
Open in desktop
Expand Up @@ -6,12 +6,12 @@

import FWCore.ParameterSet.Config as cms
from FWCore.ParameterSet.Modules import _Module

import six
# The following import is provided for backward compatibility reasons.
# The function used to be defined in this file.
from FWCore.ParameterSet.MassReplace import massReplaceInputTag as MassReplaceInputTag

import hashlib
import sys
import re
import collections
Expand Down Expand Up @@ -213,7 +213,7 @@ def __init__(self, options, process = None, with_output = False, with_input = Fa
(hasattr(self._options,"datatier") and \
self._options.datatier and \
'DQMIO' in self._options.datatier):
print("removing ENDJOB from steps since not compatible with DQMIO dataTier")
print("removing ENDJOB from steps since not compatible with DQMIO dataTier")
self._options.step=self._options.step.replace(',ENDJOB','')


Expand Down Expand Up @@ -281,8 +281,8 @@ def profileOptions(self):

if len(profileOpts):
#type, given as first argument is unused here
profileOpts.pop(0)
if len(profileOpts):
profileOpts.pop(0)
if len(profileOpts):
startEvent = profileOpts.pop(0)
if not startEvent.isdigit():
raise Exception("%s is not a number" % startEvent)
Expand All @@ -297,12 +297,13 @@ def profileOptions(self):


if not profilerFormat:
profilerFormat = "%s___%s___%s___%s___%s___%s___%%I.gz" % (self._options.evt_type.replace("_cfi", ""),
self._options.step,
self._options.pileup,
self._options.conditions,
self._options.datatier,
self._options.profileTypeLabel)
profilerFormat = "%s___%s___%%I.gz" % (
self._options.evt_type.replace("_cfi", ""),
hashlib.md5(
str(self._options.step) + str(self._options.pileup) + str(self._options.conditions) +
str(self._options.datatier) + str(self._options.profileTypeLabel)
).hexdigest()
)
if not profilerJobFormat and profilerFormat.endswith(".gz"):
profilerJobFormat = profilerFormat.replace(".gz", "_EndOfJob.gz")
elif not profilerJobFormat:
Expand Down Expand Up @@ -454,7 +455,7 @@ def filesFromOption(self):
if self._options.inputCommands:
self._options.inputCommands+=',drop LHEXMLStringProduct_*_*_*,'
else:
self._options.inputCommands='keep *, drop LHEXMLStringProduct_*_*_*,'
self._options.inputCommands='keep *, drop LHEXMLStringProduct_*_*_*,'

if self.process.source and self._options.inputCommands:
if not hasattr(self.process.source,'inputCommands'): self.process.source.inputCommands=cms.untracked.vstring()
Expand Down Expand Up @@ -558,7 +559,7 @@ def addOutput(self):
if self._options.timeoutOutput:
CppType='TimeoutPoolOutputModule'
if theStreamType=='DQM' and theTier=='DQMIO': CppType='DQMRootOutputModule'
output = cms.OutputModule(CppType,
output = cms.OutputModule(CppType,
theEventContent.clone(),
fileName = cms.untracked.string(theFileName),
dataset = cms.untracked.PSet(
Expand All @@ -568,7 +569,7 @@ def addOutput(self):
if not theSelectEvent and hasattr(self.process,'generation_step') and theStreamType!='LHE':
output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('generation_step'))
if not theSelectEvent and hasattr(self.process,'filtering_step'):
output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('filtering_step'))
output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('filtering_step'))
if theSelectEvent:
output.SelectEvents =cms.untracked.PSet(SelectEvents = cms.vstring(theSelectEvent))

Expand Down Expand Up @@ -803,7 +804,7 @@ def dropSecondDropStar(iec):
self.process.source.dropDescendantsOfDroppedBranches = cms.untracked.bool(False)


return
return

def addConditions(self):
"""Add conditions to the process"""
Expand Down Expand Up @@ -857,7 +858,7 @@ def addCustomise(self,unsch=0):
allFcn.extend(custMap[opt])
for fcn in allFcn:
if allFcn.count(fcn)!=1:
raise Exception("cannot specify twice "+fcn+" as a customisation method")
raise Exception("cannot specify twice "+fcn+" as a customisation method")

for f in custMap:
# let python search for that package and do syntax checking at the same time
Expand Down Expand Up @@ -1010,7 +1011,7 @@ def define_Configs(self):
self.GENDefaultSeq='fixGenInfo'

if self._options.scenario=='cosmics':
self._options.pileup='Cosmics'
self._options.pileup='Cosmics'
self.DIGIDefaultCFF="Configuration/StandardSequences/DigiCosmics_cff"
self.RECODefaultCFF="Configuration/StandardSequences/ReconstructionCosmics_cff"
self.SKIMDefaultCFF="Configuration/StandardSequences/SkimsCosmics_cff"
Expand Down Expand Up @@ -1376,7 +1377,7 @@ def prepare_GEN(self, sequence = None):
except ImportError:
raise Exception("VertexSmearing type or beamspot "+self._options.beamspot+" unknown.")

if self._options.scenario == 'HeavyIons':
if self._options.scenario == 'HeavyIons':
if self._options.pileup=='HiMixGEN':
self.loadAndRemember("Configuration/StandardSequences/GeneratorMix_cff")
else:
Expand All @@ -1390,7 +1391,7 @@ def prepare_GEN(self, sequence = None):

if 'reGEN' in self.stepMap:
#stop here
return
return

""" Enrich the schedule with the summary of the filter step """
#the gen filter in the endpath
Expand Down Expand Up @@ -1614,7 +1615,7 @@ def leave(self,v): pass
self.scheduleSequence(filterSeq,'filtering_step')
self.nextScheduleIsConditional=True
## put it before all the other paths
self.productionFilterSequence = filterSeq
self.productionFilterSequence = filterSeq

return

Expand Down Expand Up @@ -1669,7 +1670,7 @@ def prepare_PATGEN(self, sequence = "miniGEN"):
''' Enrich the schedule with PATGEN '''
self.loadDefaultOrSpecifiedCFF(sequence,self.PATGENDefaultCFF) #this is unscheduled
self.labelsToAssociate.append('patGENTask')
if not self._options.runUnscheduled:
if not self._options.runUnscheduled:
raise Exception("MiniGEN production can only run in unscheduled mode, please run cmsDriver with --runUnscheduled")
if self._options.isData:
raise Exception("PATGEN step can only run on MC")
Expand Down Expand Up @@ -2102,7 +2103,7 @@ def create_process(self):
#yes, the cfg code gets out of sync here if a process is passed in. That could be fixed in the future
#assuming there is some way for the fwk to get the list of modifiers (and their stringified name)
if self.process == None:
if len(modifiers)>0:
if len(modifiers)>0:
self.process = cms.Process(self._options.name,*modifiers)
else:
self.process = cms.Process(self._options.name)
Expand Down Expand Up @@ -2228,7 +2229,7 @@ def prepare(self, doChecking = False):
self.pythonCfgCode +="process.options.numberOfStreams=cms.untracked.uint32("+self._options.nStreams+")\n"
self.pythonCfgCode +="process.options.numberOfConcurrentLuminosityBlocks=cms.untracked.uint32("+self._options.nConcurrentLumis+")\n"
self.process.options.numberOfThreads=cms.untracked.uint32(int(self._options.nThreads))
self.process.options.numberOfStreams=cms.untracked.uint32(int(self._options.nStreams))
self.process.options.numberOfStreams=cms.untracked.uint32(int(self._options.nStreams))
self.process.options.numberOfConcurrentLuminosityBlocks=cms.untracked.uint32(int(self._options.nConcurrentLumis))
#repacked version
if self._options.isRepacked:
Expand All @@ -2244,7 +2245,7 @@ def prepare(self, doChecking = False):
if len(self.conditionalPaths):
self.pythonCfgCode +='\tif not path in %s: continue\n'%str(self.conditionalPaths)
if len(self.excludedPaths):
self.pythonCfgCode +='\tif path in %s: continue\n'%str(self.excludedPaths)
self.pythonCfgCode +='\tif path in %s: continue\n'%str(self.excludedPaths)
self.pythonCfgCode +='\tgetattr(process,path).insert(0, process.%s)\n'%(self.productionFilterSequence,)
pfs = getattr(self.process,self.productionFilterSequence)
for path in self.process.paths:
Expand All @@ -2256,7 +2257,7 @@ def prepare(self, doChecking = False):
# dump customise fragment
self.pythonCfgCode += self.addCustomise()

if self._options.runUnscheduled:
if self._options.runUnscheduled:
# prune and delete paths
#this is not supporting the blacklist at this point since I do not understand it
self.pythonCfgCode+="#do not add changes to your config after this point (unless you know what you are doing)\n"
Expand Down