Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Edm pick events crab3 updates from cmssw 7 4 0 #9999

Merged
merged 3 commits into from
Jul 8, 2015
Merged

Edm pick events crab3 updates from cmssw 7 4 0 #9999

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6d01334
CRAB3 updates for edmPickEvents
ExoPie Jun 30, 2015
f018a70
CRAB3 updates for edmPickEvents, includes fix from Dinko
ExoPie Jun 30, 2015
73198b6
CRAB3 updates for edmPickEvents
ExoPie Jun 30, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 74 additions & 51 deletions PhysicsTools/Utilities/scripts/edmPickEvents.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@
# Oleksiy Atramentov June 21, 2008
# Charles Plager Sept 7, 2010
# Volker Adler Apr 16, 2014

# Raman Khurana June 18, 2015
# Dinko Ferencek June 27, 2015
import os
import sys
import optparse
Expand All @@ -14,6 +15,7 @@
import das_client
import json
from pprint import pprint
from datetime import datetime


help = """
Expand All @@ -25,7 +27,6 @@

edmPickEvent.py dataset listOfEvents.txt


listOfEvents is a text file:
# this line is ignored as a comment
# since '#' is a valid comment character
Expand Down Expand Up @@ -124,17 +125,18 @@ def guessEmail():
return '%s@%s' % (commands.getoutput ('whoami'),
'.'.join(commands.getoutput('hostname').split('.')[-2:]))


def setupCrabDict (options):
date = datetime.now().strftime('%Y%m%d_%H%M%S')
crab = {}
base = options.base
crab['runEvent'] = '%s_runEvents.txt' % base
crab['copyPickMerge'] = fullCPMpath()
crab['output'] = '%s.root' % base
crab['crabcfg'] = '%s_crab.config' % base
crab['json'] = '%s.json' % base
crab['dataset'] = Event.dataset
crab['email'] = options.email
crab['runEvent'] = '%s_runEvents.txt' % base
crab['copyPickMerge'] = fullCPMpath()
crab['output'] = '%s.root' % base
crab['crabcfg'] = '%s_crab.py' % base
crab['json'] = '%s.json' % base
crab['dataset'] = Event.dataset
crab['email'] = options.email
crab['WorkArea'] = date
if options.crabCondor:
crab['scheduler'] = 'condor'
# crab['useServer'] = ''
Expand All @@ -144,42 +146,50 @@ def setupCrabDict (options):
crab['useServer'] = ''
return crab


# crab template
crabTemplate = '''
# CRAB documentation:
# https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrab
#
# Once you are happy with this file, please run
# crab -create -cfg %(crabcfg)s
# crab -submit -cfg %(crabcfg)s

[CMSSW]
pycfg_params = eventsToProcess_load=%(runEvent)s outputFile=%(output)s

lumi_mask = %(json)s
total_number_of_lumis = -1
lumis_per_job = 1
pset = %(copyPickMerge)s
datasetpath = %(dataset)s
output_file = %(output)s

[USER]
return_data = 1
email = %(email)s

# if you want to copy the data or put it in a storage element, do it
# here.


[CRAB]
# use "glite" in general; you can "condor" if you run on CAF at FNAL or USG
# site AND you know the files are available locally
scheduler = %(scheduler)s
jobtype = cmssw
%(useServer)s
'''
## Edited By Raman Khurana
##
## CRAB documentation : https://twiki.cern.ch/twiki/bin/view/CMSPublic/SWGuideCrab
##
## CRAB 3 parameters : https://twiki.cern.ch/twiki/bin/view/CMSPublic/CRAB3ConfigurationFile#CRAB_configuration_parameters
##
## Once you are happy with this file, please run
## crab submit

## In CRAB3 the configuration file is in Python language. It consists of creating a Configuration object imported from the WMCore library:

from WMCore.Configuration import Configuration
config = Configuration()

## Once the Configuration object is created, it is possible to add new sections into it with corresponding parameters
config.section_("General")
config.General.requestName = 'pickEvents'
config.General.workArea = 'crab_pickevents_%(WorkArea)s'


config.section_("JobType")
config.JobType.pluginName = 'Analysis'
config.JobType.psetName = '%(copyPickMerge)s'
config.JobType.pyCfgParams = ['eventsToProcess_load=%(runEvent)s', 'outputFile=%(output)s']

config.section_("Data")
config.Data.inputDataset = '%(dataset)s'

config.Data.inputDBS = 'global'
config.Data.splitting = 'LumiBased'
config.Data.unitsPerJob = 5
config.Data.lumiMask = '%(json)s'
#config.Data.publication = True
#config.Data.publishDbsUrl = 'phys03'
#config.Data.publishDataName = 'CRAB3_CSA_DYJets'
#config.JobType.allowNonProductionCMSSW=True

config.section_("Site")
## Change site name accordingly
config.Site.storageSite = "T2_US_Wisconsin"

'''

########################
## ################## ##
Expand All @@ -191,14 +201,17 @@ def setupCrabDict (options):
email = guessEmail()
parser = optparse.OptionParser ("Usage: %prog [options] dataset events_or_events.txt", description='''This program
facilitates picking specific events from a data set. For full details, please visit
https://twiki.cern.ch/twiki/bin/view/CMS/PickEvents ''')
https://twiki.cern.ch/twiki/bin/view/CMSPublic/WorkBookPickEvents ''')
parser.add_option ('--output', dest='base', type='string',
default='pickevents',
help='Base name to use for output files (root, JSON, run and event list, etc.; default "%default")')
parser.add_option ('--runInteractive', dest='runInteractive', action='store_true',
help = 'Call "cmsRun" command if possible. Can take a long time.')
parser.add_option ('--printInteractive', dest='printInteractive', action='store_true',
help = 'Print "cmsRun" command instead of running it.')
parser.add_option ('--maxEventsInteractive', dest='maxEventsInteractive', type='int',
default=20,
help = 'Maximum number of events allowed to be processed interactively.')
parser.add_option ('--crab', dest='crab', action='store_true',
help = 'Force CRAB setup instead of interactive mode')
parser.add_option ('--crabCondor', dest='crabCondor', action='store_true',
Expand Down Expand Up @@ -241,7 +254,11 @@ def setupCrabDict (options):
eventList.append(event)
source.close()

if len (eventList) > 20:
if not eventList:
print "No events defined. Aborting."
sys.exit()

if len (eventList) > options.maxEventsInteractive:
options.crab = True

if options.crab:
Expand All @@ -250,7 +267,7 @@ def setupCrabDict (options):
## CRAB ##
##########
if options.runInteractive:
raise RuntimeError, "This job is can not be run interactive, but rather by crab. Please call without '--runInteractive' flag."
raise RuntimeError, "This job cannot be run interactively, but rather by crab. Please call without the '--runInteractive' flag or increase the '--maxEventsInteractive' value."
runsAndLumis = [ (event.run, event.lumi) for event in eventList]
json = LumiList (lumis = runsAndLumis)
eventsToProcess = '\n'.join(\
Expand All @@ -268,19 +285,25 @@ def setupCrabDict (options):
print "You are running on condor. Please make sure you have read instructions on\nhttps://twiki.cern.ch/twiki/bin/view/CMS/CRABonLPCCAF\n"
if not os.path.exists ('%s/.profile' % os.environ.get('HOME')):
print "** WARNING: ** You are missing ~/.profile file. Please see CRABonLPCCAF instructions above.\n"
print "Setup your environment for CRAB. Then edit %(crabcfg)s to make any desired changed. The run:\n\ncrab -create -cfg %(crabcfg)s\ncrab -submit\n" % crabDict
print "Setup your environment for CRAB and edit %(crabcfg)s to make any desired changed. Then run:\n\ncrab submit -c %(crabcfg)s\n" % crabDict

else:

#################
## Interactive ##
#################
files = []
eventPurgeList = []
for event in eventList:
files.extend( getFileNames (event) )
if not eventList:
print "No events defind. Aborting."
sys.exit()
eventFiles = getFileNames (event)
if eventFiles == ['[]']: # event not contained in the input dataset
print "** WARNING: ** According to a DAS query, run = %i; lumi = %i; event = %i not contained in %s. Skipping."%(event.run,event.lumi,event.event,event.dataset)
eventPurgeList.append( event )
else:
files.extend( eventFiles )
# Purge events
for event in eventPurgeList:
eventList.remove( event )
# Purge duplicate files
fileSet = set()
uniqueFiles = []
Expand Down