Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Improvements to LumiList on 70x branch #4238

Merged
merged 1 commit into from Jun 18, 2014
Merged

Improvements to LumiList on 70x branch #4238

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
94 changes: 86 additions & 8 deletions FWCore/PythonUtilities/python/LumiList.py
View file
Open in desktop
Expand Up @@ -10,8 +10,10 @@
"""


import copy
import json
import re
import urllib2

class LumiList(object):
"""
Expand All @@ -29,7 +31,8 @@ class LumiList(object):
'2': [1,4,5,20]
}
where the first key is the run number and the list is a list of
individual lumi sections
individual lumi sections. This form also takes a list of these objects
which can be much faster than LumiList += LumiList
Run lumi pairs:
[[1,1], [1,2],[1,4], [2,1], [2,5], [1,10]]
where each pair in the list is an individual run&lumi
Expand All @@ -40,16 +43,21 @@ class LumiList(object):
"""


def __init__(self, filename = None, lumis = None, runsAndLumis = None, runs = None, compactList = None):
def __init__(self, filename = None, lumis = None, runsAndLumis = None, runs = None, compactList = None, url = None):
"""
Constructor takes filename (JSON), a list of run/lumi pairs,
or a dict with run #'s as the keys and a list of lumis as the values, or just a list of runs
"""
self.compactList = {}
self.duplicates = {}
if filename:
self.filename = filename
jsonFile = open(self.filename,'r')
self.compactList = json.load(jsonFile)
elif url:
self.url = url
jsonFile = urllib2.urlopen(url)
self.compactList = json.load(jsonFile)
elif lumis:
runsAndLumis = {}
for (run, lumi) in lumis:
Expand All @@ -58,16 +66,24 @@ def __init__(self, filename = None, lumis = None, runsAndLumis = None, runs = No
runsAndLumis[run] = []
runsAndLumis[run].append(lumi)

if isinstance(runsAndLumis, list):
queued = {}
for runLumiList in runsAndLumis:
for run, lumis in runLumiList.items():
queued.setdefault(run, []).extend(lumis)
runsAndLumis = queued

if runsAndLumis:
for run in runsAndLumis.keys():
runString = str(run)
lastLumi = -1000
lumiList = runsAndLumis[run]
if lumiList:
self.compactList[runString] = []
self.duplicates[runString] = []
for lumi in sorted(lumiList):
if lumi == lastLumi:
pass # Skip duplicates
self.duplicates[runString].append(lumi)
elif lumi != lastLumi + 1: # Break in lumi sequence
self.compactList[runString].append([lumi, lumi])
else:
Expand Down Expand Up @@ -137,9 +153,9 @@ def __and__(self, other): # Things in both
unique = [lumiList[0]]
for pair in lumiList[1:]:
if pair[0] == unique[-1][1]+1:
unique[-1][1] = pair[1]
unique[-1][1] = copy.deepcopy(pair[1])
else:
unique.append(pair)
unique.append(copy.deepcopy(pair))

result[run] = unique
return LumiList(compactList = result)
Expand All @@ -155,16 +171,16 @@ def __or__(self, other):
unique = [overlap[0]]
for pair in overlap[1:]:
if pair[0] >= unique[-1][0] and pair[0] <= unique[-1][1]+1 and pair[1] > unique[-1][1]:
unique[-1][1] = pair[1]
unique[-1][1] = copy.deepcopy(pair[1])
elif pair[0] > unique[-1][1]:
unique.append(pair)
unique.append(copy.deepcopy(pair))
result[run] = unique
return LumiList(compactList = result)


def __add__(self, other):
# + is the same as |
return self|other
return self.__or__(other)

def __len__(self):
'''Returns number of runs in list'''
Expand Down Expand Up @@ -199,6 +215,13 @@ def getCompactList(self):
return self.compactList


def getDuplicates(self):
"""
Return the list of duplicates found during construction as a LumiList
"""
return LumiList(runsAndLumis = self.duplicates)


def getLumis(self):
"""
Return the list of pairs representation
Expand Down Expand Up @@ -284,6 +307,22 @@ def removeRuns (self, runList):
if self.compactList.has_key (run):
del self.compactList[run]

return


def selectRuns (self, runList):
'''
Selects only runs from runList in collection
'''
runsToDelete = []
for run in self.compactList.keys():
if int(run) not in runList and run not in runList:
runsToDelete.append(run)

for run in runsToDelete:
del self.compactList[run]

return

def contains (self, run, lumiSection = None):
'''
Expand Down Expand Up @@ -533,6 +572,13 @@ def testOr(self):
self.assertTrue((a|b).getCMSSWString() == (b|a).getCMSSWString())
self.assertTrue((a|b).getCMSSWString() == (a+b).getCMSSWString())

# Test list constuction (faster)

multiple = [alumis, blumis, clumis]
easy = LumiList(runsAndLumis = multiple)
hard = a + b
hard += c
self.assertTrue(hard.getCMSSWString() == easy.getCMSSWString())

def testAnd(self):
"""
Expand Down Expand Up @@ -560,6 +606,38 @@ def testAnd(self):
self.assertTrue((a&b).getCMSSWString() == (b&a).getCMSSWString())
self.assertTrue((a|b).getCMSSWString() != r.getCMSSWString())

def testRemoveSelect(self):
"""
a-b for lots of cases
"""

alumis = {'1' : range(2,20) + range(31,39) + range(45,49),
'2' : range(6,20) + range (30,40),
'3' : range(10,20) + range (30,40) + range(50,60),
'4' : range(10,20) + range (30,80),
}

result = {'2' : range(6,20) + range (30,40),
'4' : range(10,20) + range (30,80),
}

rem = LumiList(runsAndLumis = alumis)
sel = LumiList(runsAndLumis = alumis)
res = LumiList(runsAndLumis = result)

rem.removeRuns([1,3])
sel.selectRuns([2,4])

self.assertTrue(rem.getCMSSWString() == res.getCMSSWString())
self.assertTrue(sel.getCMSSWString() == res.getCMSSWString())
self.assertTrue(sel.getCMSSWString() == rem.getCMSSWString())

def testURL(self):
URL = 'https://cms-service-dqm.web.cern.ch/cms-service-dqm/CAF/certification/Collisions12/8TeV/Reprocessing/Cert_190456-195530_8TeV_08Jun2012ReReco_Collisions12_JSON.txt'
ll = LumiList(url=URL)
self.assertTrue(len(ll) > 0)


def testWrite(self):
alumis = {'1' : range(2,20) + range(31,39) + range(45,49),
'2' : range(6,20) + range (30,40),
Expand Down