From a8de6a4a0d75f2d4b606849dabb618a1ac232245 Mon Sep 17 00:00:00 2001 From: Eric Vaandering Date: Fri, 30 May 2014 13:55:20 -0500 Subject: [PATCH] Add run masking and URL reading to LumiList, Add list constuction mode and a way to find duplicates during construction, Fix a subtle bug do to re-used objects --- FWCore/PythonUtilities/python/LumiList.py | 94 +++++++++++++++++++++-- 1 file changed, 86 insertions(+), 8 deletions(-) diff --git a/FWCore/PythonUtilities/python/LumiList.py b/FWCore/PythonUtilities/python/LumiList.py index b1d9815648c65..2715989041f4b 100644 --- a/FWCore/PythonUtilities/python/LumiList.py +++ b/FWCore/PythonUtilities/python/LumiList.py @@ -10,8 +10,10 @@ """ +import copy import json import re +import urllib2 class LumiList(object): """ @@ -29,7 +31,8 @@ class LumiList(object): '2': [1,4,5,20] } where the first key is the run number and the list is a list of - individual lumi sections + individual lumi sections. This form also takes a list of these objects + which can be much faster than LumiList += LumiList Run lumi pairs: [[1,1], [1,2],[1,4], [2,1], [2,5], [1,10]] where each pair in the list is an individual run&lumi @@ -40,16 +43,21 @@ class LumiList(object): """ - def __init__(self, filename = None, lumis = None, runsAndLumis = None, runs = None, compactList = None): + def __init__(self, filename = None, lumis = None, runsAndLumis = None, runs = None, compactList = None, url = None): """ Constructor takes filename (JSON), a list of run/lumi pairs, or a dict with run #'s as the keys and a list of lumis as the values, or just a list of runs """ self.compactList = {} + self.duplicates = {} if filename: self.filename = filename jsonFile = open(self.filename,'r') self.compactList = json.load(jsonFile) + elif url: + self.url = url + jsonFile = urllib2.urlopen(url) + self.compactList = json.load(jsonFile) elif lumis: runsAndLumis = {} for (run, lumi) in lumis: @@ -58,6 +66,13 @@ def __init__(self, filename = None, lumis = None, runsAndLumis = None, runs = No runsAndLumis[run] = [] runsAndLumis[run].append(lumi) + if isinstance(runsAndLumis, list): + queued = {} + for runLumiList in runsAndLumis: + for run, lumis in runLumiList.items(): + queued.setdefault(run, []).extend(lumis) + runsAndLumis = queued + if runsAndLumis: for run in runsAndLumis.keys(): runString = str(run) @@ -65,9 +80,10 @@ def __init__(self, filename = None, lumis = None, runsAndLumis = None, runs = No lumiList = runsAndLumis[run] if lumiList: self.compactList[runString] = [] + self.duplicates[runString] = [] for lumi in sorted(lumiList): if lumi == lastLumi: - pass # Skip duplicates + self.duplicates[runString].append(lumi) elif lumi != lastLumi + 1: # Break in lumi sequence self.compactList[runString].append([lumi, lumi]) else: @@ -137,9 +153,9 @@ def __and__(self, other): # Things in both unique = [lumiList[0]] for pair in lumiList[1:]: if pair[0] == unique[-1][1]+1: - unique[-1][1] = pair[1] + unique[-1][1] = copy.deepcopy(pair[1]) else: - unique.append(pair) + unique.append(copy.deepcopy(pair)) result[run] = unique return LumiList(compactList = result) @@ -155,16 +171,16 @@ def __or__(self, other): unique = [overlap[0]] for pair in overlap[1:]: if pair[0] >= unique[-1][0] and pair[0] <= unique[-1][1]+1 and pair[1] > unique[-1][1]: - unique[-1][1] = pair[1] + unique[-1][1] = copy.deepcopy(pair[1]) elif pair[0] > unique[-1][1]: - unique.append(pair) + unique.append(copy.deepcopy(pair)) result[run] = unique return LumiList(compactList = result) def __add__(self, other): # + is the same as | - return self|other + return self.__or__(other) def __len__(self): '''Returns number of runs in list''' @@ -199,6 +215,13 @@ def getCompactList(self): return self.compactList + def getDuplicates(self): + """ + Return the list of duplicates found during construction as a LumiList + """ + return LumiList(runsAndLumis = self.duplicates) + + def getLumis(self): """ Return the list of pairs representation @@ -284,6 +307,22 @@ def removeRuns (self, runList): if self.compactList.has_key (run): del self.compactList[run] + return + + + def selectRuns (self, runList): + ''' + Selects only runs from runList in collection + ''' + runsToDelete = [] + for run in self.compactList.keys(): + if int(run) not in runList and run not in runList: + runsToDelete.append(run) + + for run in runsToDelete: + del self.compactList[run] + + return def contains (self, run, lumiSection = None): ''' @@ -533,6 +572,13 @@ def testOr(self): self.assertTrue((a|b).getCMSSWString() == (b|a).getCMSSWString()) self.assertTrue((a|b).getCMSSWString() == (a+b).getCMSSWString()) + # Test list constuction (faster) + + multiple = [alumis, blumis, clumis] + easy = LumiList(runsAndLumis = multiple) + hard = a + b + hard += c + self.assertTrue(hard.getCMSSWString() == easy.getCMSSWString()) def testAnd(self): """ @@ -560,6 +606,38 @@ def testAnd(self): self.assertTrue((a&b).getCMSSWString() == (b&a).getCMSSWString()) self.assertTrue((a|b).getCMSSWString() != r.getCMSSWString()) + def testRemoveSelect(self): + """ + a-b for lots of cases + """ + + alumis = {'1' : range(2,20) + range(31,39) + range(45,49), + '2' : range(6,20) + range (30,40), + '3' : range(10,20) + range (30,40) + range(50,60), + '4' : range(10,20) + range (30,80), + } + + result = {'2' : range(6,20) + range (30,40), + '4' : range(10,20) + range (30,80), + } + + rem = LumiList(runsAndLumis = alumis) + sel = LumiList(runsAndLumis = alumis) + res = LumiList(runsAndLumis = result) + + rem.removeRuns([1,3]) + sel.selectRuns([2,4]) + + self.assertTrue(rem.getCMSSWString() == res.getCMSSWString()) + self.assertTrue(sel.getCMSSWString() == res.getCMSSWString()) + self.assertTrue(sel.getCMSSWString() == rem.getCMSSWString()) + + def testURL(self): + URL = 'https://cms-service-dqm.web.cern.ch/cms-service-dqm/CAF/certification/Collisions12/8TeV/Reprocessing/Cert_190456-195530_8TeV_08Jun2012ReReco_Collisions12_JSON.txt' + ll = LumiList(url=URL) + self.assertTrue(len(ll) > 0) + + def testWrite(self): alumis = {'1' : range(2,20) + range(31,39) + range(45,49), '2' : range(6,20) + range (30,40),