Merge cms-sw#4238 into CMSSW_7_1_X.

FWCore/PythonUtilities/python/LumiList.py

@@ -10,8 +10,10 @@
 """
 
 
+import copy
 import json
 import re
+import urllib2
 
 class LumiList(object):
     """
@@ -29,7 +31,8 @@ class LumiList(object):
         '2': [1,4,5,20]
         }
         where the first key is the run number and the list is a list of
-        individual lumi sections
+        individual lumi sections. This form also takes a list of these objects
+        which can be much faster than LumiList += LumiList
     Run  lumi pairs:
         [[1,1], [1,2],[1,4], [2,1], [2,5], [1,10]]
         where each pair in the list is an individual run&lumi
@@ -40,16 +43,21 @@ class LumiList(object):
     """
 
 
-    def __init__(self, filename = None, lumis = None, runsAndLumis = None, runs = None, compactList = None):
+    def __init__(self, filename = None, lumis = None, runsAndLumis = None, runs = None, compactList = None, url = None):
         """
         Constructor takes filename (JSON), a list of run/lumi pairs,
         or a dict with run #'s as the keys and a list of lumis as the values, or just a list of runs
         """
         self.compactList = {}
+        self.duplicates = {}
         if filename:
             self.filename = filename
             jsonFile = open(self.filename,'r')
             self.compactList = json.load(jsonFile)
+        elif url:
+            self.url = url
+            jsonFile = urllib2.urlopen(url)
+            self.compactList = json.load(jsonFile)
         elif lumis:
             runsAndLumis = {}
             for (run, lumi) in lumis:
@@ -58,16 +66,24 @@ def __init__(self, filename = None, lumis = None, runsAndLumis = None, runs = No
                     runsAndLumis[run] = []
                 runsAndLumis[run].append(lumi)
 
+        if isinstance(runsAndLumis, list):
+            queued = {}
+            for runLumiList in runsAndLumis:
+                for run, lumis in runLumiList.items():
+                    queued.setdefault(run, []).extend(lumis)
+            runsAndLumis = queued
+
         if runsAndLumis:
             for run in runsAndLumis.keys():
                 runString = str(run)
                 lastLumi = -1000
                 lumiList = runsAndLumis[run]
                 if lumiList:
                     self.compactList[runString] = []
+                    self.duplicates[runString] = []
                     for lumi in sorted(lumiList):
                         if lumi == lastLumi:
-                            pass # Skip duplicates
+                            self.duplicates[runString].append(lumi)
                         elif lumi != lastLumi + 1: # Break in lumi sequence
                             self.compactList[runString].append([lumi, lumi])
                         else:
@@ -137,9 +153,9 @@ def __and__(self, other): # Things in both
                 unique = [lumiList[0]]
             for pair in lumiList[1:]:
                 if pair[0] == unique[-1][1]+1:
-                    unique[-1][1] = pair[1]
+                    unique[-1][1] = copy.deepcopy(pair[1])
                 else:
-                    unique.append(pair)
+                    unique.append(copy.deepcopy(pair))
 
             result[run] = unique
         return LumiList(compactList = result)
@@ -155,16 +171,16 @@ def __or__(self, other):
             unique = [overlap[0]]
             for pair in overlap[1:]:
                 if pair[0] >= unique[-1][0] and pair[0] <= unique[-1][1]+1 and pair[1] > unique[-1][1]:
-                    unique[-1][1] = pair[1]
+                    unique[-1][1] = copy.deepcopy(pair[1])
                 elif pair[0] > unique[-1][1]:
-                    unique.append(pair)
+                    unique.append(copy.deepcopy(pair))
             result[run] = unique
         return LumiList(compactList = result)
 
 
     def __add__(self, other):
         # + is the same as |
-        return self|other
+        return self.__or__(other)
 
     def __len__(self):
         '''Returns number of runs in list'''
@@ -199,6 +215,13 @@ def getCompactList(self):
         return self.compactList
 
 
+    def getDuplicates(self):
+        """
+        Return the list of duplicates found during construction as a LumiList
+        """
+        return LumiList(runsAndLumis = self.duplicates)
+
+
     def getLumis(self):
         """
         Return the list of pairs representation
@@ -284,6 +307,22 @@ def removeRuns (self, runList):
             if self.compactList.has_key (run):
                 del self.compactList[run]
 
+        return
+
+
+    def selectRuns (self, runList):
+        '''
+        Selects only runs from runList in collection
+        '''
+        runsToDelete = []
+        for run in self.compactList.keys():
+            if int(run) not in runList and run not in runList:
+                runsToDelete.append(run)
+
+        for run in runsToDelete:
+            del self.compactList[run]
+
+        return
 
     def contains (self, run, lumiSection = None):
         '''
@@ -533,6 +572,13 @@ def testOr(self):
         self.assertTrue((a|b).getCMSSWString() == (b|a).getCMSSWString())
         self.assertTrue((a|b).getCMSSWString() == (a+b).getCMSSWString())
 
+        # Test list constuction (faster)
+
+        multiple = [alumis, blumis, clumis]
+        easy = LumiList(runsAndLumis = multiple)
+        hard = a + b
+        hard += c
+        self.assertTrue(hard.getCMSSWString() == easy.getCMSSWString())
 
     def testAnd(self):
         """
@@ -560,6 +606,38 @@ def testAnd(self):
         self.assertTrue((a&b).getCMSSWString() == (b&a).getCMSSWString())
         self.assertTrue((a|b).getCMSSWString() != r.getCMSSWString())
 
+    def testRemoveSelect(self):
+        """
+        a-b for lots of cases
+        """
+
+        alumis = {'1' : range(2,20) + range(31,39) + range(45,49),
+                  '2' : range(6,20) + range (30,40),
+                  '3' : range(10,20) + range (30,40) + range(50,60),
+                  '4' : range(10,20) + range (30,80),
+                 }
+
+        result = {'2' : range(6,20) + range (30,40),
+                  '4' : range(10,20) + range (30,80),
+                 }
+
+        rem = LumiList(runsAndLumis = alumis)
+        sel = LumiList(runsAndLumis = alumis)
+        res = LumiList(runsAndLumis = result)
+
+        rem.removeRuns([1,3])
+        sel.selectRuns([2,4])
+
+        self.assertTrue(rem.getCMSSWString() == res.getCMSSWString())
+        self.assertTrue(sel.getCMSSWString() == res.getCMSSWString())
+        self.assertTrue(sel.getCMSSWString() == rem.getCMSSWString())
+
+    def testURL(self):
+        URL = 'https://cms-service-dqm.web.cern.ch/cms-service-dqm/CAF/certification/Collisions12/8TeV/Reprocessing/Cert_190456-195530_8TeV_08Jun2012ReReco_Collisions12_JSON.txt'
+        ll = LumiList(url=URL)
+        self.assertTrue(len(ll) > 0)
+
+
     def testWrite(self):
         alumis = {'1' : range(2,20) + range(31,39) + range(45,49),
                   '2' : range(6,20) + range (30,40),