Merge pull request #30600 from mteroerd/ape-restructure

APE tool change of configuration, additional python3 compatibility changes

Alignment/APEEstimation/test/autoSubmitter/autoSubmitter.py

@@ -112,6 +112,7 @@ class ApeMeasurement:
     curIteration = 0
     firstIteration = 0
     maxIterations = 15
+    maxEvents = None
     status = STATE_NONE
     dataset = None
     alignment = None
@@ -120,29 +121,40 @@ class ApeMeasurement:
     startTime = ""
     finishTime = ""
 
-    def __init__(self, name, dataset, alignment, config, additionalOptions={}):
-        self.name = name
-        self.alignment = alignment
-        self.dataset = dataset
-        self.curIteration = 0
+    def __init__(self, name, config, settings):
+        self.name = name        
         self.status = STATE_ITERATION_START
         self.runningJobs = []
         self.failedJobs = []
         self.startTime = subprocess.check_output(["date"]).strip()
 
-        self.maxEvents = self.dataset.maxEvents
-
-        for key, value in additionalOptions.items():
-            setattr(self, key, value)
+        # load conditions from dictionary, overwrite defaults if defined
+        for key, value in settings.items():
+            if not key.startswith("condition "):
+                setattr(self, key, value)
+
+        # Replace names with actual Dataset and Alignment objects
+        # In principle, one could preload all these once so they are not
+        # redefined for each measurement, but right now this does not 
+        # seem necessary
+        self.dataset = Dataset(config, settings["dataset"])
+        self.alignment = Alignment(config, settings["alignment"])
+
+        # If not defined here, replace by setting from Dataset
+        if not "maxEvents" in settings:
+            self.maxEvents = self.dataset.maxEvents
+
         self.firstIteration=int(self.firstIteration)
         self.maxIterations=int(self.maxIterations)
         self.curIteration = self.firstIteration
         self.maxEvents = int(self.maxEvents)
         if self.alignment.isDesign:
             self.maxIterations = 0
 
+        self.conditions, dummy, self.validConditions = loadConditions(settings) 
 
-        if not self.alignment.validConditions or not self.dataset.validConditions or not self.existingFiles:
+        # see if sanity checks passed
+        if not self.alignment.validConditions or not self.dataset.validConditions or not self.dataset.existingFiles or not self.validConditions:
             self.status = STATE_INVALID_CONDITIONS
             self.print_status()
             self.finishTime = subprocess.check_output(["date"]).strip()
@@ -167,10 +179,12 @@ def print_status(self):
     def submit_jobs(self):
         toSubmit = []
 
-        allConditions = self.alignment.conditions+self.dataset.conditions
+        allConditions = self.alignment.conditions+self.dataset.conditions+self.conditions
         allConditions = list({v['record']:v for v in allConditions}.values()) # should we clean for duplicate records? the record last defined (from dataset) 
                                                                               # will be kept in case of overlap, which is the same as if there was no overlap removal
 
+        ensurePathExists("{}/test/autoSubmitter/workingArea".format(base))
+
         # If conditions are made, create file to load them from
         rawFileName = "None"
         conditionsFileName = "None"
@@ -415,19 +429,16 @@ def run_iteration(self):
             if self.status == STATE_ITERATION_START:
                 # start bjobs
                 print("APE Measurement {} just started iteration {}".format(self.name, self.curIteration))
-                while True: 
-                    try:
-                        self.submit_jobs()
-                        save("measurements", measurements)
-                        break
-                    except KeyboardInterrupt:
-                        exit()
-                    except:
-                        # this is needed in case the scheduler goes down
-                        print("Error submitting, waiting for 1 minute")
-                        time.sleep(60)
+
+
+                try:
+                    self.submit_jobs()
+                    save("measurements", measurements)
+                except:
+                    # this is needed in case the scheduler goes down
+                    print("Error submitting jobs for APE measurement {}".format(self.name))
+                    return
 
-                    continue # no reason to immediately check jobs
             if self.status == STATE_BJOBS_WAITING:
                 # check if bjobs are finished
                 self.check_jobs()
@@ -520,6 +531,7 @@ def main():
         print("No CMSSW environment was set, exiting")
         sys.exit()
 
+
     killTargets = []
     purgeTargets = []
     for toConvert in args.kill:
@@ -569,31 +581,22 @@ def main():
         config = ConfigParser.RawConfigParser()
         config.optionxform = str 
         config.read(args.configs)
-
-        for name, opts in config.items("measurements"):
+
+        # read measurement names
+        meas = [str(x.split("ape:")[1]) for x in list(config.keys()) if x.startswith("ape:")]
+
+        for name in meas:
             if name in [x.name for x in measurements]:
                 print("Error: APE Measurement with name {} already exists, skipping".format(name))
                 continue
+            settings = dict(config.items("ape:{}".format(name)))
 
-            settings = opts.split(" ")
-            if len(settings) < 2:
-                print("Error: number of arguments for APE Measurement {} is insufficient".format(name))
-                sys.exit()
-
-            dataset = Dataset(config, settings[0].strip())
-            alignment = Alignment(config, settings[1].strip())
-            additionalOptions = {}
-
-            for i in range(2, len(settings)):
-                setting = settings[i].strip()
-                key, value = setting.split("=")
-                additionalOptions[key] = value
-
-            measurement = ApeMeasurement(name, dataset, alignment, config, additionalOptions)
-
-            measurements.append(measurement)
+            measurement = ApeMeasurement(name, config, settings)
 
-            print("APE Measurement {} was started".format(measurement.name))
+            if measurement.status >= STATE_ITERATION_START and measurement.status <= STATE_FINISHED:
+                measurements.append(measurement)
+                print("APE Measurement {} was started".format(measurement.name))
+
 
 
     while True:

Alignment/APEEstimation/test/autoSubmitter/config.ini

@@ -31,15 +31,13 @@ condition Alignments+Deformations:sqlite_file:/afs/asdf.db
 # in this case, the Rcd name has to be guessed by the autoSubmitter, 
 # so functionality is limited
 
-# define measurements like this
-[measurements]
-# name: dataset alignment
-# unique names are important as this name will be used as a folder name
-# where all relevant files are stored
-# additional arguments are added with spaces in the structure option=value
-exampleName: exampleDataset alignmentObject 
-# Additional arguments can be (default arguments in brackets):
-# firstIteration (0)
-# maxIterations (15) 
-# maxEvents (take value from dataset)
-exampleName2: exampleDataset alignmentObject firstIteration=0 maxIterations=10 maxEvents=1000
+[ape:name]
+dataset: exampleDataset
+alignment: alignmentObject
+
+[ape:name2]
+dataset: exampleDataset
+alignment: alignmentObject
+firstIteration: 0
+maxIterations: 15
+maxEvents: 1000

Alignment/APEEstimation/test/plottingTools/iterationsPlotter.py

@@ -45,7 +45,7 @@ def convertName(self, name):
         return out
 
     def makeHists(self, sectorRange, coordinate):
-        sectors = range(sectorRange[0],sectorRange[1]+1)
+        sectors = list(range(sectorRange[0],sectorRange[1]+1))
         numSectors = len(sectors)
 
         fi = ROOT.TFile(self.inFile, "READ")