Merge pull request #21472 from ghellwig/mps-fixes_94X

[9.4.X] HTCondor migration, fixes, improvements and preparation for AFS migration in MillePede workflow

Alignment/MillePedeAlignmentAlgorithm/python/alignmentsetup/SetCondition.py

@@ -21,4 +21,7 @@ def setCondition(process,
     if label is not None:
         args["label"] = cms.untracked.string(label)
 
+    process.GlobalTag.toGet \
+        = cms.VPSet(filter(lambda x: x.record.value() != record,
+                           process.GlobalTag.toGet.value()))
     process.GlobalTag.toGet.append(cms.PSet(**args))

Alignment/MillePedeAlignmentAlgorithm/python/alignmentsetup/helper.py

@@ -22,12 +22,13 @@ def checked_out_MPS():
     return checked_out, git_initialized
 
 
-def set_pede_option(process, option):
+def set_pede_option(process, option, drop = False):
     """Utility function to set or override pede `option` defined in `process`.
 
     Arguments:
     - `process`: cms.Process object
     - `option`: option string
+    - `drop`: if set to 'True' the `option` is dropped completely
     """
 
     existing_options = process.AlignmentProducer.algoConfig.pedeSteerer.options
@@ -37,9 +38,10 @@ def set_pede_option(process, option):
         if existing_options[i].split()[0] == option.split()[0]:
            existing_options[i] = option.strip()
            exists = True
+           if drop: existing_options.pop(i)
            break
 
-    if not exists: existing_options.append(option.strip())
+    if not exists and not drop: existing_options.append(option.strip())
 
 
 def add_filter(process, ed_filter):

Alignment/MillePedeAlignmentAlgorithm/python/mpslib/Mpslibclass.py

@@ -35,7 +35,7 @@
 #       JOBSP1      - spare
 #       JOBSP2      - possible weight for pede
 #       JOBSP3      - possible name as given to mps_setup.pl -N <name> ...
-#       JOBID       - what is this?
+#       JOBID       - ID of the LSF/HTCondor job
 
 import datetime
 import time
@@ -100,7 +100,7 @@ def read_db(self, db_file_name = __default_db_file_name):
             parts = line.split(":")                 #read each line and split into parts list
             self.JOBNUMBER.append(int(parts[0]))
             self.JOBDIR.append(parts[1].strip())
-            self.JOBID.append(int(parts[2]))
+            self.JOBID.append(parts[2])
             self.JOBSTATUS.append(parts[3].strip())
             self.JOBNTRY.append(int(parts[4]))
             self.JOBRUNTIME.append(int(parts[5]))   #int float?
@@ -145,7 +145,7 @@ def print_memdb(self):
         print '###     dir      jobid    stat  try  rtime      nevt  remark   weight  name'
         print "------------------------------------------------------------------------------"
         for i in xrange(self.nJobs):
-            print '%03d  %6s  %9d  %6s  %3d  %5d  %8d  %8s  %5s  %s' % (
+            print '%03d  %6s  %9s  %6s  %3d  %5d  %8d  %8s  %5s  %s' % (
                 self.JOBNUMBER[i],
                 self.JOBDIR[i],
                 self.JOBID[i],
@@ -160,7 +160,7 @@ def print_memdb(self):
         #print merge Jobs if merge mode
         if self.driver == 'merge':
             for i in xrange(self.nJobs,len(self.JOBDIR)):
-                print '%s  %6s  %9d  %6s  %3d  %5d  %8d  %8s  %5s  %s' % (
+                print '%s  %6s  %9s  %6s  %3d  %5d  %8d  %8s  %5s  %s' % (
                     'MMM',
                     self.JOBDIR[i],
                     self.JOBID[i],
@@ -216,7 +216,7 @@ def write_db(self):
 
         #write mps.db jobinfo
         for i in xrange(len(self.JOBID)):
-            DBFILE.write('%03d:%s:%05d:%s:%s:%s:%s:%s:%s:%s:%s:%s:%s\n' %
+            DBFILE.write('%03d:%s:%s:%s:%s:%s:%s:%s:%s:%s:%s:%s:%s\n' %
                          (i+1,
                           self.JOBDIR[i],
                           self.JOBID[i],

Alignment/MillePedeAlignmentAlgorithm/python/mpslib/tools.py

@@ -1,5 +1,7 @@
 import os
+import re
 import sys
+import shutil
 import importlib
 import sqlalchemy
 import subprocess
@@ -32,7 +34,8 @@ def create_single_iov_db(inputs, run_number, output_db):
             sys.exit(1)
 
     result = {}
-    if os.path.exists(output_db): os.remove(output_db)
+    remove_existing_object(output_db)
+
     for record,tag in inputs.iteritems():
         result[record] = {"connect": "sqlite_file:"+output_db,
                           "tag": "_".join([tag["tag"], tag["since"]])}
@@ -182,3 +185,77 @@ def get_iovs(db, tag):
     session.close()
 
     return sorted([int(item[0]) for item in iovs])
+
+
+def replace_factors(product_string, name, value):
+    """Takes a `product_string` and replaces all factors with `name` by `value`.
+
+    Arguments:
+    - `product_string`: input string containing a product
+    - `name`: name of the factor
+    - `value`: value of the factor
+    """
+
+    value = str(value)                                 # ensure it's a string
+    return re.sub(r"^"+name+r"$", value,               # single factor
+                  re.sub(r"[*]"+name+r"$", r"*"+value, # rhs
+                         re.sub(r"^"+name+r"[*]", value+r"*", # lhs
+                                re.sub(r"[*]"+name+r"[*]", r"*"+value+r"*",
+                                       product_string))))
+
+def compute_product_string(product_string):
+    """Takes `product_string` and returns the product of the factors as string.
+
+    Arguments:
+    - `product_string`: string containing product ('<factor>*<factor>*...')
+    """
+
+    factors = [float(f) for f in product_string.split("*")]
+    return str(reduce(lambda x,y: x*y, factors))
+
+
+def check_proxy():
+    """Check if GRID proxy has been initialized."""
+
+    try:
+        with open(os.devnull, "w") as dump:
+            subprocess.check_call(["voms-proxy-info", "--exists"],
+                                  stdout = dump, stderr = dump)
+    except subprocess.CalledProcessError:
+        return False
+    return True
+
+
+def remove_existing_object(path):
+    """
+    Tries to remove file or directory located at `path`. If the user
+    has no delete permissions, the object is moved to a backup
+    file. If this fails it tries 5 times in total and then asks to
+    perform a cleanup by a user with delete permissions.
+
+    Arguments:
+    - `name`: name of the object to be (re)moved
+    """
+
+    if os.path.exists(path):
+        remove_method = shutil.rmtree if os.path.isdir(path) else os.remove
+        move_method = shutil.move if os.path.isdir(path) else os.rename
+        try:
+            remove_method(path)
+        except OSError as e:
+            if e.args != (13, "Permission denied"): raise
+            backup_path = path.rstrip("/")+"~"
+            for _ in xrange(5):
+                try:
+                    if os.path.exists(backup_path): remove_method(backup_path)
+                    move_method(path, backup_path)
+                    break
+                except OSError as e:
+                    if e.args != (13, "Permission denied"): raise
+                    backup_path += "~"
+        if os.path.exists(path):
+            msg = ("Cannot remove '{}' due to missing 'delete' ".format(path)
+                   +"permissions and the limit of 5 backups is reached. Please "
+                   "ask a user with 'delete' permissions to clean up.")
+            print msg
+            sys.exit(1)