Merge pull request #24414 from adewit/tkal-mp-devs

Tracker alignment - several MP developments
cms-sw · Aug 31, 2018 · af20b1d · af20b1d
2 parents 19ae827 + d6d0626
commit af20b1d
Show file tree

Hide file tree

Showing 4 changed files with 121 additions and 29 deletions.
diff --git a/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_alisetup.py b/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_alisetup.py
@@ -182,9 +182,8 @@ def _create_mass_storage_directory(self):
 
         # set directory on eos
         self._mss_dir = self._general_options.get("massStorageDir",
-                                                  "/eos/cms/store/caf/user/"
-                                                  +os.environ["USER"])
-        self._mss_dir = os.path.join(self._mss_dir, "MPproduction",
+                                                  "/eos/cms/store/group/alca_millepede/")
+        self._mss_dir = os.path.join(self._mss_dir, "MPproductionFiles",
                                      self._mps_dir_name)
 
         cmd = ["mkdir", "-p", self._mss_dir]

diff --git a/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_fire.py b/Alignment/MillePedeAlignmentAlgorithm/scripts/mps_fire.py
@@ -141,6 +141,13 @@ def write_HTCondor_submit_file(path, script, config, lib):
                     action="store_true",
                     help=("force the submission of the Pede job in case some "+
                           "Mille jobs are not in the OK state"))
+parser.add_argument("--force-merge-manual", dest="forceMergeManual", default=False,
+                    action="store_true",
+                    help=("force the submission of the Pede job in case some "+
+                          "Mille jobs are not in the OK state. Unlike --forceMerge "+
+                          "this option assumes the user has edited theScript.sh and "+
+                          "alignment_merge.py to consistently pick up only the mille "+
+                          "output files that exist"))
 parser.add_argument("-p", "--forward-proxy", dest="forwardProxy", default=False,
                     action="store_true",
                     help="forward VOMS proxy to batch system")
@@ -278,7 +285,7 @@ def write_HTCondor_submit_file(path, script, config, lib):
         if lib.JOBSTATUS[i] != 'SETUP':
             print('Merge job %d status %s not submitted.' % \
                   (jobNumFrom1, lib.JOBSTATUS[i]))
-        elif not (mergeOK or args.forceMerge):
+        elif not (mergeOK or args.forceMerge or args.forceMergeManual):
             print('Merge job',jobNumFrom1,'not submitted since Mille jobs error/unfinished (Use -m -f to force).')
         else:
             # some paths for clarity

diff --git a/Alignment/MillePedeAlignmentAlgorithm/templates/mps_runMille_template.sh b/Alignment/MillePedeAlignmentAlgorithm/templates/mps_runMille_template.sh
@@ -22,8 +22,69 @@ clean_up () {
 #LSF signals according to http://batch.web.cern.ch/batch/lsf-return-codes.html
 trap clean_up HUP INT TERM SEGV USR2 XCPU XFSZ IO
 
+# a helper function to repeatedly try failing copy commands
+untilSuccess () {
+# trying "${1} ${2} ${3} > /dev/null" until success, if ${4} is a
+# positive number run {1} with -f flag,
+# break after ${5} tries (with four arguments do up to 5 tries).
+    if  [[ ${#} -lt 4 || ${#} -gt 5 ]]
+    then
+        echo ${0} needs 4 or 5 arguments
+        return 1
+    fi
+
+    TRIES=0
+    MAX_TRIES=5
+    if [[ ${#} -eq 5 ]]
+    then
+        MAX_TRIES=${5}
+    fi
+
+
+    if [[ ${4} -gt 0 ]]
+    then 
+        ${1} -f ${2} ${3} > /dev/null
+    else 
+        ${1} ${2} ${3} > /dev/null
+    fi
+    while [[ ${?} -ne 0 ]]
+    do # if not successfull, retry...
+        if [[ ${TRIES} -ge ${MAX_TRIES} ]]
+        then # ... but not until infinity!
+            if [[ ${4} -gt 0 ]]
+            then
+                echo ${0}: Give up doing \"${1} -f ${2} ${3} \> /dev/null\".
+                return 1
+            else
+                echo ${0}: Give up doing \"${1} ${2} ${3} \> /dev/null\".
+                return 1
+            fi
+        fi
+        TRIES=$((${TRIES}+1))
+        if [[ ${4} -gt 0 ]]
+        then
+            echo ${0}: WARNING, problems with \"${1} -f ${2} ${3} \> /dev/null\", try again.
+            sleep $((${TRIES}*5)) # for before each wait a litte longer...
+            ${1} -f ${2} ${3} > /dev/null
+        else
+            echo ${0}: WARNING, problems with \"${1} ${2} ${3} \> /dev/null\", try again.
+            sleep $((${TRIES}*5)) # for before each wait a litte longer...
+            ${1} ${2} ${3} > /dev/null
+        fi
+    done
+
+    if [[ ${4} -gt 0 ]]
+    then
+        echo successfully executed \"${1} -f ${2} ${3} \> /dev/null\"
+    else
+        echo successfully executed \"${1} ${2} ${3} \> /dev/null\"
+    fi
+    return 0
+}
+
 export X509_USER_PROXY=${RUNDIR}/.user_proxy
 
+
 # The batch job directory (will vanish after job end):
 BATCH_DIR=$(pwd)
 echo "Running at $(date) \n        on $HOST \n        in directory $BATCH_DIR."
@@ -58,9 +119,9 @@ if [ "$MSSDIRPOOL" != "cmscafuser" ]; then
   export STAGER_TRACE=
   nsrm -f $MSSDIR/milleBinaryISN.dat.gz
   echo "rfcp milleBinaryISN.dat.gz $MSSDIR/"
-  rfcp milleBinaryISN.dat.gz    $MSSDIR/
-  rfcp treeFile*root         $MSSDIR/treeFileISN.root
-  rfcp millePedeMonitor*root $MSSDIR/millePedeMonitorISN.root
+  untilSuccess rfcp milleBinaryISN.dat.gz   $MSSDIR/ 0
+  untilSuccess rfcp treeFile*root         $MSSDIR/treeFileISN.root 0
+  untilSuccess rfcp millePedeMonitor*root $MSSDIR/millePedeMonitorISN.root 0
 else
   MSSCAFDIR=`echo $MSSDIR | perl -pe 's/\/castor\/cern.ch\/cms//gi'`
   # ensure the directories exists
@@ -69,7 +130,7 @@ else
   mkdir -p ${MSSCAFDIR}/monitors
   # copy the files
   echo "xrdcp -f milleBinaryISN.dat.gz ${MSSCAFDIR}/binaries/milleBinaryISN.dat.gz > /dev/null"
-  xrdcp -f milleBinaryISN.dat.gz    ${MSSCAFDIR}/binaries/milleBinaryISN.dat.gz    > /dev/null
-  xrdcp -f treeFile.root            ${MSSCAFDIR}/tree_files/treeFileISN.root       > /dev/null
-  xrdcp -f millePedeMonitorISN.root ${MSSCAFDIR}/monitors/millePedeMonitorISN.root > /dev/null
+  untilSuccess xrdcp milleBinaryISN.dat.gz    ${MSSCAFDIR}/binaries/milleBinaryISN.dat.gz  1
+  untilSuccess xrdcp treeFile.root            ${MSSCAFDIR}/tree_files/treeFileISN.root 1
+  untilSuccess xrdcp millePedeMonitorISN.root ${MSSCAFDIR}/monitors/millePedeMonitorISN.root 1
 fi
diff --git a/Alignment/MillePedeAlignmentAlgorithm/templates/mps_runPede_rfcp_template.sh b/Alignment/MillePedeAlignmentAlgorithm/templates/mps_runPede_rfcp_template.sh
@@ -57,48 +57,73 @@ trap clean_up HUP INT TERM SEGV USR2 XCPU XFSZ IO
 
 # a helper function to repeatedly try failing copy commands
 untilSuccess () {
-# trying "${1} ${2} ${3} > /dev/null" until success,
-# break after ${4} tries (with three arguments do up to 5 tries).
-    if  [[ ${#} -lt 3 || ${#} -gt 4 ]]
+# trying "${1} ${2} ${3} > /dev/null" until success, if ${4} is a
+# positive number run {1} with -f flag,
+# break after ${5} tries (with four arguments do up to 5 tries).
+    if  [[ ${#} -lt 4 || ${#} -gt 5 ]]
     then
-        echo ${0} needs 3 or 4 arguments
+        echo ${0} needs 4 or 5 arguments
         return 1
     fi
 
     TRIES=0
     MAX_TRIES=5
-    if [[ ${#} -eq 4 ]]
+    if [[ ${#} -eq 5 ]]
     then
-        MAX_TRIES=${4}
+        MAX_TRIES=${5}
     fi
 
-    ${1} ${2} ${3} > /dev/null
+
+    if [[ ${4} -gt 0 ]]
+    then 
+        ${1} -f ${2} ${3} > /dev/null
+    else 
+        ${1} ${2} ${3} > /dev/null
+    fi
     while [[ ${?} -ne 0 ]]
     do # if not successfull, retry...
         if [[ ${TRIES} -ge ${MAX_TRIES} ]]
         then # ... but not until infinity!
-            echo ${0}: Give up doing \"${1} ${2} ${3} \> /dev/null\".
-            return 1
+            if [[ ${4} -gt 0 ]]
+            then
+                echo ${0}: Give up doing \"${1} -f ${2} ${3} \> /dev/null\".
+                return 1
+            else
+                echo ${0}: Give up doing \"${1} ${2} ${3} \> /dev/null\".
+                return 1
+            fi
         fi
         TRIES=$((${TRIES}+1))
-        echo ${0}: WARNING, problems with \"${1} ${2} ${3} \> /dev/null\", try again.
-        sleep $((${TRIES}*5)) # for before each wait a litte longer...
-        ${1} ${2} ${3} > /dev/null
+        if [[ ${4} -gt 0 ]]
+        then
+            echo ${0}: WARNING, problems with \"${1} -f ${2} ${3} \> /dev/null\", try again.
+            sleep $((${TRIES}*5)) # for before each wait a litte longer...
+            ${1} -f ${2} ${3} > /dev/null
+        else
+            echo ${0}: WARNING, problems with \"${1} ${2} ${3} \> /dev/null\", try again.
+            sleep $((${TRIES}*5)) # for before each wait a litte longer...
+            ${1} ${2} ${3} > /dev/null
+        fi
     done
 
-    echo successsfully executed \"${1} ${2} ${3} \> /dev/null\"
+    if [[ ${4} -gt 0 ]]
+    then
+        echo successfully executed \"${1} -f ${2} ${3} \> /dev/null\"
+    else
+        echo successfully executed \"${1} ${2} ${3} \> /dev/null\"
+    fi
     return 0
 }
 
 copytreefile () {
     CHECKFILE=`echo ${TREEFILELIST} | grep -i ${2}`
     if [[ -z "${TREEFILELIST}" ]]
     then
-        untilSuccess ${1} ${2} ${3}
+        untilSuccess ${1} ${2} ${3} ${4}
     else
         if [[ -n "${CHECKFILE}" ]]
         then
-            untilSuccess ${1} ${2} ${3}
+            untilSuccess ${1} ${2} ${3} ${4}
         fi
     fi
 }
@@ -113,13 +138,13 @@ if [ "${MSSDIRPOOL}" != "cmscafuser" ]; then
   export STAGE_SVCCLASS=${MSSDIRPOOL}
   export STAGER_TRACE=
   echo stager_get -M ${MSSDIR}/milleBinaryISN.dat.gz >> stager_get-commands.txt
-  echo untilSuccess rfcp ${MSSDIR}/milleBinaryISN.dat.gz ${BATCH_DIR} >> parallel-copy-commands.txt
+  echo untilSuccess rfcp ${MSSDIR}/milleBinaryISN.dat.gz ${BATCH_DIR} 0 >> parallel-copy-commands.txt
   echo stager_get -M ${MSSDIR}/treeFileISN.root >> stager_get-commands.txt
-  echo copytreefile rfcp ${MSSDIR}/treeFileISN.root ${BATCH_DIR} >> parallel-copy-commands.txt
+  echo copytreefile rfcp ${MSSDIR}/treeFileISN.root ${BATCH_DIR} 0 >> parallel-copy-commands.txt
 else
   MSSCAFDIR=`echo ${MSSDIR} | perl -pe 's/\/castor\/cern.ch\/cms//gi'`
-  echo untilSuccess xrdcp ${MSSCAFDIR}/binaries/milleBinaryISN.dat.gz milleBinaryISN.dat.gz >> parallel-copy-commands.txt
-  echo copytreefile xrdcp ${MSSCAFDIR}/tree_files/treeFileISN.root treeFileISN.root >> parallel-copy-commands.txt
+  echo untilSuccess xrdcp ${MSSCAFDIR}/binaries/milleBinaryISN.dat.gz milleBinaryISN.dat.gz 1 >> parallel-copy-commands.txt
+  echo copytreefile xrdcp ${MSSCAFDIR}/tree_files/treeFileISN.root treeFileISN.root 1 >> parallel-copy-commands.txt
 fi
 xargs -a stager_get-commands.txt -n 1 -P 10 -I {} bash -c '$@' _ {}
 xargs -a parallel-copy-commands.txt -n 1 -P 10 -I {} bash -c '$@' _ {}