Merge pull request #24952 from muhammadansariqbal/CMSSW_10_3_X_EOSMig…

…ration

Tracker Alignment - Fix to conddb function in validation and migration to common EOS storage

Alignment/OfflineValidation/python/TkAlAllInOneTool/betterConfigParser.py

@@ -157,10 +157,12 @@ def getGeneral( self ):
             "jobmode":"interactive",
             "datadir":os.getcwd(),
             "logdir":os.getcwd(),
-            "eosdir": "",
             }
-        self.checkInput("general", knownSimpleOptions = defaults.keys())
-        general = self.getResultingSection( "general", defaultDict = defaults )
+	mandatories = [
+            "eosdir",
+        ]
+        self.checkInput("general", knownSimpleOptions = defaults.keys() + mandatories)
+        general = self.getResultingSection( "general", defaultDict = defaults, demandPars = mandatories )
         internal_section = "internals"
         if not self.has_section(internal_section):
             self.add_section(internal_section)

Alignment/OfflineValidation/python/TkAlAllInOneTool/configTemplates.py

@@ -59,10 +59,10 @@
 #rfmkdir -p .oO[datadir]Oo. &>! /dev/null
 
 #remove possible result file from previous runs
-previous_results=$(ls /eos/cms/store/caf/user/$USER/.oO[eosdir]Oo.)
+previous_results=$(ls /eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo.)
 for file in ${previous_results}
 do
-    if [ ${file} = /eos/cms/store/caf/user/$USER/.oO[eosdir]Oo./.oO[outputFile]Oo. ]
+    if [ ${file} = /eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./.oO[outputFile]Oo. ]
     then
         xrdcp -f root://eoscms//eos/cms${file} root://eoscms//eos/cms${file}.bak
     fi
@@ -98,7 +98,7 @@
 find . -maxdepth 1 -name "LOGFILE*.oO[alignmentName]Oo.*" -print | xargs -I {} bash -c "rfcp {} .oO[logdir]Oo."
 
 #copy root files to eos
-mkdir -p /eos/cms/store/caf/user/$USER/.oO[eosdir]Oo.
+mkdir -p /eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo.
 if [ .oO[parallelJobs]Oo. -eq 1 ]
 then
     root_files=$(ls --color=never -d *.oO[alignmentName]Oo.*.root)
@@ -109,7 +109,7 @@
 
 for file in ${root_files}
 do
-    xrdcp -f ${file} root://eoscms//eos/cms/store/caf/user/$USER/.oO[eosdir]Oo.
+    xrdcp -f ${file} root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo.
     echo ${file}
 done
 
@@ -237,11 +237,11 @@
 
 ###############################################################################
 # download root files from eos
-root_files=$(ls /eos/cms/store/caf/user/$USER/.oO[eosdir]Oo. \
+root_files=$(ls /eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo. \
              | grep ".root$" | grep -v "result.root$")
 #for file in ${root_files}
 #do
-#    xrdcp -f root://eoscms//eos/cms/store/caf/user/$USER/.oO[eosdir]Oo./${file} .
+#    xrdcp -f root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./${file} .
 #    echo ${file}
 #done
 
@@ -284,11 +284,11 @@
 
 ###############################################################################
 # download root files from eos
-root_files=$(ls /eos/cms/store/caf/user/$USER/.oO[eosdir]Oo. \
+root_files=$(ls /eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo. \
              | grep ".root$" | grep -v "result.root$")
 #for file in ${root_files}
 #do
-#    xrdcp -f root://eoscms//eos/cms/store/caf/user/$USER/.oO[eosdir]Oo./${file} .
+#    xrdcp -f root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./${file} .
 #    echo ${file}
 #done
 
@@ -326,20 +326,20 @@
 compareAlignmentsExecution="""
 #merge for .oO[validationId]Oo. if it does not exist or is not up-to-date
 echo -e "\n\nComparing validations"
-mkdir -p /eos/cms/store/caf/user/$USER/.oO[eosdir]Oo./
+mkdir -p /eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./
 cp .oO[Alignment/OfflineValidation]Oo./scripts/compareFileAges.C .
-root -x -q -b -l "compareFileAges.C(\\\"root://eoscms.cern.ch//eos/cms/store/caf/user/$USER/.oO[eosdir]Oo./.oO[validationId]Oo._result.root\\\", \\\".oO[compareStringsPlain]Oo.\\\")"
+root -x -q -b -l "compareFileAges.C(\\\"root://eoscms.cern.ch//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./.oO[validationId]Oo._result.root\\\", \\\".oO[compareStringsPlain]Oo.\\\")"
 comparisonNeeded=${?}
 
 if [[ ${comparisonNeeded} -eq 1 ]]
 then
     cp .oO[compareAlignmentsPath]Oo. .
     root -x -q -b -l '.oO[compareAlignmentsName]Oo.++(\".oO[compareStrings]Oo.\", ".oO[legendheader]Oo.", ".oO[customtitle]Oo.", ".oO[customrighttitle]Oo.", .oO[bigtext]Oo.)'
     mv result.root .oO[validationId]Oo._result.root
-    xrdcp -f .oO[validationId]Oo._result.root root://eoscms//eos/cms/store/caf/user/$USER/.oO[eosdir]Oo.
+    xrdcp -f .oO[validationId]Oo._result.root root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo.
 else
     echo ".oO[validationId]Oo._result.root is up-to-date, no need to compare again."
-    xrdcp -f root://eoscms//eos/cms/store/caf/user/$USER/.oO[eosdir]Oo./.oO[validationId]Oo._result.root .
+    xrdcp -f root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./.oO[validationId]Oo._result.root .
 fi
 """
 

Alignment/OfflineValidation/python/TkAlAllInOneTool/genericValidation.py

@@ -398,7 +398,7 @@ def getRepMap(self, alignment = None):
         outputfile = os.path.expandvars(replaceByMap(
                            "%s_%s_.oO[name]Oo..root" % (self.outputBaseName, self.name)
                                  , result))
-        resultfile = os.path.expandvars(replaceByMap(("/store/caf/user/$USER/.oO[eosdir]Oo./" +
+        resultfile = os.path.expandvars(replaceByMap(("/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./" +
                            "%s_%s_.oO[name]Oo..root" % (self.resultBaseName, self.name))
                                  , result))
         result.update({

Alignment/OfflineValidation/python/TkAlAllInOneTool/geometryComparison.py

@@ -257,7 +257,7 @@ def createScript(self, path):
                         "-c \"rfcp {} .oO[datadir]Oo./.oO[name]Oo."
                         ".Comparison_common"+name+"_Images/.oO[common]Oo._.oO[name]Oo..Visualization.gif\"\n")
 
-                resultingFile = replaceByMap(("/store/caf/user/$USER/.oO[eosdir]Oo./compared%s_"
+                resultingFile = replaceByMap(("/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./compared%s_"
                                               ".oO[name]Oo..root"%name), repMap)
                 resultingFile = os.path.expandvars( resultingFile )
                 resultingFile = os.path.abspath( resultingFile )

Alignment/OfflineValidation/python/TkAlAllInOneTool/helperFunctions.py

@@ -195,6 +195,7 @@ def cppboolstring(string, name):
     """
     return pythonboolstring(string, name).lower()
 
+conddbcode = None
 def conddb(*args):
     """
     Wrapper for conddb, so that you can run
@@ -204,29 +205,36 @@ def conddb(*args):
     getcommandoutput2(conddb ...) doesn't work, it imports
     the wrong sqlalchemy in CondCore/Utilities/python/conddblib.py
     """
-    from tempfile import NamedTemporaryFile
+    global conddbcode
+    from tempfile import mkdtemp, NamedTemporaryFile
 
-    with open(getCommandOutput2("which conddb").strip()) as f:
-        conddb = f.read()
+    if conddbcode is None:
+        conddbfile = getCommandOutput2("which conddb").strip()
+        tmpdir = mkdtemp()
+        getCommandOutput2("2to3 -f print -o " + tmpdir + " -n -w " + conddbfile)
+
+        with open(os.path.join(tmpdir, "conddb")) as f:
+            conddb = f.read()
+
+        conddbcode = conddb.replace("sys.exit", "sysexit")
 
     def sysexit(number):
         if number != 0:
             raise AllInOneError("conddb exited with status {}".format(number))
     namespace = {"sysexit": sysexit, "conddboutput": ""}
 
-    conddb = conddb.replace("sys.exit", "sysexit")
-
     bkpargv = sys.argv
     sys.argv[1:] = args
     bkpstdout = sys.stdout
-    with NamedTemporaryFile(bufsize=0) as sys.stdout:
-        exec(conddb, namespace)
-        namespace["main"]()
-        with open(sys.stdout.name) as f:
-            result = f.read()
-
-    sys.argv[:] = bkpargv
-    sys.stdout = bkpstdout
+    try:
+        with NamedTemporaryFile(bufsize=0) as sys.stdout:
+            exec(conddbcode, namespace)
+            namespace["main"]()
+            with open(sys.stdout.name) as f:
+                result = f.read()
+    finally:
+        sys.argv[:] = bkpargv
+        sys.stdout = bkpstdout
 
     return result
 

Alignment/OfflineValidation/python/TkAlAllInOneTool/offlineValidationTemplates.py

@@ -108,7 +108,7 @@
   p.setTreeBaseDir(".oO[OfflineTreeBaseDir]Oo.");
   p.plotDMR(".oO[DMRMethod]Oo.",.oO[DMRMinimum]Oo.,".oO[DMROptions]Oo.");
   p.plotSurfaceShapes(".oO[SurfaceShapes]Oo.");
-  p.plotChi2("root://eoscms//eos/cms/store/caf/user/$USER/.oO[eosdir]Oo./.oO[validationId]Oo._result.root");
+  p.plotChi2("root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./.oO[validationId]Oo._result.root");
   vector<int> moduleids = {.oO[moduleid]Oo.};
   for (auto moduleid : moduleids) {
   	p.residual_by_moduleID(moduleid);

Alignment/OfflineValidation/python/TkAlAllInOneTool/primaryVertexValidationTemplates.py

@@ -202,10 +202,10 @@
 
 ls -lh .
 
-eos mkdir -p /store/caf/user/$USER/.oO[eosdir]Oo./plots/
+eos mkdir -p /store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./plots/
 for RootOutputFile in $(ls *root )
 do
-    xrdcp -f ${RootOutputFile} root://eoscms//eos/cms/store/caf/user/$USER/.oO[eosdir]Oo./${RootOutputFile}
+    xrdcp -f ${RootOutputFile} root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./${RootOutputFile}
     rfcp ${RootOutputFile}  .oO[workingdir]Oo.
 done
 
@@ -219,12 +219,12 @@
 
 mkdir -p .oO[plotsdir]Oo.
 for PngOutputFile in $(ls *png ); do
-    xrdcp -f ${PngOutputFile}  root://eoscms//eos/cms/store/caf/user/$USER/.oO[eosdir]Oo./plots/${PngOutputFile}
+    xrdcp -f ${PngOutputFile}  root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./plots/${PngOutputFile}
     rfcp ${PngOutputFile}  .oO[plotsdir]Oo.
 done
 
 for PdfOutputFile in $(ls *pdf ); do
-    xrdcp -f ${PdfOutputFile}  root://eoscms//eos/cms/store/caf/user/$USER/.oO[eosdir]Oo./plots/${PdfOutputFile}
+    xrdcp -f ${PdfOutputFile}  root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./plots/${PdfOutputFile}
     rfcp ${PdfOutputFile}  .oO[plotsdir]Oo.
 done
 
@@ -292,12 +292,12 @@
 root -x -b -q .oO[plottingscriptname]Oo.++
 
 for PdfOutputFile in $(ls *pdf ); do
-    xrdcp -f ${PdfOutputFile}  root://eoscms//eos/cms/store/caf/user/$USER/.oO[eosdir]Oo./plots/${PdfOutputFile}
+    xrdcp -f ${PdfOutputFile}  root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./plots/${PdfOutputFile}
     rfcp ${PdfOutputFile}  .oO[datadir]Oo./.oO[PlotsDirName]Oo.
 done
 
 for PngOutputFile in $(ls *png ); do
-    xrdcp -f ${PngOutputFile}  root://eoscms//eos/cms/store/caf/user/$USER/.oO[eosdir]Oo./plots/${PngOutputFile}
+    xrdcp -f ${PngOutputFile}  root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./plots/${PngOutputFile}
     rfcp ${PngOutputFile}  .oO[datadir]Oo./.oO[PlotsDirName]Oo.
 done
 

Alignment/OfflineValidation/python/TkAlAllInOneTool/zMuMuValidation.py

@@ -108,7 +108,7 @@ def appendToPlots(self):
         returned, else the validation is appended to the list
         """
         repMap = self.getRepMap()
-        return replaceByMap('    filenames.push_back("root://eoscms//eos/cms/store/caf/user/$USER/.oO[eosdir]Oo./BiasCheck.root");  titles.push_back(".oO[title]Oo.");  colors.push_back(.oO[color]Oo.);  linestyles.push_back(.oO[style]Oo.);\n', repMap)
+        return replaceByMap('    filenames.push_back("root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./BiasCheck.root");  titles.push_back(".oO[title]Oo.");  colors.push_back(.oO[color]Oo.);  linestyles.push_back(.oO[style]Oo.);\n', repMap)
 
     @classmethod
     def plottingscriptname(cls):

Alignment/OfflineValidation/python/TkAlAllInOneTool/zMuMuValidationTemplates.py

@@ -249,16 +249,16 @@
 if [[ .oO[zmumureference]Oo. == *store* ]]; then xrdcp -f .oO[zmumureference]Oo. BiasCheck_Reference.root; else ln -fs .oO[zmumureference]Oo. ./BiasCheck_Reference.root; fi
 root -q -b -l MultiHistoOverlap_.oO[resonance]Oo..C
 
-eos mkdir -p /store/caf/user/$USER/.oO[eosdir]Oo./plots/
+eos mkdir -p /store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./plots/
 for RootOutputFile in $(ls *root )
 do
-    xrdcp -f ${RootOutputFile}  root://eoscms//eos/cms/store/caf/user/$USER/.oO[eosdir]Oo./
+    xrdcp -f ${RootOutputFile}  root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./
     rfcp ${RootOutputFile}  .oO[workingdir]Oo.
 done
 
 mkdir -p .oO[plotsdir]Oo.
 for PngOutputFile in $(ls *png ); do
-    xrdcp -f ${PngOutputFile}  root://eoscms//eos/cms/store/caf/user/$USER/.oO[eosdir]Oo./plots/
+    xrdcp -f ${PngOutputFile}  root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./plots/
     rfcp ${PngOutputFile}  .oO[plotsdir]Oo.
 done
 

Alignment/OfflineValidation/scripts/validateAlignments.py

@@ -360,7 +360,7 @@ def createMergeScript( path, validations, options ):
                 repMapTemp["doMerge"] += '\n\n\n\necho -e "\n\nMerging results from %s jobs with alignment %s"\n\n' % (validationType.valType,validation.alignmentToValidate.name)
                 repMapTemp["doMerge"] += validation.doMerge()
                 for f in validation.getRepMap()["outputFiles"]:
-                    longName = os.path.join("/eos/cms/store/caf/user/$USER/",
+                    longName = os.path.join("/eos/cms/store/group/alca_trackeralign/AlignmentValidation/",
                                             validation.getRepMap()["eosdir"], f)
                     repMapTemp["rmUnmerged"] += "    rm "+longName+"\n"
 
@@ -397,7 +397,7 @@ def createMergeScript( path, validations, options ):
                     repMap["beforeMerge"] += validationType.doInitMerge()
                 repMap["doMerge"] += validation.doMerge()
                 for f in validation.getRepMap()["outputFiles"]:
-                    longName = os.path.join("/eos/cms/store/caf/user/$USER/",
+                    longName = os.path.join("/eos/cms/store/group/alca_trackeralign/AlignmentValidation/",
                                             validation.getRepMap()["eosdir"], f)
                     repMap["rmUnmerged"] += "    rm "+longName+"\n"