Merge pull request #11828 from hroskes/no-cmsStage-76X

No more cmsStage etc. scripts

Alignment/OfflineValidation/python/TkAlAllInOneTool/configTemplates.py

@@ -38,21 +38,23 @@
 #ulimit -v 3072000
 #export STAGE_SVCCLASS=cmscafuser
 #save path to the LSF batch working directory  (/pool/lsf)
+
 export LSFWORKDIR=`pwd -P`
 echo LSF working directory is $LSFWORKDIR
 source /afs/cern.ch/cms/caf/setup.sh
+eos='/afs/cern.ch/project/eos/installation/cms/bin/eos.select'
 cd .oO[CMSSW_BASE]Oo./src
 export SCRAM_ARCH=.oO[SCRAM_ARCH]Oo.
 eval `scramv1 ru -sh`
 #rfmkdir -p .oO[datadir]Oo. &>! /dev/null
 
 #remove possible result file from previous runs
-previous_results=$(cmsLs -l /store/caf/user/$USER/.oO[eosdir]Oo. | awk '{print $5}')
+previous_results=$($eos ls /store/caf/user/$USER/.oO[eosdir]Oo.)
 for file in ${previous_results}
 do
     if [ ${file} = /store/caf/user/$USER/.oO[eosdir]Oo./.oO[outputFile]Oo. ]
     then
-        cmsStage -f ${file} ${file}.bak
+        xrdcp -f root://eoscms//eos/cms${file} root://eoscms//eos/cms${file}.bak
     fi
 done
 
@@ -86,7 +88,7 @@
 find . -maxdepth 1 -name "LOGFILE*.oO[alignmentName]Oo.*" -print | xargs -I {} bash -c "rfcp {} .oO[logdir]Oo."
 
 #copy root files to eos
-cmsMkdir /store/caf/user/$USER/.oO[eosdir]Oo.
+$eos mkdir -p /store/caf/user/$USER/.oO[eosdir]Oo.
 if [ .oO[parallelJobs]Oo. -eq 1 ]
 then
     root_files=$(ls --color=never -d *.oO[alignmentName]Oo.*.root)
@@ -97,7 +99,7 @@
 
 for file in ${root_files}
 do
-    cmsStage -f ${file} /store/caf/user/$USER/.oO[eosdir]Oo.
+    xrdcp -f ${file} root://eoscms//eos/cms/store/caf/user/$USER/.oO[eosdir]Oo.
     echo ${file}
 done
 
@@ -114,6 +116,7 @@
 ######################################################################
 mergeTemplate="""
 #!/bin/bash
+eos='/afs/cern.ch/project/eos/installation/cms/bin/eos.select'
 CWD=`pwd -P`
 cd .oO[CMSSW_BASE]Oo./src
 export SCRAM_ARCH=.oO[SCRAM_ARCH]Oo.
@@ -134,13 +137,13 @@
 
 ###############################################################################
 # download root files from eos
-root_files=$(cmsLs -l /store/caf/user/$USER/.oO[eosdir]Oo. | awk '{print $5}' \
+root_files=$($eos ls /store/caf/user/$USER/.oO[eosdir]Oo. \
              | grep ".root$" | grep -v "result.root$")
-for file in ${root_files}
-do
-    cmsStage -f ${file} .
-    # echo ${file}
-done
+#for file in ${root_files}
+#do
+#    xrdcp -f root://eoscms//eos/cms/store/caf/user/$USER/.oO[eosdir]Oo./${file} .
+#    echo ${file}
+#done
 
 
 #run
@@ -181,7 +184,7 @@
 compareAlignmentsExecution="""
 #merge for .oO[validationId]Oo. if it does not exist or is not up-to-date
 echo -e "\n\nComparing validations"
-cmsMkdir /store/caf/user/$USER/.oO[eosdir]Oo./
+$eos mkdir -p /store/caf/user/$USER/.oO[eosdir]Oo./
 cp .oO[CMSSW_BASE]Oo./src/Alignment/OfflineValidation/scripts/compareFileAges.C .
 root -x -q -b -l "compareFileAges.C(\\\"root://eoscms.cern.ch//eos/cms/store/caf/user/$USER/.oO[eosdir]Oo./.oO[validationId]Oo._result.root\\\", \\\".oO[compareStringsPlain]Oo.\\\")"
 comparisonNeeded=${?}
@@ -191,10 +194,10 @@
     cp .oO[CMSSW_BASE]Oo./src/Alignment/OfflineValidation/scripts/compareAlignments.cc .
     root -x -q -b -l 'compareAlignments.cc++(\".oO[compareStrings]Oo.\")'
     mv result.root .oO[validationId]Oo._result.root
-    cmsStage -f .oO[validationId]Oo._result.root /store/caf/user/$USER/.oO[eosdir]Oo.
+    xrdcp -f .oO[validationId]Oo._result.root root://eoscms//eos/cms/store/caf/user/$USER/.oO[eosdir]Oo.
 else
     echo ".oO[validationId]Oo._result.root is up-to-date, no need to compare again."
-    cmsStage -f /store/caf/user/$USER/.oO[eosdir]Oo./.oO[validationId]Oo._result.root .
+    xrdcp -f root://eoscms//eos/cms/store/caf/user/$USER/.oO[eosdir]Oo./.oO[validationId]Oo._result.root .
 fi
 """
 

Alignment/OfflineValidation/python/TkAlAllInOneTool/geometryComparison.py

@@ -11,7 +11,7 @@ class GeometryComparison(GenericValidation):
     Object representing a geometry comparison job.
     """
     def __init__( self, valName, alignment, referenceAlignment,
-                  config, copyImages = True, randomWorkdirPart = None):
+                  config, copyImages = True):
         """
         Constructor of the GeometryComparison class.
 
@@ -24,9 +24,6 @@ def __init__( self, valName, alignment, referenceAlignment,
                     configuration of the validations
         - `copyImages`: Boolean which indicates whether png- and pdf-files
                         should be copied back from the batch farm
-        - `randomWorkDirPart`: If this option is ommitted a random number is
-                               generated to create unique path names for the
-                               individual validation instances.
         """
 	defaults = {
 	    "3DSubdetector1":"1",
@@ -37,8 +34,6 @@ def __init__( self, valName, alignment, referenceAlignment,
         GenericValidation.__init__(self, valName, alignment, config, 
 				   "compare", addDefaults=defaults, 
 				   addMandatories = mandatories)
-        if not randomWorkdirPart == None:
-            self.randomWorkdirPart = randomWorkdirPart
         self.referenceAlignment = referenceAlignment
         referenceName = "IDEAL"
         if not self.referenceAlignment == "IDEAL":
@@ -82,14 +77,14 @@ def getRepMap(self, alignment = None):
     def createConfiguration(self, path ):
         # self.__compares
         repMap = self.getRepMap()
-        cfgFileName = "TkAlCompareToNTuple.%s.%s_cfg.py"%(
-            self.alignmentToValidate.name, self.randomWorkdirPart)
+        cfgFileName = "TkAlCompareToNTuple.%s_cfg.py"%(
+            self.alignmentToValidate.name)
         cfgs = {cfgFileName: configTemplates.intoNTuplesTemplate}
         repMaps = {cfgFileName: repMap}
         if not self.referenceAlignment == "IDEAL":
             referenceRepMap = self.getRepMap( self.referenceAlignment )
-            cfgFileName = "TkAlCompareToNTuple.%s.%s_cfg.py"%(
-                self.referenceAlignment.name, self.randomWorkdirPart )
+            cfgFileName = "TkAlCompareToNTuple.%s_cfg.py"%(
+                self.referenceAlignment.name )
             cfgs[cfgFileName] = configTemplates.intoNTuplesTemplate
             repMaps[cfgFileName] = referenceRepMap
 
@@ -230,8 +225,9 @@ def createScript(self, path):
                                               ".oO[name]Oo..root"%name), repMap)
                 resultingFile = os.path.expandvars( resultingFile )
                 resultingFile = os.path.abspath( resultingFile )
+                resultingFile = "root://eoscms//eos/cms" + resultingFile   #needs to be AFTER abspath so that it doesn't eat the //
                 repMap["runComparisonScripts"] += \
-                    ("cmsStage -f OUTPUT_comparison.root %s\n"
+                    ("xrdcp -f OUTPUT_comparison.root %s\n"
                      %resultingFile)
                 self.filesToCompare[ name ] = resultingFile
 

Alignment/OfflineValidation/python/TkAlAllInOneTool/preexistingValidation.py

@@ -42,6 +42,7 @@ def __init__(self, valName, config, valType,
         config.checkInput("preexisting"+valType+":"+self.name,
                           knownSimpleOptions = knownOpts,
                           ignoreOptions = ignoreOpts)
+        self.jobmode = None
 
     def getRepMap(self):
         result = self.general
@@ -74,7 +75,7 @@ def getCompareStrings( self, requestId = None, plain = False ):
     def createFiles(self, *args, **kwargs):
         raise AllInOneError("Shouldn't be here...")
     def createConfiguration(self, *args, **kwargs):
-        raise AllInOneError("Shouldn't be here...")
+        pass
     def createScript(self, *args, **kwargs):
         raise AllInOneError("Shouldn't be here...")
     def createCrabCfg(self, *args, **kwargs):
@@ -88,7 +89,7 @@ def __init__(self, valName, config,
             "DMRMinimum":"30",
             "DMROptions":"",
             "OfflineTreeBaseDir":"TrackHitFilter",
-            "SurfaceShapes":"none"
+            "SurfaceShapes":"coarse",
             }
         defaults.update(addDefaults)
         PreexistingValidation.__init__(self, valName, config, "offline",

Alignment/OfflineValidation/python/TkAlAllInOneTool/zMuMuValidation.py

@@ -24,6 +24,8 @@ def __init__(self, valName, alignment, config,
         GenericValidationData.__init__(self, valName, alignment, config,
                                        "zmumu", addDefaults=defaults,
                                        addMandatories=mandatories)
+        if self.general["zmumureference"].startswith("/store"):
+            self.general["zmumureference"] = "root://eoscms//eos/cms" + self.general["zmumureference"]
         if self.NJobs > 1:
             raise AllInOneError("Parallel jobs not implemented for the Z->mumu validation!\n"
                                 "Please set parallelJobs = 1.")
@@ -42,12 +44,16 @@ def createCrabCfg(self, path):
         return GenericValidationData.createCrabCfg(self, path, self.crabCfgBaseName)
 
     def getRepMap(self, alignment = None):
+        if alignment == None:
+            alignment = self.alignmentToValidate
         repMap = GenericValidationData.getRepMap(self, alignment) 
         repMap.update({
             "nEvents": self.general["maxevents"],
-#             "outputFile": "zmumuHisto.root"
             "outputFile": ("0_zmumuHisto.root"
                            ",genSimRecoPlots.root"
-                           ",FitParameters.txt")
+                           ",FitParameters.txt"),
+            "eosdir": os.path.join(self.general["eosdir"], "%s/%s/%s" % (self.outputBaseName, self.name, alignment.name)),
+            "workingdir": ".oO[datadir]Oo./%s/%s/%s" % (self.outputBaseName, self.name, alignment.name),
+            "plotsdir": ".oO[datadir]Oo./%s/%s/%s/plots" % (self.outputBaseName, self.name, alignment.name),
                 })
         return repMap

Alignment/OfflineValidation/python/TkAlAllInOneTool/zMuMuValidationTemplates.py

@@ -14,16 +14,16 @@
 .oO[datasetDefinition]Oo.
 # process.load("Alignment.OfflineValidation..oO[dataset]Oo._cff")
 
-process.load("Geometry.CMSCommonData.cmsIdealGeometryXML_cfi")
-process.load("Geometry.CommonDetUnit.globalTrackingGeometry_cfi")
+#process.load("Geometry.CMSCommonData.cmsIdealGeometryXML_cfi")
+#process.load("Geometry.CommonDetUnit.globalTrackingGeometry_cfi")
 process.load("RecoMuon.DetLayers.muonDetLayerGeometry_cfi")
 process.load("Geometry.MuonNumbering.muonNumberingInitialization_cfi")
 process.load("RecoMuon.TrackingTools.MuonServiceProxy_cff")
 
 ########### standard includes ##############################
 process.load("Configuration.StandardSequences..oO[magneticField]Oo._cff")
 process.load("Configuration.StandardSequences.Reconstruction_cff")
-process.load("Configuration.Geometry.GeometryDB_cff")
+process.load("Configuration.Geometry.GeometryRecoDB_cff")
 
 
 ########### DATABASE conditions ############################
@@ -114,21 +114,29 @@
     parSmear = cms.vdouble(),
 
     ### taken from J/Psi #########################
-    ResolFitType = cms.int32(14), 
-    parResol = cms.vdouble(0.007,0.015, -0.00077, 0.0063, 0.0018, 0.0164),
-    parResolFix = cms.vint32(0, 0, 0,0, 0,0),
-    parResolOrder = cms.vint32(0, 0, 0, 0, 0, 0),
+#    ResolFitType = cms.int32(14), 
+#    parResol = cms.vdouble(0.007,0.015, -0.00077, 0.0063, 0.0018, 0.0164),
+#    parResolFix = cms.vint32(0, 0, 0,0, 0,0),
+#    parResolOrder = cms.vint32(0, 0, 0, 0, 0, 0),
+    ResolFitType = cms.int32(0), 
+    parResol = cms.vdouble(0),
+    parResolFix = cms.vint32(0),
+    parResolOrder = cms.vint32(0),
 
 
     # -------------------- #
     # Scale fit parameters #
     # -------------------- #
 
     # -----------------------------------------------------------------------------------
-    ScaleFitType = cms.int32(18),
-    parScaleOrder = cms.vint32(0, 0, 0, 0),
-    parScaleFix =   cms.vint32(0, 0, 0, 0),
-    parScale = cms.vdouble(1, 1, 1, 1),
+#    ScaleFitType = cms.int32(18),
+#    parScaleOrder = cms.vint32(0, 0, 0, 0),
+#    parScaleFix =   cms.vint32(0, 0, 0, 0),
+#    parScale = cms.vdouble(1, 1, 1, 1),
+    ScaleFitType = cms.int32(0),
+    parScaleOrder = cms.vint32(0),
+    parScaleFix =   cms.vint32(0),
+    parScale = cms.vdouble(0),
 
 
     
@@ -211,20 +219,20 @@
 zMuMuScriptTemplate="""
 #!/bin/bash 
 source /afs/cern.ch/cms/caf/setup.sh
+eos='/afs/cern.ch/project/eos/installation/cms/bin/eos.select'
 
 echo  -----------------------
 echo  Job started at `date`
 echo  -----------------------
 
 cwd=`pwd`
 cd .oO[CMSSW_BASE]Oo./src
-# export SCRAM_ARCH=slc5_amd64_gcc462
 export SCRAM_ARCH=.oO[SCRAM_ARCH]Oo.
 eval `scram runtime -sh`
 cd $cwd
 
 rfmkdir -p .oO[datadir]Oo.
-
+rfmkdir -p .oO[workingdir]Oo.
 rfmkdir -p .oO[logdir]Oo.
 rm -f .oO[logdir]Oo./*.stdout
 rm -f .oO[logdir]Oo./*.stderr
@@ -244,11 +252,6 @@
 
 ls -lh . 
 
-source /afs/cern.ch/sw/lcg/external/gcc/4.8.1/x86_64-slc6/setup.sh
-source /afs/cern.ch/sw/lcg/app/releases/ROOT/5.34.10/x86_64-slc6-gcc48-opt/root/bin/thisroot.sh
-
-# cd .oO[CMSSW_BASE]Oo./src/MuonAnalysis/MomentumScaleCalibration/test/Macros/RooFit
-# ln -fs .oO[workdir]Oo./0_zmumuHisto.root .
 cp .oO[CMSSW_BASE]Oo./src/MuonAnalysis/MomentumScaleCalibration/test/Macros/RooFit/CompareBiasZValidation.cc .
 cp .oO[CMSSW_BASE]Oo./src/MuonAnalysis/MomentumScaleCalibration/test/Macros/RooFit/Legend.h .
 cp .oO[CMSSW_BASE]Oo./src/MuonAnalysis/MomentumScaleCalibration/test/Macros/RooFit/FitMassSlices.cc .
@@ -257,26 +260,25 @@
 cp .oO[CMSSW_BASE]Oo./src/MuonAnalysis/MomentumScaleCalibration/test/Macros/RooFit/FitWithRooFit.cc .
 cp .oO[CMSSW_BASE]Oo./src/MuonAnalysis/MomentumScaleCalibration/test/Macros/RooFit/FitMass1D.cc .
 
-root -q -b "CompareBiasZValidation.cc+(\\\"\\\")"
+root -q -b -l "CompareBiasZValidation.cc+()"
 
- 
-# mv BiasCheck.root .oO[workdir]Oo. 
-
-# cd .oO[workdir]Oo.
 cp  .oO[CMSSW_BASE]Oo./src/MuonAnalysis/MomentumScaleCalibration/test/Macros/RooFit/tdrstyle.C .
 cp  .oO[CMSSW_BASE]Oo./src/MuonAnalysis/MomentumScaleCalibration/test/Macros/RooFit/MultiHistoOverlap_.oO[resonance]Oo..C .
-# ln -fs /afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN2/TMP_EM/ZMuMu/data/MC/BiasCheck_DYToMuMu_Summer11_TkAlZMuMu_IDEAL.root  ./BiasCheck_Reference.root
-if [[ .oO[zmumureference]Oo. == *store* ]]; then cmsStage -f .oO[zmumureference]Oo. BiasCheck_Reference.root; else ln -fs .oO[zmumureference]Oo. ./BiasCheck_Reference.root; fi
-root -q -b MultiHistoOverlap_.oO[resonance]Oo..C
 
-cmsMkdir /store/caf/user/$USER/.oO[eosdir]Oo.
+if [[ .oO[zmumureference]Oo. == *store* ]]; then xrdcp -f .oO[zmumureference]Oo. BiasCheck_Reference.root; else ln -fs .oO[zmumureference]Oo. ./BiasCheck_Reference.root; fi
+root -q -b -l MultiHistoOverlap_.oO[resonance]Oo..C
+
+$eos mkdir -p /store/caf/user/$USER/.oO[eosdir]Oo./plots/
 for RootOutputFile in $(ls *root )
 do
-    cmsStage -f ${RootOutputFile}  /store/caf/user/$USER/.oO[eosdir]Oo./
+    xrdcp -f ${RootOutputFile}  root://eoscms//eos/cms/store/caf/user/$USER/.oO[eosdir]Oo./
+    rfcp ${RootOutputFile}  .oO[workingdir]Oo.
 done
 
+mkdir -p .oO[plotsdir]Oo.
 for PngOutputFile in $(ls *png ); do
-    rfcp ${PngOutputFile}  .oO[datadir]Oo.
+    xrdcp -f ${PngOutputFile}  root://eoscms//eos/cms/store/caf/user/$USER/.oO[eosdir]Oo./plots/
+    rfcp ${PngOutputFile}  .oO[plotsdir]Oo.
 done