Skip to content

Commit

Permalink
Merge pull request #22831 from ggovi/conddb-utilities-o2o-steering-fi…
Browse files Browse the repository at this point in the history 
…les-10-1-X

Steering scripts for O2Os
  • Loading branch information
@cmsbuild
cmsbuild committed Apr 13, 2018
2 parents c542b0a + 5fb1804 commit 08c852c
Show file tree
Hide file tree
Showing 48 changed files with 1,000 additions and 0 deletions.
79 changes: 79 additions & 0 deletions CondCore/Utilities/o2o/deploy_offline.sh
View file
@@ -0,0 +1,79 @@
#!/bin/sh

home=~
localhome=/data/condbpro
root=/data/O2O
cmsswroot=/cvmfs/cms.cern.ch/
extroot=/data/ext
release=CMSSW_10_0_5
arch=slc6_amd64_gcc630
sourceroot=CondCore/Utilities/o2o/templates
source=$cmsswroot/$arch/cms/cmssw/$release/src/$sourceroot

files=( setup.sh
setStrip.sh
SiStripDCS.sh
ecal_laser.sh
EcalLaser.sh
EcalLaserTest.sh
EcalLaser_express.sh
EcalLaser_expressTest.sh )

folders=( EcalLaser
EcalLaserTest
EcalLaser_express
EcalLaser_expressTest
SiStrip )

cd $root
if [ ! -d scripts ]; then
mkdir scripts
fi
if [ ! -d logs ]; then
mkdir logs
fi

sed_fmt () {
var=$(echo $1 | sed -e "s#/#\\\/#g")
}

replace_params () {
params=( @root
@home
@cmsswroot
@extroot )
tgt_file=$1
var=''
# replace path params
sed_fmt $root
tgt_root=$var
sed -i -e s/@root/$tgt_root/g $tgt_file
sed_fmt $cmsswroot
tgt_cmsswroot=$var
sed -i -e s/@cmsswroot/$tgt_cmsswroot/g $tgt_file
sed_fmt $extroot
tgt_extroot=$var
sed -i -e s/@extroot/$tgt_extroot/g $tgt_file
sed_fmt $home
tgt_home=$var
# relace non-path params
sed -i -e s/@home/$tgt_home/g $tgt_file
sed -i -e s/@release/$release/g $tgt_file
sed -i -e s/@arch/$arch/g $tgt_file
}

for file in "${files[@]}"
do
cp $source/$file scripts/
tgt_file=scripts/$file
replace_params $tgt_file
done
for f in "${folders[@]}"
do
if [ ! -d $f ]; then
mkdir -p $f
fi
if [ ! -d logs/$f ]; then
mkdir logs/$f
fi
done
89 changes: 89 additions & 0 deletions CondCore/Utilities/o2o/deploy_online.sh
View file
@@ -0,0 +1,89 @@
#!/bin/sh

home=~
localhome=/data/popconpro
root=/data/O2O
cmsswroot=/opt/offline
extroot=/data/ext
release=CMSSW_10_0_5
arch=slc7_amd64_gcc630
sourceroot=CondCore/Utilities/o2o/templates
source=$cmsswroot/$arch/cms/cmssw/$release/src/$sourceroot

files=( setup.sh
runStart.sh
runTestStart.sh
runStop.sh
runTestStop.sh
EcalDAQ.sh
EcalDAQTest.sh
EcalDCS.sh
EcalDCSTest.sh
EcalTPG.sh
RunInfoStart.sh
RunInfoStop.sh
RunInfoStartTest.sh
RunInfoStopTest.sh )

folders=( EcalDAQ
EcalDAQTest
EcalDCS
EcalDCSTest
EcalTPG
RunInfoStart
RunInfoStop
RunInfoStartTest
RunInfoStopTest )

cd $root
if [ ! -d scripts ]; then
mkdir scripts
fi
if [ ! -d logs ]; then
mkdir logs
fi

sed_fmt () {
var=$(echo $1 | sed -e "s#/#\\\/#g")
}

replace_params () {
params=( @root
@home
@cmsswroot
@extroot )
tgt_file=$1
var=''
# replace path params
sed_fmt $root
tgt_root=$var
sed -i -e s/@root/$tgt_root/g $tgt_file
sed_fmt $cmsswroot
tgt_cmsswroot=$var
sed -i -e s/@cmsswroot/$tgt_cmsswroot/g $tgt_file
sed_fmt $extroot
tgt_extroot=$var
sed -i -e s/@extroot/$tgt_extroot/g $tgt_file
sed_fmt $home
tgt_home=$var
# relace non-path params
sed -i -e s/@home/$tgt_home/g $tgt_file
sed -i -e s/@release/$release/g $tgt_file
sed -i -e s/@arch/$arch/g $tgt_file
}

for file in "${files[@]}"
do
cp $source/$file scripts/
tgt_file=scripts/$file
replace_params $tgt_file
done
for f in "${folders[@]}"
do
if [ ! -d $f ]; then
mkdir -p $f
fi
if [ ! -d logs/$f ]; then
mkdir logs/$f
fi
done
3 changes: 3 additions & 0 deletions CondCore/Utilities/o2o/deployed/offline/EcalLaser.sh
View file
@@ -0,0 +1,3 @@
source /data/O2O/scripts/setupO2O.sh -s Ecal -j Laser
SRCDIR=$RELEASEDIR/src/CondTools/Ecal/python
submit_popCon EcalLaser $SRCDIR/EcalLaser_prompt_popcon.py
3 changes: 3 additions & 0 deletions CondCore/Utilities/o2o/deployed/offline/EcalLaserTest.sh
View file
@@ -0,0 +1,3 @@
source /data/O2O/scripts/setupO2O.sh -s Ecal -j LaserTest
SRCDIR=$RELEASEDIR/src/CondTools/Ecal/python
submit_popCon EcalLaserTest $SRCDIR/EcalLaser_prompt_popcon.py
3 changes: 3 additions & 0 deletions CondCore/Utilities/o2o/deployed/offline/EcalLaser_express.sh
View file
@@ -0,0 +1,3 @@
source /data/O2O/scripts/setupO2O.sh -s Ecal -j Laser_express
SRCDIR=$RELEASEDIR/src/CondTools/Ecal/python
submit_popCon EcalLaser_express $SRCDIR/EcalLaser_express_popcon.py
3 changes: 3 additions & 0 deletions CondCore/Utilities/o2o/deployed/offline/EcalLaser_expressTest.sh
View file
@@ -0,0 +1,3 @@
source /data/O2O/scripts/setupO2O_new.sh -s Ecal -j Laser_expressTest
SRCDIR=$RELEASEDIR/src/CondTools/Ecal/python
submit_test_popCon EcalLaser_expressTest $SRCDIR/EcalLaser_express_popcon.py
4 changes: 4 additions & 0 deletions CondCore/Utilities/o2o/deployed/offline/SiStripDCS.crontab
View file
@@ -0,0 +1,4 @@
5 * * * * /data/O2O/scripts/SiStripDCS.sh SiStripDetVOff_1hourDelay
20 * * * * /data/O2O/scripts/SiStripDCS.sh SiStripDetVOff_13hourDelay
40 * * * * /data/O2O/scripts/SiStripDCS.sh SiStripDetVOff_prompt
45 * * * * /data/O2O/scripts/SiStripDCS.sh SiStripDetVOff_sampling_test
18 changes: 18 additions & 0 deletions CondCore/Utilities/o2o/deployed/offline/SiStripDCS.sh
View file
@@ -0,0 +1,18 @@
#!/bin/sh
#
# wrapper script to run DCS O2O
# Last updated: Apr 21, 2017
# Author: Huilin Qu
#
# Usage: SiStripDCS.sh JOB_NAME

JOBNAME=$1

O2O_HOME=/data/O2O/

source $O2O_HOME/scripts/setStripO2O.sh $JOBNAME
o2oRun_SiStripDCS.py $JOBNAME |tee -a $LOGFILE

# Exit with status of last command
# Make sure the last command is o2oRun_SiStripDCS.py!
exit $?
1 change: 1 addition & 0 deletions CondCore/Utilities/o2o/deployed/offline/runTimeBasedO2O.crontab
View file
@@ -0,0 +1 @@
0 */2 * * * /bin/sh /data/O2O/scripts/runTimeBasedO2O.sh >/dev/null 2>&1
26 changes: 26 additions & 0 deletions CondCore/Utilities/o2o/deployed/offline/runTimeBasedO2O.sh
View file
@@ -0,0 +1,26 @@
#!/bin/bash
####### ------ beginning -------- #######################
BASEDIR=/data/O2O
LOGFILE=${BASEDIR}/logs/TimeBasedO2O.log
DATE=`date`
echo " " | tee -a $LOGFILE
echo "----- new cronjob started for Time Based O2O at -----" | tee -a $LOGFILE
echo "$DATE" | tee -a $LOGFILE

#Check if the exportation scripts are running, if so exits without launching them
PID_0=`ps aux | grep '/bin/sh /data/O2O/scripts/EcalLaser.sh' | grep -v grep | awk '{print $2}'`
if [ "${PID_0}" ]
then
echo "Ecal Laser Exportation script still running with pid ${PID_0}: exiting" | tee -a $LOGFILE
exit 1
else
/bin/sh /data/O2O/scripts/EcalLaser.sh
fi
PID_1=`ps aux | grep '/bin/sh /data/O2O/scripts/EcalLaser_express.sh' | grep -v grep | awk '{print $2}'`
if [ "${PID_1}" ]
then
echo "Ecal LaserExpress Exportation script still running with pid ${PID_1}: exiting" | tee -a $LOGFILE
exit 1
else
/bin/sh /data/O2O/scripts/EcalLaser_express.sh
fi
33 changes: 33 additions & 0 deletions CondCore/Utilities/o2o/deployed/offline/setStripO2O.sh
View file
@@ -0,0 +1,33 @@
#!/bin/sh
#
# set the CMSSW environtmet for SiStripO2O
# and the proxies for conditionUploader
#

O2ONAME=$1

# deployment-specific params
BASEDIR=/data/O2O
SCRAM_ARCH=slc6_amd64_gcc630
RELEASE=CMSSW_10_0_5
RELEASEDIR=/cvmfs/cms.cern.ch/slc6_amd64_gcc630/cms/cmssw/${RELEASE}

source /cvmfs/cms.cern.ch/cmsset_default.sh
cd ${RELEASEDIR}/src
eval `scramv1 ru -sh`
cd -

O2O_HOME=/data/O2O/SiStrip/

# for sqlalchmey (?)
export PYTHON_EGG_CACHE=/data/condbpro
# path to .netrc file and .cms_cond dir
export COND_AUTH_PATH=/data/O2O/SiStrip

# save log files produced by o2oRun.py on disk
export JOBDIR=${O2O_HOME}/jobs/${O2ONAME}
export O2O_LOG_FOLDER=${BASEDIR}/logs/${O2ONAME}
export LOGFILE=${BASEDIR}/logs/$O2ONAME.log

# temperoray fix for TNS_ADMIN
export TNS_ADMIN=/cvmfs/cms.cern.ch/slc6_amd64_gcc530/cms/oracle-env/29/etc
100 changes: 100 additions & 0 deletions CondCore/Utilities/o2o/deployed/offline/setupO2O.sh
View file
@@ -0,0 +1,100 @@
#!/bin/sh

# deployment-specific params
BASEDIR=/data/O2O
RELEASE=CMSSW_10_0_5
RELEASEDIR=/cvmfs/cms.cern.ch/slc6_amd64_gcc630/cms/cmssw/${RELEASE}

# command params
OPTIND=1

SUBSYS=""
JOBNAME=""

while getopts "h?s:j:" opt; do
case $opt in
h|\?)
echo "Mo' to spiego..."
exit 0
;;
s) SUBSYS=$OPTARG
;;
j) JOBNAME=$OPTARG
;;
esac
done

shift $((OPTIND-1))

[ "$1" = "--" ] && shift

O2ONAME=$SUBSYS$JOBNAME
#echo "name=$O2ONAME, subsystem=$SUBSYS, job=$JOBNAME"

# o2o specific params
LOGFILE=${BASEDIR}/logs/$O2ONAME.log
JOBDIR=${BASEDIR}/${SUBSYS}/${JOBNAME}
DATE=`date`

# functions
function logRun(){
echo "----- new job started for $1 at -----" | tee -a $LOGFILE
echo $DATE | tee -a $LOGFILE
}

function log() {
echo "[`date`] : $@ " | tee -a $LOGFILE
}

function submit_command() {
logRun $1
o2o run -n $1 "$2" | tee -a $LOGFILE
}

function submit_test_command() {
logRun $1
o2o --db dev run -n $1 "$2" | tee -a $LOGFILE
}

function submit_cmsRun() {
COMMAND="cmsRun $2 destinationDatabase={db} destinationTag={tag}"
logRun $1
o2o run -n $1 "$COMMAND" | tee -a $LOGFILE
}

function submit_test_cmsRun() {
COMMAND="cmsRun $2 destinationDatabase={db} destinationTag={tag}"
logRun $1
o2o --db dev run -n $1 "$COMMAND" | tee -a $LOGFILE
}

function submit_popCon() {
COMMAND="popconRun $2 -d {db} -t {tag} -c"
logRun $1
o2o run -n $1 "$COMMAND" | tee -a $LOGFILE
}

function submit_test_popCon() {
COMMAND="popconRun $2 -d {db} -t {tag} -c"
logRun $1
o2o --db dev run -n $1 "$COMMAND" | tee -a $LOGFILE
}

# global variables
export PYTHON_EGG_CACHE=/data/condbpro
export SCRAM_ARCH=slc6_amd64_gcc630
export O2O_LOG_FOLDER=/data/O2O/logs/${O2ONAME}
export COND_AUTH_PATH=$BASEDIR
source /cvmfs/cms.cern.ch/cmsset_default.sh

cd ${RELEASEDIR}/src
eval `scramv1 run -sh`
# set up OCCI workaround
export LD_PRELOAD=$CMS_ORACLEOCCI_LIB

# workaround for oracle tnsnames
export TNS_ADMIN=/cvmfs/cms.cern.ch/slc6_amd64_gcc530/cms/oracle-env/29/etc

cd ${JOBDIR}


3 changes: 3 additions & 0 deletions CondCore/Utilities/o2o/deployed/online/EcalDAQ.sh
View file
@@ -0,0 +1,3 @@
source /data/O2O/scripts/setupO2O.sh -s Ecal -j DAQ
SRCDIR=$RELEASEDIR/src/CondTools/Ecal/python
submit_popCon EcalDAQ $SRCDIR/EcalDAQ_popcon.py
3 changes: 3 additions & 0 deletions CondCore/Utilities/o2o/deployed/online/EcalDAQTest.sh
View file
@@ -0,0 +1,3 @@
source /data/O2O/scripts/setupO2O.sh -s Ecal -j DAQTest
SRCDIR=$RELEASEDIR/src/CondTools/Ecal/python
submit_popCon EcalDAQTest $SRCDIR/EcalDAQ_popcon.py
3 changes: 3 additions & 0 deletions CondCore/Utilities/o2o/deployed/online/EcalDCS.sh
View file
@@ -0,0 +1,3 @@
source /data/O2O/scripts/setupO2O.sh -s Ecal -j DCS
SRCDIR=$RELEASEDIR/src/CondTools/Ecal/python
submit_popCon EcalDCS $SRCDIR/EcalDCS_popcon.py
3 changes: 3 additions & 0 deletions CondCore/Utilities/o2o/deployed/online/EcalDCSTest.sh
View file
@@ -0,0 +1,3 @@
source /data/O2O/scripts/setupO2O.sh -s Ecal -j DCSTest
SRCDIR=$RELEASEDIR/src/CondTools/Ecal/python
submit_popCon EcalDCSTest $SRCDIR/EcalDCS_popcon.py

0 comments on commit 08c852c

Please sign in to comment.