Merge pull request #22095 from hqucms/sistrip-o2o-test

Add unit tests for SiStrip DAQ and DCS O2O.

CondTools/SiStrip/python/SiStripDCS_popcon.py

@@ -1,20 +1,21 @@
-import socket
 import datetime
 import os
 import FWCore.ParameterSet.Config as cms
 import FWCore.ParameterSet.VarParsing as VarParsing
 
-sourceConnection = 'oracle://cms_omds_adg/CMS_TRK_R'
-if socket.getfqdn().find('.cms') != -1:
-    sourceConnection = 'oracle://cms_omds_lb/CMS_TRK_R'
-
 options = VarParsing.VarParsing()
 options.register('delay'
                 , 1  # default value
                 , VarParsing.VarParsing.multiplicity.singleton
                 , VarParsing.VarParsing.varType.int
                 , "Time delay (in hours) for the O2O. The O2O then queries the PVSS DB from last IOV until (current hour - delay), ignoring minutes and seconds."
                   )
+options.register('sourceConnection'
+                , 'oracle://cms_omds_adg/CMS_TRK_R'  # default value
+                , VarParsing.VarParsing.multiplicity.singleton
+                , VarParsing.VarParsing.varType.string
+                , "Connection string to the PVSS DB."
+                  )
 options.register('destinationConnection'
                 , 'sqlite_file:SiStripDetVOff.db'  # default value
                 , VarParsing.VarParsing.multiplicity.singleton
@@ -61,7 +62,7 @@
 # -----------------------------------------------------------------------------
 process.SiStripDetVOffBuilder = cms.Service(
     "SiStripDetVOffBuilder",
-    onlineDB = cms.string(sourceConnection),
+    onlineDB=cms.string(options.sourceConnection),
     authPath=cms.string(authPath),
 
     # Format for date/time vector:  year, month, day, hour, minute, second, nanosecond      

CondTools/SiStrip/python/SiStripO2O_cfg_template.py

@@ -24,6 +24,11 @@
 _CFGLINES_
 )
 
+if 'CONFDB' not in os.environ:
+    import CondCore.Utilities.credentials as auth
+    user, _, passwd = auth.get_credentials('COND_AUTH_PATH', 'cms_omds_tunnel/cms_trk_r')
+    process.SiStripConfigDb.ConfDb = '{user}/{passwd}@{path}'.format(user=user, passwd=passwd, path='cms_omds_tunnel')
+
 process.SiStripDetInfoFileReader = cms.Service("SiStripDetInfoFileReader")
 
 process.load("OnlineDB.SiStripO2O.SiStripO2OCalibrationFactors_cfi")

CondTools/SiStrip/scripts/SiStripDCSPopCon.py

@@ -32,8 +32,8 @@ def runjob(args):
         os.remove(output_db)
 
     # run cmssw job: raise error if failed
-    command = 'cmsRun {cfg} delay={delay} destinationConnection={destFile} conddbConnection={conddb} tag={tag}'.format(
-        cfg=cfg, delay=args.delay, destFile='sqlite:///%s' % output_db, conddb=args.condDbRead, tag=args.inputTag)
+    command = 'cmsRun {cfg} delay={delay} destinationConnection={destFile} sourceConnection={sourceDb} conddbConnection={conddb} tag={tag}'.format(
+        cfg=cfg, delay=args.delay, destFile='sqlite:///%s' % output_db, sourceDb=args.sourceDb, conddb=args.condDbRead, tag=args.inputTag)
     pipe = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
     atexit.register(partial(helper.kill_subproc_noexcept, pipe))
     out = pipe.communicate()[0]
@@ -69,6 +69,7 @@ def main():
     parser = argparse.ArgumentParser(description='Run a single O2O job for SiStrip DCS and upload the payloads to condition database.')
     parser.add_argument('--delay', required=True, help='Time delay (in hours) for the O2O. The O2O then queries the PVSS DB from last IOV until (current hour - delay), ignoring minutes and seconds.')
     parser.add_argument('--destTags', required=True, help='Destination tag name(s) for upload. Use comma to separate multiple values.')
+    parser.add_argument('--sourceDb', required=True, help='Connection string for the source database.')
     parser.add_argument('--destDb', required=True, help='Destination DB to upload.')
     parser.add_argument('--inputTag', required=True, help='Tag name to be used in the sqlite file.')
     parser.add_argument('--condDbRead', default='oracle://cms_orcon_adg/CMS_CONDITIONS', help='Connection string for the DB from which the fast O2O retrives payloads.')

CondTools/SiStrip/scripts/o2oRun_SiStripDCS.py

@@ -28,6 +28,7 @@ def run(args):
     o2ocmd += ' --destTags {destTags}'
     o2ocmd += ' --destDb {destDb}'
     o2ocmd += ' --inputTag {inputTag}'
+    o2ocmd += ' --sourceDb {sourceDb}'
     o2ocmd += ' --condDbRead {condDbRead}'
     if args.debug:
         o2ocmd += ' --debug'

CondTools/SiStrip/test/BuildFile.xml

@@ -0,0 +1,4 @@
+<architecture name="slc.*_amd64_.*">
+  <test name="SiStripDCS_O2O_test" command="testSiStripDCSO2O.sh"/>
+  <test name="SiStripDAQ_O2O_test" command="testSiStripDAQO2O.sh"/>
+</architecture>

CondTools/SiStrip/test/testSiStripDAQO2O.sh

@@ -0,0 +1,85 @@
+#!/bin/sh
+function die { echo $1: status $2 ; exit $2; }
+
+iov=308698
+tag="SiStripBadChannel_FromOnline_GR10_v1_hlt"
+
+# set up jobdir
+# O2O runs under $JOBDIR/{since}/{analyzer}
+export JOBDIR=`pwd`
+outputdir="$JOBDIR/$iov/SiStripO2OBadStrip"
+if [ -d "$outputdir" ]; then
+	rm -r $outputdir
+fi
+mkdir -p $outputdir
+
+# config file corresponding to $iov
+cfgfile="$JOBDIR/sistrip-daq-test-cfg.txt"
+cat << EOF > $cfgfile
+        PartTIBD= cms.untracked.PSet(
+                PartitionName = cms.untracked.string("TI_27-JAN-2010_2"),
+                ForceCurrentState = cms.untracked.bool(False),
+                ForceVersions = cms.untracked.bool(True),
+                CablingVersion = cms.untracked.vuint32(74,0),
+                FecVersion = cms.untracked.vuint32(900,0),
+                FedVersion = cms.untracked.vuint32(1521,0),
+                DcuDetIdsVersion = cms.untracked.vuint32(9,12),
+                MaskVersion = cms.untracked.vuint32(125,0),
+                DcuPsuMapVersion = cms.untracked.vuint32(273,0)
+        ),
+        PartTOB= cms.untracked.PSet(
+                PartitionName = cms.untracked.string("TO_30-JUN-2009_1"),
+                ForceCurrentState = cms.untracked.bool(False),
+                ForceVersions = cms.untracked.bool(True),
+                CablingVersion = cms.untracked.vuint32(73,0),
+                FecVersion = cms.untracked.vuint32(901,0),
+                FedVersion = cms.untracked.vuint32(1520,0),
+                DcuDetIdsVersion = cms.untracked.vuint32(9,12),
+                MaskVersion = cms.untracked.vuint32(120,0),
+                DcuPsuMapVersion = cms.untracked.vuint32(274,0)
+        ),
+        PartTECP= cms.untracked.PSet(
+                PartitionName = cms.untracked.string("TP_09-JUN-2009_1"),
+                ForceCurrentState = cms.untracked.bool(False),
+                ForceVersions = cms.untracked.bool(True),
+                CablingVersion = cms.untracked.vuint32(71,1),
+                FecVersion = cms.untracked.vuint32(899,0),
+                FedVersion = cms.untracked.vuint32(1522,0),
+                DcuDetIdsVersion = cms.untracked.vuint32(9,0),
+                MaskVersion = cms.untracked.vuint32(118,0),
+                DcuPsuMapVersion = cms.untracked.vuint32(266,1)
+        ),
+        PartTECM= cms.untracked.PSet(
+                PartitionName = cms.untracked.string("TM_09-JUN-2009_1"),
+                ForceCurrentState = cms.untracked.bool(False),
+                ForceVersions = cms.untracked.bool(True),
+                CablingVersion = cms.untracked.vuint32(69,1),
+                FecVersion = cms.untracked.vuint32(898,0),
+                FedVersion = cms.untracked.vuint32(1523,0),
+                DcuDetIdsVersion = cms.untracked.vuint32(9,0),
+                MaskVersion = cms.untracked.vuint32(124,0),
+                DcuPsuMapVersion = cms.untracked.vuint32(267,1)
+        )
+EOF
+
+# export the reference payload from prod db
+refdb="${tag}_ref.db"
+if [ -f "$refdb" ]; then
+	rm  $refdb
+fi
+conddb_import -f frontier://FrontierProd/CMS_CONDITIONS -c sqlite:$refdb -i $tag -t $tag -b $iov -e $iov --reserialize
+
+# run DAQ O2O test
+SiStripDAQPopCon.py SiStripO2OBadStrip $iov $cfgfile --destTags SiStripBadChannel_FromOnline_GR10_v1_hlt --destDb None --inputTag SiStripBadChannel_FromOnline_GR10_v1_hlt --condDbRead frontier://FrontierProd/CMS_CONDITIONS --no-upload --bookkeeping-db private || die "Failure running SiStripDAQPopCon.py" $?
+
+# compare the new payload with the reference payload
+tagdiff=$( conddb --db $refdb diff --destdb $outputdir/SiStripO2OBadStrip_$iov.db -s $tag )
+if [ $? -ne 0 ]; then
+	die "DAQO2OTest: Cannot compare the tags!" $?
+fi
+if [ $( printf $tagdiff | grep -c $iov ) -ne "0" ]; then
+	printf $tagdiff
+	die "DAQO2OTest: Payload hash does not match!" 1
+fi
+
+exit

CondTools/SiStrip/test/testSiStripDCSO2O.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+function die { echo $1: status $2 ; exit $2; }
+
+# set up jobdir
+# O2O runs under $JOBDIR/{delay}hourDelay
+export JOBDIR=`pwd`
+outputdir="$JOBDIR/1hourDelay"
+if [ -d "$outputdir" ]; then
+	rm -r $outputdir
+fi
+mkdir -p $outputdir
+
+# copy the second to last IOV
+conddb --yes copy SiStripDetVOff_1hourDelay_v1_Validation --destdb $outputdir/SiStripDetVOff_1hourDelay_O2OTEST.db --o2oTest
+# run a test DCS O2O
+SiStripDCSPopCon.py --delay 1 --destTags SiStripDetVOff_1hourDelay_v1_Validation --destDb None --inputTag SiStripDetVOff_1hourDelay_v1_Validation --sourceDb oracle://cms_omds_adg/CMS_TRK_R --condDbRead sqlite:///$outputdir/SiStripDetVOff_1hourDelay_O2OTEST.db --no-upload || die "Failure running SiStripDCSPopCon.py" $?
+# check if new IOV is produced
+conddb --db $outputdir/SiStripDetVOff_1.db list SiStripDetVOff_1hourDelay_v1_Validation || die "No new IOV produced" $?