HLT tools: improve support for ConfDB v2 (updated)

HLTrigger/Configuration/common/utils.sh

@@ -0,0 +1,46 @@
+#! /bin/bash
+
+function parse_HLT_schema() {
+  # check input
+  if [ $# != 1 ]; then
+    echo 'parse_HLT_schema: wrong number of parameters' 1>&2
+    return 1
+  fi
+
+  # default values
+  Vx="v2"
+  DB="$1"
+
+  # parse the connection string [version/]database
+  if [[ "$DB" =~ .*/.* ]]; then
+    Vx=`echo "$DB" | cut -d/ -f1`
+    DB=`echo "$DB" | cut -d/ -f2`
+  fi
+
+  echo "$Vx" "$DB"
+}
+
+function parse_HLT_menu() {
+  # check input
+  if [ $# != 1 ]; then
+    echo 'parse_HLT_menu: wrong number of parameters' 1>&2
+    return 1
+  fi
+
+  # default values
+  Vx="v2"
+  DB="offline"
+  MENU="$1"
+
+  # parse the connection string [[version/]database:]menu
+  if [[ "$1" =~ .*:.*  ]]; then
+    MENU=`echo "$1" | cut -d: -f2`
+    DB=`echo "$1" | cut -d: -f1`
+    if [[ "$DB" =~ .*/.* ]]; then
+      Vx=`echo "$DB" | cut -d/ -f1`
+      DB=`echo "$DB" | cut -d/ -f2`
+    fi
+  fi
+
+  echo "$Vx" "$DB" "$MENU"
+}

HLTrigger/Configuration/python/Tools/confdbOfflineConverter.py

@@ -49,7 +49,7 @@ def CheckTempDirectory(dir):
         return dir
 
 
-    def __init__(self, version = 'v2', database = 'hltdev', url = None, verbose = False):
+    def __init__(self, version = 'v2', database = 'offline', url = None, verbose = False):
         self.verbose = verbose
         self.version = version
         self.baseDir = '/afs/cern.ch/user/c/confdb/www/%s/lib' % version
@@ -181,7 +181,7 @@ def help():
 def main():
     args = sys.argv[1:]
     version = 'v2'
-    db      = 'hltdev'
+    db      = 'offline'
     verbose = False
 
     if not args:
@@ -202,7 +202,7 @@ def main():
 
     if '--v1' in args:
         version = 'v1'
-        db      = 'hltdev'
+        db      = 'offline'
         args.remove('--v1')
 
     if '--v2' in args:

HLTrigger/Configuration/scripts/hltGetConfiguration

@@ -17,7 +17,7 @@ formatter = FixedWidthFormatter( HelpFormatterRespectNewlines, width = textwidth
 
 parser = argparse.ArgumentParser(
   description       = 'Extract an HLT configuration (or fragment thereof) from the ConfDB database.''',
-  epilog            = 'Note: using "--online", overriding the GlobalTag or the L1 menu is meaingful only in a full, self-contained dump. A cff file does not provide the modules to be overriden.''',
+  epilog            = 'Note: using "--online", overriding the GlobalTag or the L1 menu is meaningful only in a full, self-contained dump. A cff file does not provide the modules to be overriden.''',
   argument_default  = argparse.SUPPRESS,
   formatter_class   = formatter,
   add_help          = False )
@@ -30,7 +30,7 @@ parser.add_argument('menu',
                     action  = 'store',
                     type    = options.ConnectionHLTMenu,
                     metavar = 'MENU',
-                    help    = 'HLT menu to dump from the database. Supported formats are:\n- [hltdev:]/path/to/configuration[/Vn]\n- orcoff:/path/to/configuration[/Vn]\n- run:runnumber\nThe possible databases are "hltdev" (default, used for offline development) and "orcoff" (used for online menus). If no version is specified, the latest one is automatically used.\nIf "run:" is used instead, the actual HLT menu for the given run number is looked up and used.' )
+                    help    = 'HLT menu to dump from the database. Supported formats are:\n  - /path/to/configuration[/Vn]\n  - [[{v1|v2}/]{offline|online|adg}:]/path/to/configuration[/Vn]\n  - run:runnumber\nThe possible database schemas are "v1" and "v2" (default).\nThe possible databases are "offline" (default, used for offline development), "online" (used to extract online menus within Point 5) and "adg" (used to extract the online menus outside Point 5).\nIf no menu version is specified, the latest one is automatically used.\nIf "run:" is used instead, the HLT menu used for the given run number is looked up and used.' )
 
 # options
 parser.add_argument('--process',

HLTrigger/Configuration/scripts/hltIntegrationTests

@@ -1,5 +1,14 @@
 #! /bin/bash
 
+# load common HLT functions
+if [ -f "$CMSSW_BASE/src/HLTrigger/Configuration/common/utils.sh" ]; then
+  source "$CMSSW_BASE/src/HLTrigger/Configuration/common/utils.sh"
+elif [ -f "$CMSSW_RELEASE_BASE/src/HLTrigger/Configuration/common/utils.sh" ]; then
+  source "$CMSSW_RELEASE_BASE/src/HLTrigger/Configuration/common/utils.sh"
+else
+  exit 1
+fi
+
 function log() {
   echo -e "$@"
 }
@@ -37,12 +46,14 @@ Usage:
 
 
   The supported formats for both menu specifications are:
-    - [hltdev:]/path/to/configuration[/Vn]
-    - orcoff:/path/to/configuration[/Vn]
+    - /path/to/configuration[/Vn]
+    - [[{v1|v2}/]{offline|online|adg}:]/path/to/configuration[/Vn]
     - run:runnumber
-  The possible databases are \"hltdev\" (default, used for offline development) and \"orcoff\"
-  (used for online menus). If no version is specified, the latest one is automatically used.
-  If \"run:\" is used instead, the actual HLT menu for the given run number is looked up and used.
+  The possible database schemas are \"v1\" and \"v2\" (default).
+  The possible databases are \"offline\" (default, used for offline development), \"online\" (used to
+  extract online menus within Point 5) and \"adg\" (used to extract the online menus outside Point 5).
+  If no menu version is specified, the latest one is automatically used.
+  If \"run:\" is used instead, the HLT menu used for the given run number is looked up and used.
 
   It's possible to pass arbitrary command line options to hltGetConfiguration, using \"-x --option\".
   To pass multiple options, enclose them in quotes, or use \"-x\" more than once.
@@ -62,7 +73,7 @@ Exmples:
       will test the latest version of the GRun, running the L1 emulator.
 
 
-  $NAME /users/fwyzard/physics/HLT/V6 -s orcoff:/cdaq/physics/Run2011/1e33/v1.3/HLT/V6
+  $NAME /users/fwyzard/physics/HLT/V6 -s adg:/cdaq/physics/Run2011/1e33/v1.3/HLT/V6
 
       will test the paths from /users/fwyzard/physics/HLT/V6, using the environment from the
       online menu \"1e33\" v1.3 V6
@@ -291,7 +302,8 @@ done
 # if a separate setup is requested, create the setup_cff.py file and patch all dumps to use it
 if [ "$SETUP" ]; then
   log "Extracting setup_cff dump"
-  hltConfigFromDB --cff --configName "$SETUP" --nopaths --services -FUShmDQMOutputService,-PrescaleService > setup_cff.py
+  read SETUP_Vx SETUP_DB SETUP <<< $(parse_HLT_menu "$SETUP")
+  hltConfigFromDB --$SETUP_Vx --$SETUP_DB --cff --configName "$SETUP" --nopaths --services -FUShmDQMOutputService,-PrescaleService > setup_cff.py
   sed -i -e's/process = cms.Process(.*)/&\nprocess.load("setup_cff")/' hlt.py $(for TRIGGER in $TRIGGERS; do echo "$TRIGGER".py; done)
 fi
 

HLTrigger/Configuration/scripts/hltListPaths

@@ -53,7 +53,7 @@ parser.add_argument('menu',
                     action  = 'store', 
                     type    = options.ConnectionHLTMenu,
                     metavar = 'MENU', 
-                    help    = 'HLT menu to dump from the database. Supported formats are:\n- [hltdev:]/path/to/configuration[/Vn]\n- orcoff:/path/to/configuration[/Vn]\n- run:runnumber\nThe possible databases are "hltdev" (default, used for offline development) and "orcoff" (used for online menus). If no version is specified, the latest one is automatically used.\nIf "run:" is used instead, the actual HLT menu for the given run number is looked up and used.' )
+                    help    = 'HLT menu to dump from the database. Supported formats are:\n  - /path/to/configuration[/Vn]\n  - [[{v1|v2}/]{offline|online|adg}:]/path/to/configuration[/Vn]\n  - run:runnumber\nThe possible database schemas are "v1" and "v2" (default).\nThe possible databases are "offline" (default, used for offline development), "online" (used to extract online menus within Point 5) and "adg" (used to extract the online menus outside Point 5).\nIf no menu version is specified, the latest one is automatically used.\nIf "run:" is used instead, the HLT menu used for the given run number is looked up and used.' )
 
 # options
 group = parser.add_mutually_exclusive_group()

HLTrigger/Configuration/tables/makeCosmics

@@ -8,4 +8,4 @@ TARGET="/online/cosmics/2012/7e33/v2.3/HLT"         # where to store the online-
 TABLES="cosmics"
 
 source subtables.sh
-createSubtables "hltdev" "$MASTER" "$TARGET" "$TABLES"
+createSubtables "v2/offline" "$MASTER" "$TARGET" "$TABLES"

HLTrigger/Configuration/tables/makeOnline0T

@@ -9,4 +9,4 @@ TARGET="/online/collisions/2015/25ns_0T/v1.1/HLT"   # where to store the online-
 TABLES="online_0T"
 
 source subtables.sh
-createSubtables "hltdev" "$MASTER" "$TARGET" "$TABLES"
+createSubtables "v2/offline" "$MASTER" "$TARGET" "$TABLES"

HLTrigger/Configuration/tables/makeOnlineGRun

@@ -8,4 +8,4 @@ TARGET="/online/collisions/2016/25ns15e33/v3.1/HLT" # where to store the online-
 TABLES="online_grun"
 
 source subtables.sh
-createSubtables "hltdev" "$MASTER" "$TARGET" "$TABLES"
+createSubtables "v2/offline" "$MASTER" "$TARGET" "$TABLES"

HLTrigger/Configuration/tables/makeOnlineHIon

@@ -8,4 +8,4 @@ TARGET="/online/collisions/2015/HeavyIons/v1.0/HLT" # where to store the online-
 TABLES="online_hion"
 
 source subtables.sh
-createSubtables "hltdev" "$MASTER" "$TARGET" "$TABLES"
+createSubtables "v2/offline" "$MASTER" "$TARGET" "$TABLES"

HLTrigger/Configuration/tables/makeOnlinePIon

@@ -8,4 +8,4 @@ TARGET="/online/collisions/2015/ProtonIon/v1.0/HLT"  # where to store the online
 TABLES="online_pion"
 
 source subtables.sh
-createSubtables "hltdev" "$MASTER" "$TARGET" "$TABLES"
+createSubtables "v2/offline" "$MASTER" "$TARGET" "$TABLES"

HLTrigger/Configuration/tables/makeSubTables

@@ -8,4 +8,4 @@ TARGET="/dev/CMSSW_8_0_0/TABLE"         # directory where to store the sub-table
 TABLES="GRun HIon PIon PRef"                 # which sub-tables to create
 
 source subtables.sh
-createSubtables "hltdev" "$MASTER" "$TARGET" "$TABLES"
+createSubtables "v2/offline" "$MASTER" "$TARGET" "$TABLES"

HLTrigger/Configuration/tables/subtables.sh

@@ -3,48 +3,108 @@
 # utility functions used to generate HLT tables from master table in ConfDB
 #
 
+# load common HLT functions
+if [ -f "$CMSSW_BASE/src/HLTrigger/Configuration/common/utils.sh" ]; then
+  source "$CMSSW_BASE/src/HLTrigger/Configuration/common/utils.sh"
+elif [ -f "$CMSSW_RELEASE_BASE/src/HLTrigger/Configuration/common/utils.sh" ]; then
+  source "$CMSSW_RELEASE_BASE/src/HLTrigger/Configuration/common/utils.sh"
+else
+  exit 1
+fi
+
 CONFDB_TAG="HEAD"
 
+# if set, remove the ConfDB working directory
+private=false
+
 function cleanup() {
   local TABLES="$@"
 
   # clean up
   for TABLE in $TABLES; do
     rm -f "${TABLE}_expanded.txt"
   done
+
+  if $private; then
+    rm -rf $workDir
+  fi
 }
 
 function getPathList() {
-  local DATA=$(hltConfigFromDB --cff --configName $MASTER --noedsources --noes --noservices --nosequences --nomodules)
+  local DATA=$(hltConfigFromDB --$Vx --$DB --cff --configName $MASTER --noedsources --noes --noservices --nosequences --nomodules)
   if echo "$DATA" | grep -q 'Exhausted Resultset\|CONFIG_NOT_FOUND'; then
     echo "Error: $MASTER is not a valid HLT menu"
     exit 1
   fi
   echo "$DATA" | sed -ne's/ *= *cms.\(End\)\?Path.*//p'
 }
 
+function checkJars() {
+  local BASE="$1"; shift
+  local JARS="$@"
+  for F in "$BASE/$JARS"; do
+    [ -f "$F" ] || return 1
+  done
+  return 0
+}
+
 function makeCreateConfig() {
-  # if not already present, check out and build the ConfDB converter
-  if ! [ -d "$CMSSW_BASE/hlt-confdb/.git" ]; then
-    mkdir -p "$CMSSW_BASE/hlt-confdb"
-    git clone "https://github.com/cms-sw/hlt-confdb.git" "$CMSSW_BASE/hlt-confdb" 1>&2
-  fi
-  if ! [ -f "$CMSSW_BASE/hlt-confdb/lib/cmssw-evf-confdb-gui.jar" ]; then
-    ant -f "$CMSSW_BASE/hlt-confdb/build.xml" gui 1>&2
+  local baseDir="/afs/cern.ch/user/c/confdb/www/${Vx}/lib"
+  local baseUrl="http://confdb.web.cern.ch/confdb/${Vx}/lib"
+  local JARS="ojdbc6.jar cmssw-evf-confdb-gui.jar"
+  workDir="$baseDir"
+
+  # try to read the .jar files from AFS, or download them
+  if checkJars "$baseDir" $jars; then
+    # read the .jar fles from AFS
+    workDir="$baseDir"
+  else
+    # try to use $CMSSW_BASE/tmp
+    mkdir -p "$CMSSW_BASE/tmp/confdb"
+    if [ -d "$CMSSW_BASE/tmp/confdb" ]; then
+      workDir="$CMSSW_BASE/tmp/confdb"
+    else
+      workDir=$(mktemp -d confdb.XXXXXXXXXX)
+      private=true
+    fi
+    # download the .jar files
+    for JAR in $JARS; do
+      # check if the file is already present
+      if [ -f $workDir/$JAR ]; then
+        continue
+      fi
+      # download to a temporay file and use an atomic move (in case an other istance is downloading the same file
+      local TMPJAR=$(mktemp -p "$workDir" .${JAR}.XXXXXXXXXX)
+      curl -s "$baseUrl/$JAR" -o "$TMPJAR"
+      mv -n "$TMPJAR" "$workDir/$JAR"
+      rm -f "$TMPJAR"
+    done
   fi
+
+  CLASSPATH=
+  for JAR in $JARS; do
+    CLASSPATH="$CLASSPATH${CLASSPATH:+:}$workDir/$JAR"
+  done
 }
 
 function loadConfiguration() {
-  case "$1" in 
-    hltdev)
-      # hltdev
+  case "$1" in
+    "v1/offline" | "v1/hltdev")
+      # v1 offline aka "hltdev"
       DBHOST="cmsr1-v.cern.ch"
       DBNAME="cms_cond.cern.ch"
       DBUSER="cms_hltdev_writer"
       PWHASH="0196d34dd35b04c0f3597dc89fbbe6e2"
       ;;
+    "v2/offline")
+      # v2 offline
+      DBHOST="cmsr1-v.cern.ch"
+      DBNAME="cms_cond.cern.ch"
+      DBUSER="cms_hlt_gdr_w"
+      PWHASH="0196d34dd35b04c0f3597dc89fbbe6e2"
+      ;;
     *)
-      # see $CMSSW_BASE/hlt-confdb/test/runCreateConfig for other possible settings
+      # see https://github.com/fwyzard/hlt-confdb/blob/confdbv2/test/runCreateConfig
       echo "Error, unnown database \"$1\", exiting."
       exit 1
       ;;
@@ -55,8 +115,10 @@ function runCreateConfig() {
   loadConfiguration "$1"
   java \
     -Djava.security.egd=file:///dev/urandom \
+    -Doracle.jdbc.timezoneAsRegion=false \
+    -Xss32M \
     -Xmx1024m \
-    -classpath "$CMSSW_BASE/hlt-confdb/ext/ojdbc6.jar:$CMSSW_BASE/hlt-confdb/lib/cmssw-evf-confdb-gui.jar" \
+    -classpath "$CLASSPATH" \
     confdb.db.ConfDBCreateConfig \
     --dbHost $DBHOST \
     --dbName $DBNAME \
@@ -109,6 +171,11 @@ function createSubtables() {
   local TARGET="$1";   shift
   local TABLES="$@"
 
+  # extract the schema version from the database name
+  local Vx DB
+  read Vx DB <<< $(parse_HLT_schema "$DATABASE")
+  local DATABASE="${Vx}/${DB}"
+
   # dump the requested configuration
   echo "ConfDB master: $DATABASE:$MASTER"
   echo "Subtables:     $TABLES"
@@ -129,7 +196,7 @@ function createSubtables() {
   fi
 
   # ask the user for the database password
-  readPassword $DATABASE
+  readPassword
 
   # make sure the needed sripts are available
   makeCreateConfig