add unit test to check availability of EDM files used in HLT tests

HLTrigger/Configuration/test/BuildFile.xml

@@ -0,0 +1,2 @@
+<!-- test access to EDM files used in HLT-AddOnTests and HLT-Validation tests -->
+<test name="test_AccessToEDMInputsOfHLTTests" command="testAccessToEDMInputsOfHLTTests.sh"/>

HLTrigger/Configuration/test/cmsDriver.csh

@@ -12,6 +12,14 @@ eval `scram runtime -csh`
 #  > export CMS_PATH="/cvmfs/cms-ib.cern.ch"
 #  > export SITECONFIG_PATH="/cvmfs/cms-ib.cern.ch/SITECONF/local"
 #
+# IMPORTANT:
+#  when the names of the EDM files below are changed, make sure to execute the script
+#
+#  > ./HLTrigger/Configuration/test/testAccessToEDMInputsOfHLTTests_update_filelist.sh
+#
+#  to update the list of files to be cached in the IB-EOS area
+#  (see the readme in this directory for further details)
+#
 
 #
 # GEN-SIM input files for Monte-Carlo tests

HLTrigger/Configuration/test/readme.md

@@ -0,0 +1,40 @@
+The `test/` directory of the `HLTrigger/Configuration` package contains
+
+ - scripts to copy HLT menus from the `ConfDB` database into CMSSW,
+   as both `cff` fragments (loadable via `cmsDriver.py`) and standalone `cfg` configurations (usable with `cmsRun`);
+
+ - scripts to run tests with these HLT menus
+   (a version of these tests runs in CMSSW integration buils as the so-called "HLT-Validation" tests);
+
+ - a unit test to verify the availability of the EDM input files used in the HLT tests maintained in CMSSW by the Trigger Studies Group
+   (see `test_AccessToEDMInputsOfHLTTests` below).
+
+_Important_ : when the names of the EDM files hard-coded
+in `HLTrigger/Configuration/test/cmsDriver.csh` and/or `Configuration/HLT/python/addOnTestsHLT.py` are changed,
+make sure to execute the script `HLTrigger/Configuration/test/testAccessToEDMInputsOfHLTTests.sh`
+to update the list of files to be cached in the IB-EOS area (see `test_AccessToEDMInputsOfHLTTests` below).
+
+---
+
+Unit test: `test_AccessToEDMInputsOfHLTTests`
+---
+
+This unit test executes `cmsRun` jobs to verify the availability
+of the EDM files listed in `HLTrigger/Configuration/test/testAccessToEDMInputsOfHLTTests_filelist.txt`.
+
+To run the unit test via `scram`, execute
+```bash
+scram b runtests_test_AccessToEDMInputsOfHLTTests
+```
+To run the unit test locally, execute
+```bash
+LOCALTOP="${CMSSW_BASE}" "${CMSSW_BASE}"/src/HLTrigger/Configuration/test/testAccessToEDMInputsOfHLTTests.sh
+```
+
+The unit test does not modify the content of the file `testAccessToEDMInputsOfHLTTests_filelist.txt`.
+The latter can be updated by manually executing the script `testAccessToEDMInputsOfHLTTests_update_filelist.sh`.
+The file `testAccessToEDMInputsOfHLTTests_filelist.txt` lists
+the Logical File Name (LFN) of the EDM files used in HLT tests
+for the main CMSSW development branches (name format: `CMSSW_[0-9]*_[0-9]*_X`).
+The list includes only EDM files which are available
+either in the IB-EOS cache at the CERN T2, or at any T2/T3 site.

HLTrigger/Configuration/test/testAccessToEDMInputsOfHLTTests.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# Pass in name and status
+function die {
+  echo $1: status $2
+  echo === Log file ===
+  cat ${3:-/dev/null}
+  echo === End log file ===
+  exit $2
+}
+
+# run test job
+TESTDIR="${LOCALTOP}"/src/HLTrigger/Configuration/test
+
+inputFileList="${TESTDIR}"/testAccessToEDMInputsOfHLTTests_filelist.txt
+
+if [ ! -f "${inputFileList}" ]; then
+  printf "%s\n" "ERROR -- invalid path to file listing EDM input files: ${inputFileList}"
+  exit 1
+fi
+
+LOGFILE=log_testAccessToEDMInputsOfHLTTests
+
+rm -f "${LOGFILE}"
+for inputFile in $(cat "${inputFileList}"); do
+  cmsRun "${TESTDIR}"/testAccessToEDMInputsOfHLTTests_cfg.py inputFiles="${inputFile}" &>> "${LOGFILE}" \
+    || die "Failure running testAccessToEDMInputsOfHLTTests_cfg.py" $? "${LOGFILE}"
+done
+unset inputFile

HLTrigger/Configuration/test/testAccessToEDMInputsOfHLTTests_cfg.py

@@ -0,0 +1,20 @@
+import FWCore.ParameterSet.Config as cms
+
+## VarParsing
+import FWCore.ParameterSet.VarParsing as VarParsing
+options = VarParsing.VarParsing('analysis')
+options.setDefault('maxEvents', 1)
+options.parseArguments()
+
+process = cms.Process('TEST')
+
+process.options.numberOfThreads = 1
+process.options.numberOfStreams = 0
+process.options.wantSummary = False
+process.maxEvents.input = options.maxEvents
+
+## Source
+process.source = cms.Source('PoolSource',
+  fileNames = cms.untracked.vstring(options.inputFiles),
+  inputCommands = cms.untracked.vstring('drop *')
+)

HLTrigger/Configuration/test/testAccessToEDMInputsOfHLTTests_filelist.txt

@@ -0,0 +1,26 @@
+/store/data/Run2012A/MuEG/RAW/v1/000/191/718/14932935-E289-E111-830C-5404A6388697.root
+/store/data/Run2015D/MuonEG/RAW/v1/000/256/677/00000/80950A90-745D-E511-92FD-02163E011C5D.root
+/store/data/Run2016B/JetHT/RAW/v1/000/272/762/00000/C666CDE2-E013-E611-B15A-02163E011DBE.root
+/store/data/Run2017A/HLTPhysics4/RAW/v1/000/295/606/00000/36DE5E0A-3645-E711-8FA1-02163E01A43B.root
+/store/data/Run2018D/EphemeralHLTPhysics1/RAW/v1/000/323/775/00000/2E066536-5CF2-B340-A73B-209640F29FF6.root
+/store/data/Run2018D/HIMinimumBias0/RAW/v1/000/325/112/00000/660F62BB-9932-D645-A4A4-0BBBDA3963E8.root
+/store/data/Run2022B/HLTPhysics/RAW/v1/000/355/456/00000/69b26b27-4bd1-4524-bc18-45f7b9b5e076.root
+/store/group/dpg_trigger/comm_trigger/TriggerStudiesGroup/STORM/GEN-SIM/CMSSW_5/143C21CD-E8A2-E311-87BE-0025904C66E8.root
+/store/group/dpg_trigger/comm_trigger/TriggerStudiesGroup/STORM/GEN-SIM/CMSSW_7/72B2C6B5-9740-E411-8FF7-002618FDA26D.root
+/store/group/dpg_trigger/comm_trigger/TriggerStudiesGroup/STORM/GEN-SIM/CMSSW_7/8C172C6B-28FD-E311-8A62-0026189438F4.root
+/store/hidata/HIRun2011/HIHighPt/RAW/v1/000/182/838/F20AAF66-F71C-E111-9704-BCAEC532971D.root
+/store/hidata/HIRun2015/HIHardProbes/RAW-RECO/HighPtJet-PromptReco-v1/000/263/689/00000/1802CD9A-DDB8-E511-9CF9-02163E0138CA.root
+/store/hidata/HIRun2015/HIHardProbes/RAW/v1/000/263/718/00000/08057733-02A5-E511-9C7D-02163E014606.root
+/store/hidata/HIRun2018A/HIHardProbes/RAW/v1/000/326/479/00000/0E2CC5D5-9D87-7348-9219-B00CD718C847.root
+/store/hidata/HIRun2022A/HITestRaw0/RAW/v1/000/362/321/00000/f467ee64-fc64-47a6-9d8a-7ca73ebca2bd.root
+/store/relval/CMSSW_10_3_0_pre5/RelValZEEMM_13_HI/GEN-SIM/103X_upgrade2018_realistic_v7-v1/10000/E288668E-A2D1-D446-A401-D71EA43DD796.root
+/store/relval/CMSSW_11_2_0_pre8/RelValTTbar_13/GEN-SIM/112X_mcRun3_2021_design_v10-v1/00000/3ee9ba1e-0ef8-4242-8343-cff886c9f7b3.root
+/store/relval/CMSSW_11_2_0_pre8/RelValZEE_14_HI_2021/GEN-SIM/112X_mcRun3_2021_realistic_HI_v11-v1/00000/65e018bc-2a25-4f53-b9cf-aba35a7b212d.root
+/store/relval/CMSSW_12_3_0_pre6/RelValTTbar_14TeV/GEN-SIM/123X_mcRun3_2021_realistic_v11-v1/10000/29297788-21c3-4bf2-afe0-5382f08fbcd5.root
+/store/relval/CMSSW_12_3_0_pre6/RelValZEE_14_HI_2021/GEN-SIM/123X_mcRun3_2021_realistic_HI_v11-v2/10000/3c102f6f-4b70-4b09-b646-504bc104414d.root
+/store/relval/CMSSW_5_3_16/RelValPyquen_ZeemumuJets_pt10_2760GeV/GEN-SIM/PU_STARTHI53_LV1_mar03-v2/00000/143C21CD-E8A2-E311-87BE-0025904C66E8.root
+/store/relval/CMSSW_5_3_6-START53_V14/RelValProdTTbar/GEN-SIM/v2/00000/DE03BB7E-F429-E211-A0B4-001A928116CC.root
+/store/relval/CMSSW_8_0_11/RelValProdTTbar/GEN-SIM/80X_mcRun1_realistic_v4-v1/10000/06A6C86B-C634-E611-93A5-0CC47A74525A.root
+/store/relval/CMSSW_8_0_16/RelValProdTTbar_13/GEN-SIM/80X_mcRun2_asymptotic_v16_gs7120p2-v1/10000/06F2C3AC-8957-E611-9DDF-0025905B85D8.root
+/store/relval/CMSSW_8_0_16/RelValZEEMM_13_HI/GEN-SIM/80X_mcRun2_HeavyIon_v9-v1/10000/F8FC5F64-1657-E611-A57E-002590A887F0.root
+/store/relval/CMSSW_9_0_0_pre5/RelValTTbar_13/GEN-SIM/90X_upgrade2017_realistic_v15-v1/00000/14F749AC-8AFE-E611-9821-0CC47A78A4A0.root

HLTrigger/Configuration/test/testAccessToEDMInputsOfHLTTests_update_filelist.sh

@@ -0,0 +1,93 @@
+#!/bin/bash -e
+
+# This script updates the file "${CMSSW_BASE}"/src/HLTrigger/Configuration/test/testAccessToEDMInputsOfHLTTests_filelist.txt
+# with the list of EDM files potentially used by HLT tests in the main release cycles of CMSSW (i.e. branches named CMSSW_\d_\d_X).
+
+# path to output file
+outputFile="${CMSSW_BASE}"/src/HLTrigger/Configuration/test/testAccessToEDMInputsOfHLTTests_filelist.txt
+
+# path to directory hosting this script
+TESTDIR=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd)
+
+# ensure that directory hosting this script corresponds to ${CMSSW_BASE}/src/HLTrigger/Configuration/test
+if [ "${TESTDIR}" != "${CMSSW_BASE}"/src/HLTrigger/Configuration/test ]; then
+  printf "\n%s\n" "ERROR -- the directory hosting testAccessToHLTTestInputs.sh [1] does not correspond to \${CMSSW_BASE}/src/HLTrigger/Configuration/test [2]"
+  printf "%s\n"   "         [1] ${TESTDIR}"
+  printf "%s\n\n" "         [2] ${CMSSW_BASE}/src/HLTrigger/Configuration/test"
+  exit 1
+fi
+
+# files in CMSSW using EDM inputs for HLT tests
+cmsswFiles=(
+  HLTrigger/Configuration/test/cmsDriver.csh
+  Configuration/HLT/python/addOnTestsHLT.py
+  Utilities/ReleaseScripts/scripts/addOnTests.py
+)
+
+# list of CMSSW branches to be checked
+# official-cmssw is the default name of the remote corresponding to the central CMSSW repository
+cmsswBranches=($(git branch -a | grep 'remotes/official-cmssw/CMSSW_[0-9]*_[0-9]*_X$'))
+cmsswBranches+=("HEAD") # add HEAD to include updates committed locally
+
+# create 1st temporary file (list of EDM input files used by HLT tests, incl. duplicates)
+TMPFILE1=$(mktemp)
+
+# grep from base directory
+cd "${CMSSW_BASE}"/src
+
+printf "%s\n" "-------------------------"
+printf "%s\n" "Finding list of EDM files used by HLT tests in CMSSW (branches: '^CMSSW_[0-9]*_[0-9]*_X$')..."
+
+# loop over CMSSW branches to be grep-d
+for cmsswBranch in "${cmsswBranches[@]}"; do
+  foo=($(git grep -h "[='\" ]/store/.*.root" ${cmsswBranch} -- ${cmsswFiles[*]} 2> /dev/null |
+    sed 's|=/store/| /store/|g' | sed "s|'| |g" | sed 's|"| |g' |
+    awk '{ for(i=1;i<=NF;i++) if ($i ~ /\/store\/.*.root/) print $i }'))
+  printf "\n  %s\n" "${cmsswBranch}"
+  for bar in "${foo[@]}"; do
+    printf "    %s\n" "${bar}"
+    echo "${bar}" >> "${TMPFILE1}"
+  done
+  unset foo bar
+done; unset cmsswBranch
+
+# create 2nd temporary file (list of available EDM input files used by HLT tests, without duplicates)
+TMPFILE2=$(mktemp)
+
+# edmFileIsAvailable:
+#  use LFN to check if a EDM file is in the ibeos cache,
+#  or can be accessed remotely via global redirector
+function edmFileIsAvailable() {
+  [ $# -eq 1 ] || return 1
+  # check access to ibeos cache
+  edmFileUtil -f root://eoscms.cern.ch//eos/cms/store/user/cmsbuild"${1}" &> /dev/null
+  [ $? -ne 0 ] || return 0
+  # check remote access via global redirector
+  edmFileUtil -f root://cms-xrd-global.cern.ch/"${1}" &> /dev/null
+  return $?
+}
+
+printf "%s\n" "-------------------------"
+printf "%s\n" "Checking availability of EDM files..."
+printf "%s\n" "(checks whether the file is in ibeos cache, or can be accessed remotely via global redirector)"
+
+for inputFile in $(cat "${TMPFILE1}" | sort -u); do
+  printf '\e[1;34m%-20s\e[m %s\033[0K\r' "[Checking...]" "${inputFile}"
+  if ! edmFileIsAvailable "${inputFile}"; then
+    printf '\e[1;31m%-20s\e[m %s\n' "[File not available]" "${inputFile}"
+    continue
+  fi
+  printf '\e[1;32m%-20s\e[m %s\n' "[File available]" "${inputFile}"
+  echo "${inputFile}" >> "${TMPFILE2}"
+done
+unset inputFile
+
+# create/update output file
+cat "${TMPFILE2}" | sort -u > "${outputFile}"
+
+printf "%s\n" "-------------------------"
+printf "%s\n" "File updated: ${outputFile}"
+printf "%s\n" "-------------------------"
+
+# return to test/ directory
+cd "${TESTDIR}"