Merge pull request #37283 from missirol/devel_hltIntegTests

update of `hltIntegrationTests` and path of Run-3 Data file for TSG tests

Configuration/HLT/python/addOnTestsHLT.py

@@ -31,16 +31,16 @@ def addOnTestsHLT():
                     'hlt_data_Fake2': ['cmsDriver.py RelVal -s L1REPACK:Full --data --scenario=pp -n 10 --conditions auto:run2_hlt_Fake2 --relval 9000,50 --datatier "RAW" --eventcontent RAW --customise=HLTrigger/Configuration/CustomConfigs.L1T --era Run2_2016 --fileout file:RelVal_Raw_Fake2_DATA.root --filein /store/data/Run2016B/JetHT/RAW/v1/000/272/762/00000/C666CDE2-E013-E611-B15A-02163E011DBE.root',
                                'HLTrigger/Configuration/test/OnLine_HLT_Fake2.py',
                                'cmsDriver.py RelVal -s HLT:Fake2,RAW2DIGI,L1Reco,RECO --data --scenario=pp -n 10 --conditions auto:run2_data_Fake2 --relval 9000,50 --datatier "RAW-HLT-RECO" --eventcontent FEVTDEBUGHLT --customise=HLTrigger/Configuration/CustomConfigs.L1THLT --era Run2_2016 --processName=HLTRECO --filein file:RelVal_Raw_Fake2_DATA.root --fileout file:RelVal_Raw_Fake2_DATA_HLT_RECO.root'],
-                    'hlt_data_GRun' : ['cmsDriver.py RelVal -s L1REPACK:Full --data --scenario=pp -n 10 --conditions auto:run3_hlt_GRun --relval 9000,50 --datatier "RAW" --eventcontent RAW --customise=HLTrigger/Configuration/CustomConfigs.L1T --era Run3 --fileout file:RelVal_Raw_GRun_DATA.root --filein /store/data/Commissioning2021/MinimumBias1/RAW/v1/000/346/304/00000/0949cd03-66a6-4034-a630-b9fef4dde3d2.root',
+                    'hlt_data_GRun' : ['cmsDriver.py RelVal -s L1REPACK:Full --data --scenario=pp -n 10 --conditions auto:run3_hlt_GRun --relval 9000,50 --datatier "RAW" --eventcontent RAW --customise=HLTrigger/Configuration/CustomConfigs.L1T --era Run3 --fileout file:RelVal_Raw_GRun_DATA.root --filein root://eoscms.cern.ch//eos/cms/store/group/dpg_trigger/comm_trigger/TriggerStudiesGroup/STORM/RAW/Commissioning2021_MinimumBias1_run346304/0949cd03-66a6-4034-a630-b9fef4dde3d2.root',
                                'HLTrigger/Configuration/test/OnLine_HLT_GRun.py',
                                'cmsDriver.py RelVal -s HLT:GRun,RAW2DIGI,L1Reco,RECO --data --scenario=pp -n 10 --conditions auto:run3_data_GRun --relval 9000,50 --datatier "RAW-HLT-RECO" --eventcontent FEVTDEBUGHLT --customise=HLTrigger/Configuration/CustomConfigs.L1THLT --era Run3 --processName=HLTRECO --filein file:RelVal_Raw_GRun_DATA.root --fileout file:RelVal_Raw_GRun_DATA_HLT_RECO.root'],
-                    'hlt_data_HIon' : ['cmsDriver.py RelVal -s L1REPACK:Full --data --scenario=pp -n 10 --conditions auto:run3_hlt_HIon --relval 9000,50 --datatier "RAW" --eventcontent RAW --customise=HLTrigger/Configuration/CustomConfigs.L1T --era Run3_pp_on_PbPb --fileout file:RelVal_Raw_HIon_DATA.root --filein /store/data/Commissioning2021/MinimumBias1/RAW/v1/000/346/304/00000/0949cd03-66a6-4034-a630-b9fef4dde3d2.root', # --customise_commands=\'from FWCore.ParameterSet.MassReplace import massSearchReplaceAnyInputTag; massSearchReplaceAnyInputTag(process.SimL1Emulator,"rawDataCollector","rawDataRepacker",False,True)\'',
+                    'hlt_data_HIon' : ['cmsDriver.py RelVal -s L1REPACK:Full --data --scenario=pp -n 10 --conditions auto:run3_hlt_HIon --relval 9000,50 --datatier "RAW" --eventcontent RAW --customise=HLTrigger/Configuration/CustomConfigs.L1T --era Run3_pp_on_PbPb --fileout file:RelVal_Raw_HIon_DATA.root --filein root://eoscms.cern.ch//eos/cms/store/group/dpg_trigger/comm_trigger/TriggerStudiesGroup/STORM/RAW/Commissioning2021_MinimumBias1_run346304/0949cd03-66a6-4034-a630-b9fef4dde3d2.root', # --customise_commands=\'from FWCore.ParameterSet.MassReplace import massSearchReplaceAnyInputTag; massSearchReplaceAnyInputTag(process.SimL1Emulator,"rawDataCollector","rawDataRepacker",False,True)\'',
                                'HLTrigger/Configuration/test/OnLine_HLT_HIon.py',
                                'cmsDriver.py RelVal -s HLT:HIon,RAW2DIGI,L1Reco,RECO --data --scenario=pp -n 10 --conditions auto:run3_data_HIon --relval 9000,50 --datatier "RAW-HLT-RECO" --eventcontent FEVTDEBUGHLT --customise=HLTrigger/Configuration/CustomConfigs.L1THLT --era Run3_pp_on_PbPb --processName=HLTRECO --filein file:RelVal_Raw_HIon_DATA.root --fileout file:RelVal_Raw_HIon_DATA_HLT_RECO.root'],
-                    'hlt_data_PIon' : ['cmsDriver.py RelVal -s L1REPACK:Full --data --scenario=pp -n 10 --conditions auto:run3_hlt_PIon --relval 9000,50 --datatier "RAW" --customise=HLTrigger/Configuration/CustomConfigs.L1T --era Run3 --eventcontent RAW --fileout file:RelVal_Raw_PIon_DATA.root --filein /store/data/Commissioning2021/MinimumBias1/RAW/v1/000/346/304/00000/0949cd03-66a6-4034-a630-b9fef4dde3d2.root',
+                    'hlt_data_PIon' : ['cmsDriver.py RelVal -s L1REPACK:Full --data --scenario=pp -n 10 --conditions auto:run3_hlt_PIon --relval 9000,50 --datatier "RAW" --customise=HLTrigger/Configuration/CustomConfigs.L1T --era Run3 --eventcontent RAW --fileout file:RelVal_Raw_PIon_DATA.root --filein root://eoscms.cern.ch//eos/cms/store/group/dpg_trigger/comm_trigger/TriggerStudiesGroup/STORM/RAW/Commissioning2021_MinimumBias1_run346304/0949cd03-66a6-4034-a630-b9fef4dde3d2.root',
                                'HLTrigger/Configuration/test/OnLine_HLT_PIon.py',
                                'cmsDriver.py RelVal -s HLT:PIon,RAW2DIGI,L1Reco,RECO --data --scenario=pp -n 10 --conditions auto:run3_data_PIon --relval 9000,50 --datatier "RAW-HLT-RECO" --eventcontent FEVTDEBUGHLT --customise=HLTrigger/Configuration/CustomConfigs.L1THLT --era Run3 --processName=HLTRECO --filein file:RelVal_Raw_PIon_DATA.root --fileout file:RelVal_Raw_PIon_DATA_HLT_RECO.root'],
-                    'hlt_data_PRef' : ['cmsDriver.py RelVal -s L1REPACK:Full --data --scenario=pp -n 10 --conditions auto:run3_hlt_PRef --relval 9000,50 --datatier "RAW" --customise=HLTrigger/Configuration/CustomConfigs.L1T --era Run3 --eventcontent RAW --fileout file:RelVal_Raw_PRef_DATA.root --filein /store/data/Commissioning2021/MinimumBias1/RAW/v1/000/346/304/00000/0949cd03-66a6-4034-a630-b9fef4dde3d2.root',
+                    'hlt_data_PRef' : ['cmsDriver.py RelVal -s L1REPACK:Full --data --scenario=pp -n 10 --conditions auto:run3_hlt_PRef --relval 9000,50 --datatier "RAW" --customise=HLTrigger/Configuration/CustomConfigs.L1T --era Run3 --eventcontent RAW --fileout file:RelVal_Raw_PRef_DATA.root --filein root://eoscms.cern.ch//eos/cms/store/group/dpg_trigger/comm_trigger/TriggerStudiesGroup/STORM/RAW/Commissioning2021_MinimumBias1_run346304/0949cd03-66a6-4034-a630-b9fef4dde3d2.root',
                                'HLTrigger/Configuration/test/OnLine_HLT_PRef.py',
                                'cmsDriver.py RelVal -s HLT:PRef,RAW2DIGI,L1Reco,RECO --data --scenario=pp -n 10 --conditions auto:run3_data_PRef --relval 9000,50 --datatier "RAW-HLT-RECO" --eventcontent FEVTDEBUGHLT --customise=HLTrigger/Configuration/CustomConfigs.L1THLT --era Run3 --processName=HLTRECO --filein file:RelVal_Raw_PRef_DATA.root --fileout file:RelVal_Raw_PRef_DATA_HLT_RECO.root'],
                      }

HLTrigger/Configuration/scripts/hltCompareResults

@@ -8,12 +8,9 @@ function err() {
   echo -e "$@" 1>&2
 }
 
-
-# exract list of triggers
-TRIGGERS=$(cat hlt.py | grep 'cms.Path' | sed -e's/process.\(\w\+\) = cms.Path.*/\1/' | grep -v HLTriggerFinalPath | grep -v HLT_LogMonitor)
-
-# check if thw menu as a whole was run
 touch issues.txt
+
+# check if the menu as a whole was run
 if ! [ -f "hlt.done" ]; then
   err "ERROR: Execution of the full HLT menu failed.\nPlease check the contents of 'hlt.log' for details."
   cat >> issues.txt <<EOF
@@ -22,7 +19,18 @@ EOF
   exit 1
 fi
 
-# compare the results of running each path by iteslf with those from the full menu
+# extract list of triggers
+if ! [ -f "paths.txt" ]; then
+  err "ERROR: File \"paths.txt\" does not exist (must contain the list of paths used in the integration tests)."
+  cat >> issues.txt <<EOF
+paths.txt
+EOF
+  exit 1
+fi
+
+TRIGGERS=$(cat paths.txt)
+
+# compare the results of running each path by itself with those from the full menu
 STATUS=0
 for TRIGGER in $TRIGGERS; do
   if ! [ -f ${TRIGGER}.done ]; then

HLTrigger/Configuration/scripts/hltIntegrationTests

@@ -114,7 +114,6 @@ EXTRA=""
 DATA="--data"
 HLTLISTPATHPROXY=""
 
-
 SELECTION=""
 
 # parse options
@@ -242,7 +241,8 @@ mkdir  "$WORKDIR"
 cd     "$WORKDIR"
 
 # find the list of all trigger paths
-TRIGGERS=$(hltListPaths -p "$MENU" $HLTLISTPATHPROXY | grep -v HLTriggerFinalPath)
+TRIGGERS=$(hltListPaths "$MENU" $HLTLISTPATHPROXY -p --no-dep --exclude "^HLTriggerFinalPath$" "^Dataset_.+")
+echo "${TRIGGERS[@]}" > paths.txt
 
 # print some info
 if [ "$SELECTION" == "complex" ]; then

HLTrigger/Configuration/scripts/hltListPaths

@@ -1,41 +1,106 @@
 #!/usr/bin/env python3
-
-from __future__ import print_function
 import os
-import re
+import sys
 import argparse
-from HLTrigger.Configuration.extend_argparse import *
-
-import HLTrigger.Configuration.Tools.pipe    as pipe
-import HLTrigger.Configuration.Tools.confdb  as confdb
+import re
+import FWCore.ParameterSet.Config as cms
+import HLTrigger.Configuration.Tools.pipe as pipe
 import HLTrigger.Configuration.Tools.options as options
+from HLTrigger.Configuration.extend_argparse import *
 
-
-# adapted from HLTrigger/Configuration/python/Tools/confdb.py
-def _build_query(menu):
-  if menu.run:
-    return '--runNumber %s' % menu.run
+def getPathList(config):
+  # cmd-line args to select HLT configuration
+  if config.menu.run:
+    configline = f'--runNumber {config.menu.run}'
   else:
-    return '--%s --%s --configName %s' % (menu.database, menu.version, menu.name)
+    configline = f'--{config.menu.database} --{config.menu.version} --configName {config.menu.name}'
 
-def getPathList(config):
-  cmdline = 'hltConfigFromDB --cff %s --noedsources --noes --noservices --nosequences --nomodules' % _build_query(config.menu)
+  # cmd to download HLT configuration
+  cmdline = f'hltConfigFromDB {configline} --noedsources --noes --noservices'
   if config.proxy:
-     cmdline += f" --dbproxy --dbproxyhost {config.proxy_host} --dbproxyport {config.proxy_port}"
-  data = pipe.pipe(cmdline).decode()
-  if 'Exhausted Resultset' in data or 'CONFIG_NOT_FOUND' in data:
-    raise ImportError('%s is not a valid HLT menu' % menu.value)
+    cmdline += f' --dbproxy --dbproxyhost {config.proxy_host} --dbproxyport {config.proxy_port}'
 
+  # load HLT configuration
+  try:
+    foo = {'process': None}
+    exec(pipe.pipe(cmdline).decode(), foo)
+    process = foo['process']
+  except:
+    raise Exception(f'query did not return a valid python file:\n query="{cmdline}"')
+
+  if not isinstance(process, cms.Process):
+    raise Exception(f'query did not return a valid HLT menu:\n query="{cmdline}"')
+
+  # Paths only
   if config.selection == 'paths':
-    filter = re.compile(r' *= *cms.Path.*')
+    pathDict = process.paths_()
+
+  # EndPaths only
   elif config.selection == 'endpaths':
-    filter = re.compile(r' *= *cms.EndPath.*')
-  else: # 'all'
-    filter = re.compile(r' *= *cms.(End)?Path.*')
+    pathDict = process.endpaths_()
 
-  paths  = [ filter.sub('', line) for line in data.splitlines() if filter.search(line) ]
-  return paths
+  # FinalPaths only
+  elif config.selection == 'finalpaths':
+    pathDict = process.finalpaths_()
 
+  # Paths, EndPaths, and FinalPaths ('all')
+  else:
+    pathDict = zip(process.paths_(), process.endpaths_(), process.finalpaths_())
+
+  ret = []
+  for pathName in pathDict:
+
+    # skip if name of the path matches any of
+    # the regular expressions listed in "--exclude"
+    skipPath = False
+    for excludeRegExpr in config.excludeRegExprs:
+      if bool(re.search(excludeRegExpr, pathName)):
+        skipPath = True
+        break
+    if skipPath:
+      continue
+
+    if config.no_dependent_paths:
+      # do not include "dependent paths", i.e. paths that depend on the result of other paths in the same job
+      # the current criterion to identify a path as "dependent" is that
+      # (1) the path contains a "TriggerResultsFilter" module and
+      # (2) the latter module uses the TriggerResults of the current process, and has a non-empty list of "triggerConditions"
+      path = pathDict[pathName]
+      pathIsDependent = False
+      isPath = isinstance(path, cms.Path)
+
+      for moduleName in path.moduleNames():
+        module = getattr(process, moduleName)
+        if module.type_() != 'TriggerResultsFilter' or (hasattr(module, 'triggerConditions') and len(module.triggerConditions) == 0):
+          continue
+
+        usesPathStatus = hasattr(module, 'usePathStatus') and module.usePathStatus
+        usesTrigResOfCurrentProcess = hasattr(module, 'hltResults') and module.hltResults.getProcessName() in [process.name_(), '@currentProcess']+['']*(not isPath)
+
+        if isPath:
+          if usesPathStatus:
+            pathIsDependent = True
+          elif usesTrigResOfCurrentProcess:
+            # The Path contains a TriggerResultsFilter with usePathStatus=False and forcing access to the TriggerResults of the current Process.
+            #  - This is not supported, and should result in a runtime error when using cmsRun.
+            #  - Here, a warning is returned to stderr, and the Path is omitted from the output list.
+            warning_msg = 'WARNING -- the cms.Path named "'+pathName+'" will be ignored.'
+            warning_msg += '\n'+' '*12+'- It contains a "TriggerResultsFilter" attempting to access the "TriggerResults" of the current Process (module: "'+moduleName+'").'
+            warning_msg += '\n'+' '*12+'- This is not supported, and should result in a runtime error when using cmsRun. Please check again the HLT configuration.'
+            print(warning_msg, file=sys.stderr)
+            pathIsDependent = True
+        else:
+          pathIsDependent = usesPathStatus or usesTrigResOfCurrentProcess
+
+        if pathIsDependent:
+          break
+
+      if pathIsDependent:
+        continue
+
+    ret.append(pathName)
+
+  return ret
 
 # define an argparse parser to parse our options
 textwidth = int( 80 )
@@ -49,17 +114,19 @@ formatter = FixedWidthFormatter( HelpFormatterRespectNewlines, width = textwidth
 defaults = options.HLTProcessOptions()
 
 parser = argparse.ArgumentParser(
-  description       = 'List all the Paths and EndPaths from an HLT configuration in the ConfDB database.''',
+  description       = 'List all the Paths, EndPaths and FinalPaths of an HLT configuration in the ConfDB database.',
   argument_default  = argparse.SUPPRESS,
   formatter_class   = formatter,
   add_help          = False )
 
 # required argument
 parser.add_argument('menu',
-                    action  = 'store', 
+                    action  = 'store',
                     type    = options.ConnectionHLTMenu,
-                    metavar = 'MENU', 
+                    metavar = 'MENU',
                     help    = 'HLT menu to dump from the database. Supported formats are:\n  - /path/to/configuration[/Vn]\n  - [[{v1|v2|v3}/]{run3|run2|online|adg}:]/path/to/configuration[/Vn]\n  - run:runnumber\nThe possible converters are "v1", "v2, and "v3" (default).\nThe possible databases are "run3" (default, used for offline development), "run2" (used for accessing run2 offline development menus), "online" (used to extract online menus within Point 5) and "adg" (used to extract the online menus outside Point 5).\nIf no menu version is specified, the latest one is automatically used.\nIf "run:" is used instead, the HLT menu used for the given run number is looked up and used.\nNote other converters and databases exist as options but they are only for expert/special use.' )
+
+# options
 parser.add_argument('--dbproxy',
                     dest    = 'proxy',
                     action  = 'store_true',
@@ -78,25 +145,40 @@ parser.add_argument('--dbproxyhost',
                     default = defaults.proxy_host,
                     help    = 'Host of the socks proxy (default: "localhost")' )
 
-
-# options
 group = parser.add_mutually_exclusive_group()
-group.add_argument('-p', '--only-paths', 
+group.add_argument('-p', '--only-paths',
                     dest    = 'selection',
                     action  = 'store_const',
                     const   = 'paths',
                     help    = 'List only Paths' )
-group.add_argument('-e', '--only-endpaths', 
+group.add_argument('-e', '--only-endpaths',
                     dest    = 'selection',
                     action  = 'store_const',
                     const   = 'endpaths',
                     help    = 'List only EndPaths' )
+group.add_argument('-f', '--only-finalpaths',
+                    dest    = 'selection',
+                    action  = 'store_const',
+                    const   = 'finalpaths',
+                    help    = 'List only FinalPaths' )
 group.add_argument('-a', '--all', 
                     dest    = 'selection',
                     action  = 'store_const',
                     const   = 'all',
                     default = 'all',
-                    help    = 'List both Paths and EndPaths (default)' )
+                    help    = 'List Paths, EndPaths and FinalPaths (default)' )
+
+parser.add_argument('--no-dependent-paths',
+                    dest    = 'no_dependent_paths',
+                    action  = 'store_true',
+                    default = False,
+                    help    = 'Do not list paths which depend on the result of other paths (default: false)' )
+
+parser.add_argument('--exclude',
+                    dest    = 'excludeRegExprs',
+                    nargs   = '+',
+                    default = [],
+                    help    = 'List of regular expressions to select names of paths to be ignored with re.search (default: empty)' )
 
 # redefine "--help" to be the last option, and use a customized message 
 parser.add_argument('-h', '--help', 
@@ -105,6 +187,6 @@ parser.add_argument('-h', '--help',
 
 # parse command line arguments and options
 config = parser.parse_args()
-paths  = getPathList(config)
+paths = getPathList(config)
 for path in paths:
   print(path)

HLTrigger/Configuration/test/cmsDriver.csh

@@ -33,10 +33,10 @@ set InputGenSimPRef3 = $InputGenSimGRun3
 set InputLHCRawGRun0 = root://eoscms.cern.ch//eos/cms/store/data/Run2012A/MuEG/RAW/v1/000/191/718/14932935-E289-E111-830C-5404A6388697.root
 set InputLHCRawGRun1 = root://eoscms.cern.ch//eos/cms/store/data/Run2015D/MuonEG/RAW/v1/000/256/677/00000/80950A90-745D-E511-92FD-02163E011C5D.root
 set InputLHCRawGRun2 = root://eoscms.cern.ch//eos/cms/store/data/Run2016B/JetHT/RAW/v1/000/272/762/00000/C666CDE2-E013-E611-B15A-02163E011DBE.root
-set InputLHCRawGRun3 = root://eoscms.cern.ch//eos/cms/store/data/Commissioning2021/MinimumBias1/RAW/v1/000/346/304/00000/0949cd03-66a6-4034-a630-b9fef4dde3d2.root
+set InputLHCRawGRun3 = root://eoscms.cern.ch//eos/cms/store/group/dpg_trigger/comm_trigger/TriggerStudiesGroup/STORM/RAW/Commissioning2021_MinimumBias1_run346304/0949cd03-66a6-4034-a630-b9fef4dde3d2.root
 set InputLHCRawHIon1 = root://eoscms.cern.ch//eos/cms/store/hidata/HIRun2015/HIHardProbes/RAW-RECO/HighPtJet-PromptReco-v1/000/263/689/00000/1802CD9A-DDB8-E511-9CF9-02163E0138CA.root
 #et InputLHCRawHIon3 = root://eoscms.cern.ch//eos/cms/store/hidata/HIRun2018A/HIHardProbes/RAW/v1/000/326/479/00000/853DBE29-53BA-9A44-9FDD-58E4E9064EB1.root
-set InputLHCRawHIon3 = root://eoscms.cern.ch//eos/cms/store/data/Commissioning2021/MinimumBias1/RAW/v1/000/346/304/00000/0949cd03-66a6-4034-a630-b9fef4dde3d2.root
+set InputLHCRawHIon3 = root://eoscms.cern.ch//eos/cms/store/group/dpg_trigger/comm_trigger/TriggerStudiesGroup/STORM/RAW/Commissioning2021_MinimumBias1_run346304/0949cd03-66a6-4034-a630-b9fef4dde3d2.root
 set InputLHCRawPIon2 = $InputLHCRawGRun2
 set InputLHCRawPRef2 = $InputLHCRawGRun2
 set InputLHCRawPIon3 = $InputLHCRawGRun3