From af83da0bac08aca763788d886752a4476d33f367 Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Tue, 20 May 2025 10:58:47 -0500 Subject: [PATCH 01/18] Added a tmp_path retention count of 10 and a trace-back option of no (none) to pytest.ini. --- pytest.ini | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pytest.ini b/pytest.ini index 370e923b..79a333e7 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,3 +2,11 @@ filterwarnings = ignore:.*is a deprecated alias.*:DeprecationWarning ignore:.*invalid escape sequence.*:DeprecationWarning + +tmp_path_retention_count = 10 + +# --tb not given Produces reams of output, with full source code included in tracebacks +# --tb=no Just shows location of failure in the test file: no use for tracking down errors +# --tb=short Just shows vanilla traceback: very useful, but file names are incomplete and relative +# --tb=native Slightly more info than short: still works very well. The full paths may be useful for CI +addopts = --tb=no From 780b862b5ee430c6289e57e884b080a561e8aa9c Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Tue, 20 May 2025 11:07:19 -0500 Subject: [PATCH 02/18] Updated the dfmodules_integtest_bundle.sh to bring it up-to-date with v5 software and infrastructure, and to add in features that we like from the daqsystemtest bundle script. --- scripts/dfmodules_integtest_bundle.sh | 81 ++++++++++++++++++--------- 1 file changed, 54 insertions(+), 27 deletions(-) diff --git a/scripts/dfmodules_integtest_bundle.sh b/scripts/dfmodules_integtest_bundle.sh index 3caef7de..97d99bcb 100755 --- a/scripts/dfmodules_integtest_bundle.sh +++ b/scripts/dfmodules_integtest_bundle.sh @@ -1,7 +1,9 @@ #!/bin/bash -# 11-Oct-2023, KAB +# 29-Apr-2025, KAB -integtest_list=( "large_trigger_record_test.py" "disabled_output_test.py" "multi_output_file_test.py" "insufficient_disk_space_test.py" ) +integtest_list=( "max_file_size_test.py" "multiple_data_writers_test.py" "hdf5_compression_test.py" ) +#integtest_list=( "large_trigger_record_test.py" "disabled_output_test.py" "multi_output_file_test.py" "insufficient_disk_space_test.py" ) +let last_test_index=${#integtest_list[@]}-1 usage() { declare -r script_name=$(basename "$0") @@ -11,9 +13,8 @@ Usage: Options: -h, --help : prints out usage information - -s -f - -l + -l -n -N --stop-on-failure : causes the script to stop when one of the integtests reports a failure @@ -31,11 +32,9 @@ Options: TEMP=`getopt -o hs:f:l:n:N: --long help,stop-on-failure,stop-on-skip -- "$@"` eval set -- "$TEMP" -let session_number=1 let first_test_index=0 -let last_test_index=999 -let individual_run_count=1 -let overall_run_count=1 +let individual_test_requested_iterations=1 +let full_set_requested_interations=1 let stop_on_failure=0 let stop_on_skip=0 @@ -45,10 +44,6 @@ while true; do usage exit 0 ;; - -s) - let session_number=$2 - shift 2 - ;; -f) let first_test_index=$2 shift 2 @@ -58,11 +53,11 @@ while true; do shift 2 ;; -n) - let individual_run_count=$2 + let individual_test_requested_iterations=$2 shift 2 ;; -N) - let overall_run_count=$2 + let full_set_requested_interations=$2 shift 2 ;; --stop-on-failure) @@ -80,37 +75,55 @@ while true; do esac done +# check if the numad daemon is running +numad_grep_output=`ps -ef | grep numad | grep -v grep` +if [[ "${numad_grep_output}" != "" ]]; then + echo "*********************************************************************" + echo "*** DANGER, DANGER, 'numad' appears to be running on this computer!" + echo "*** 'ps' output: ${numad_grep_output}" + echo "*** now if you want to abort this testing." + echo "*********************************************************************" + sleep 3 +fi + # other setup TIMESTAMP=`date '+%Y%m%d%H%M%S'` mkdir -p /tmp/pytest-of-${USER} ITGRUNNER_LOG_FILE="/tmp/pytest-of-${USER}/dfmodules_integtest_bundle_${TIMESTAMP}.log" +let total_number_of_tests=(1+${last_test_index}-${first_test_index})*${individual_test_requested_iterations}*${full_set_requested_interations} # run the tests -let overall_loop_count=0 -while [[ ${overall_loop_count} -lt ${overall_run_count} ]]; do - +let overall_test_index=0 # this is only used for user feedback +let full_set_loop_count=0 +while [[ ${full_set_loop_count} -lt ${full_set_requested_interations} ]]; do let test_index=0 for TEST_NAME in ${integtest_list[@]}; do if [[ ${test_index} -ge ${first_test_index} && ${test_index} -le ${last_test_index} ]]; then let individual_loop_count=0 - while [[ ${individual_loop_count} -lt ${individual_run_count} ]]; do - echo "===== Running ${TEST_NAME}" >> ${ITGRUNNER_LOG_FILE} + while [[ ${individual_loop_count} -lt ${individual_test_requested_iterations} ]]; do + let overall_test_index=${overall_test_index}+1 + echo "" + echo -e "\U0001F535 \033[0;34mStarting test ${overall_test_index} of ${total_number_of_tests}...\033[0m \U0001F535" | tee -a ${ITGRUNNER_LOG_FILE} + + echo -e "\u2B95 \033[0;1mRunning ${TEST_NAME}\033[0m \u2B05" | tee -a ${ITGRUNNER_LOG_FILE} if [[ -e "./${TEST_NAME}" ]]; then - pytest -s ./${TEST_NAME} --nanorc-option partition-number ${session_number} | tee -a ${ITGRUNNER_LOG_FILE} + pytest -s ./${TEST_NAME} | tee -a ${ITGRUNNER_LOG_FILE} elif [[ -e "${DBT_AREA_ROOT}/sourcecode/dfmodules/integtest/${TEST_NAME}" ]]; then - pytest -s ${DBT_AREA_ROOT}/sourcecode/dfmodules/integtest/${TEST_NAME} --nanorc-option partition-number ${session_number} | tee -a ${ITGRUNNER_LOG_FILE} + if [[ -w "${DBT_AREA_ROOT}" ]]; then + pytest -s ${DBT_AREA_ROOT}/sourcecode/dfmodules/integtest/${TEST_NAME} | tee -a ${ITGRUNNER_LOG_FILE} + else + pytest -s -p no:cacheprovider ${DBT_AREA_ROOT}/sourcecode/dfmodules/integtest/${TEST_NAME} | tee -a ${ITGRUNNER_LOG_FILE} + fi else - pytest -s ${DFMODULES_SHARE}/integtest/${TEST_NAME} --nanorc-option partition-number ${session_number} | tee -a ${ITGRUNNER_LOG_FILE} + pytest -s -p no:cacheprovider ${DFMODULES_SHARE}/integtest/${TEST_NAME} | tee -a ${ITGRUNNER_LOG_FILE} fi let pytest_return_code=${PIPESTATUS[0]} let individual_loop_count=${individual_loop_count}+1 if [[ ${stop_on_failure} -gt 0 ]]; then - search_result=`tail -20 ${ITGRUNNER_LOG_FILE} | grep -i fail` - #echo "failure search result is ${search_result}" - if [[ ${search_result} != "" || ${pytest_return_code} -ne 0 ]]; then + if [[ ${pytest_return_code} -ne 0 ]]; then break 3 fi fi @@ -127,7 +140,7 @@ while [[ ${overall_loop_count} -lt ${overall_run_count} ]]; do let test_index=${test_index}+1 done - let overall_loop_count=${overall_loop_count}+1 + let full_set_loop_count=${full_set_loop_count}+1 done # print out summary information @@ -140,4 +153,18 @@ echo "" | tee -a ${ITGRUNNER_L date | tee -a ${ITGRUNNER_LOG_FILE} echo "Log file is: ${ITGRUNNER_LOG_FILE}" | tee -a ${ITGRUNNER_LOG_FILE} echo "" | tee -a ${ITGRUNNER_LOG_FILE} -grep '=====' ${ITGRUNNER_LOG_FILE} | egrep ' in |Running' | tee -a ${ITGRUNNER_LOG_FILE} +egrep $'=====|\u2B95' ${ITGRUNNER_LOG_FILE} | egrep ' in |Running' | tee -a ${ITGRUNNER_LOG_FILE} + +# check again if the numad daemon is running +numad_grep_output=`ps -ef | grep numad | grep -v grep` +if [[ "${numad_grep_output}" != "" ]]; then + echo "" | tee -a ${ITGRUNNER_LOG_FILE} + echo "********************************************************************************" | tee -a ${ITGRUNNER_LOG_FILE} + echo "*** WARNING: 'numad' appears to be running on this computer!" | tee -a ${ITGRUNNER_LOG_FILE} + echo "*** 'ps' output: ${numad_grep_output}" | tee -a ${ITGRUNNER_LOG_FILE} + echo "*** This daemon can adversely affect the running of these tests, especially ones" | tee -a ${ITGRUNNER_LOG_FILE} + echo "*** that are resource intensive in the Readout Apps. This is because numad moves" | tee -a ${ITGRUNNER_LOG_FILE} + echo "*** processes (threads?) to different cores/numa nodes periodically, and that" | tee -a ${ITGRUNNER_LOG_FILE} + echo "*** context switch can disrupt the stable running of the DAQ processes." | tee -a ${ITGRUNNER_LOG_FILE} + echo "********************************************************************************" | tee -a ${ITGRUNNER_LOG_FILE} +fi From bd57f83f13e2eb7079fc20ae5c3c60f234a3c44a Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Tue, 20 May 2025 11:09:43 -0500 Subject: [PATCH 03/18] Cleaned up disabled_output_test.py, insufficient_disk_space_test.py, and large_trigger_record_test.py a bit. These integtests may not be fully operational in v5, though. --- integtest/disabled_output_test.py | 30 +++++++++-------------- integtest/insufficient_disk_space_test.py | 26 ++++++-------------- integtest/large_trigger_record_test.py | 5 +--- 3 files changed, 20 insertions(+), 41 deletions(-) diff --git a/integtest/disabled_output_test.py b/integtest/disabled_output_test.py index d06cf386..fda6688b 100644 --- a/integtest/disabled_output_test.py +++ b/integtest/disabled_output_test.py @@ -25,7 +25,6 @@ wib1_frag_hsi_trig_params = { "fragment_type_description": "WIB", "fragment_type": "ProtoWIB", - "hdf5_source_subsystem": "Detector_Readout", "expected_fragment_count": number_of_data_producers, "min_size_bytes": 37656, "max_size_bytes": 37656, @@ -33,7 +32,6 @@ wib1_frag_multi_trig_params = { "fragment_type_description": "WIB", "fragment_type": "ProtoWIB", - "hdf5_source_subsystem": "Detector_Readout", "expected_fragment_count": number_of_data_producers, "min_size_bytes": 72, "max_size_bytes": 54000, @@ -41,7 +39,6 @@ wib2_frag_hsi_trig_params = { "fragment_type_description": "WIB", "fragment_type": "WIB", - "hdf5_source_subsystem": "Detector_Readout", "expected_fragment_count": (number_of_data_producers), "min_size_bytes": 29808, "max_size_bytes": 30280, @@ -49,7 +46,6 @@ wib2_frag_multi_trig_params = { "fragment_type_description": "WIB", "fragment_type": "WIB", - "hdf5_source_subsystem": "Detector_Readout", "expected_fragment_count": (number_of_data_producers), "min_size_bytes": 72, "max_size_bytes": 54000, @@ -57,7 +53,6 @@ wibeth_frag_hsi_trig_params = { "fragment_type_description": "WIBEth", "fragment_type": "WIBEth", - "hdf5_source_subsystem": "Detector_Readout", "expected_fragment_count": (number_of_data_producers), "min_size_bytes": 7272, "max_size_bytes": 14472, @@ -65,7 +60,6 @@ wibeth_frag_multi_trig_params = { "fragment_type_description": "WIBEth", "fragment_type": "WIBEth", - "hdf5_source_subsystem": "Detector_Readout", "expected_fragment_count": (number_of_data_producers), "min_size_bytes": 72, "max_size_bytes": 14472, @@ -73,23 +67,20 @@ triggercandidate_frag_params = { "fragment_type_description": "Trigger Candidate", "fragment_type": "Trigger_Candidate", - "hdf5_source_subsystem": "Trigger", "expected_fragment_count": 1, - "min_size_bytes": 72, + "min_size_bytes": 128, "max_size_bytes": 280, } triggeractivity_frag_params = { "fragment_type_description": "Trigger Activity", "fragment_type": "Trigger_Activity", - "hdf5_source_subsystem": "Trigger", "expected_fragment_count": 1, "min_size_bytes": 72, "max_size_bytes": 216, } -triggertp_frag_params = { - "fragment_type_description": "Trigger with TPs", +triggerprimitive_frag_params = { + "fragment_type_description": "Trigger Primitive", "fragment_type": "Trigger_Primitive", - "hdf5_source_subsystem": "Trigger", "expected_fragment_count": 2, # number of readout apps (1) times 2 "min_size_bytes": 72, "max_size_bytes": 16000, @@ -97,7 +88,6 @@ hsi_frag_params = { "fragment_type_description": "HSI", "fragment_type": "Hardware_Signal", - "hdf5_source_subsystem": "HW_Signals_Interface", "expected_fragment_count": 1, "min_size_bytes": 72, "max_size_bytes": 100, @@ -225,7 +215,7 @@ def test_data_files(run_nanorc): # fragment_check_list.append(wib1_frag_multi_trig_params) # ProtoWIB # fragment_check_list.append(wib2_frag_multi_trig_params) # DuneWIB fragment_check_list.append(wibeth_frag_multi_trig_params) # WIBEth - fragment_check_list.append(triggertp_frag_params) + fragment_check_list.append(triggerprimitive_frag_params) fragment_check_list.append(triggeractivity_frag_params) else: # fragment_check_list.append(wib1_frag_hsi_trig_params) # ProtoWIB @@ -235,17 +225,19 @@ def test_data_files(run_nanorc): # Run some tests on the output data file assert len(run_nanorc.data_files) == expected_number_of_data_files + all_ok = True for idx in range(len(run_nanorc.data_files)): data_file = data_file_checks.DataFile(run_nanorc.data_files[idx]) - assert data_file_checks.sanity_check(data_file) - assert data_file_checks.check_file_attributes(data_file) - assert data_file_checks.check_event_count( + all_ok &= data_file_checks.sanity_check(data_file) + all_ok &= data_file_checks.check_file_attributes(data_file) + all_ok &= data_file_checks.check_event_count( data_file, local_expected_event_count, local_event_count_tolerance ) for jdx in range(len(fragment_check_list)): - assert data_file_checks.check_fragment_count( + all_ok &= data_file_checks.check_fragment_count( data_file, fragment_check_list[jdx] ) - assert data_file_checks.check_fragment_sizes( + all_ok &= data_file_checks.check_fragment_sizes( data_file, fragment_check_list[jdx] ) + assert all_ok diff --git a/integtest/insufficient_disk_space_test.py b/integtest/insufficient_disk_space_test.py index 3587a71c..7d65f7ae 100644 --- a/integtest/insufficient_disk_space_test.py +++ b/integtest/insufficient_disk_space_test.py @@ -53,7 +53,6 @@ wibeth_frag_hsi_trig_params = { "fragment_type_description": "WIBEth", "fragment_type": "WIBEth", - "hdf5_source_subsystem": "Detector_Readout", "expected_fragment_count": (number_of_data_producers * number_of_readout_apps), "min_size_bytes": 35157672, "max_size_bytes": 35157672, @@ -61,31 +60,20 @@ triggercandidate_frag_params = { "fragment_type_description": "Trigger Candidate", "fragment_type": "Trigger_Candidate", - "hdf5_source_subsystem": "Trigger", "expected_fragment_count": 1, - "min_size_bytes": 72, + "min_size_bytes": 128, "max_size_bytes": 280, } triggeractivity_frag_params = { "fragment_type_description": "Trigger Activity", "fragment_type": "Trigger_Activity", - "hdf5_source_subsystem": "Trigger", "expected_fragment_count": number_of_readout_apps, "min_size_bytes": 72, "max_size_bytes": 400, } -triggertp_frag_params = { - "fragment_type_description": "Trigger with TPs", - "fragment_type": "Trigger_Primitive", - "hdf5_source_subsystem": "Trigger", - "expected_fragment_count": ((number_of_data_producers * number_of_readout_apps)), - "min_size_bytes": 72, - "max_size_bytes": 16000, -} hsi_frag_params = { "fragment_type_description": "HSI", "fragment_type": "Hardware_Signal", - "hdf5_source_subsystem": "HW_Signals_Interface", "expected_fragment_count": 1, "min_size_bytes": 72, "max_size_bytes": 100, @@ -237,20 +225,22 @@ def test_data_files(run_nanorc): # Run some tests on the output data file assert len(run_nanorc.data_files) == expected_number_of_data_files + all_ok = True for idx in range(len(run_nanorc.data_files)): data_file = data_file_checks.DataFile(run_nanorc.data_files[idx]) - assert data_file_checks.sanity_check(data_file) - assert data_file_checks.check_file_attributes(data_file) - assert data_file_checks.check_event_count( + all_ok &= data_file_checks.sanity_check(data_file) + all_ok &= data_file_checks.check_file_attributes(data_file) + all_ok &= data_file_checks.check_event_count( data_file, local_expected_event_count, local_event_count_tolerance ) for jdx in range(len(fragment_check_list)): - assert data_file_checks.check_fragment_count( + all_ok &= data_file_checks.check_fragment_count( data_file, fragment_check_list[jdx] ) - assert data_file_checks.check_fragment_sizes( + all_ok &= data_file_checks.check_fragment_sizes( data_file, fragment_check_list[jdx] ) + assert all_ok def test_cleanup(run_nanorc): diff --git a/integtest/large_trigger_record_test.py b/integtest/large_trigger_record_test.py index 5a2a11ca..ada9d309 100644 --- a/integtest/large_trigger_record_test.py +++ b/integtest/large_trigger_record_test.py @@ -39,7 +39,6 @@ wibeth_frag_55pct_params = { "fragment_type_description": "WIBEth", "fragment_type": "WIBEth", - "hdf5_source_subsystem": "Detector_Readout", "expected_fragment_count": (number_of_data_producers * number_of_readout_apps), "min_size_bytes": 38678472, "max_size_bytes": 38678472, @@ -47,7 +46,6 @@ wibeth_frag_125pct_params = { "fragment_type_description": "WIBEth", "fragment_type": "WIBEth", - "hdf5_source_subsystem": "Detector_Readout", "expected_fragment_count": (number_of_data_producers * number_of_readout_apps), "min_size_bytes": 91411272, "max_size_bytes": 91411272, @@ -55,9 +53,8 @@ triggercandidate_frag_params = { "fragment_type_description": "Trigger Candidate", "fragment_type": "Trigger_Candidate", - "hdf5_source_subsystem": "Trigger", "expected_fragment_count": 1, - "min_size_bytes": 72, + "min_size_bytes": 128, "max_size_bytes": 280, } ignored_logfile_problems = { From 337bac3109790799efcfc42ef61d4c1144500f7a Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Tue, 20 May 2025 11:11:09 -0500 Subject: [PATCH 04/18] Updated the README file to better reflect current conditions. --- integtest/README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/integtest/README.md b/integtest/README.md index 680d8ae7..c4a2dd22 100644 --- a/integtest/README.md +++ b/integtest/README.md @@ -1,4 +1,4 @@ -* 19-Jul-2023, KAB, ELF, and others: notes on existing integtests +* 29-Apr-2025, KAB, ELF, and others: notes on existing integtests "integtests" are intended to be automated integration and/or system tests that make use of the "pytest" framework to validate the operation of the DAQ system in various scenarios. @@ -6,11 +6,14 @@ Here is a sample command for invoking a test (feel free to keep or drop the options in brackets, as you prefer): ``` -pytest -s disabled_output_test.py [--nanorc-option partition-number 2] [--nanorc-option timeout 300] +pytest -s max_file_size_test.py [--nanorc-option log-level debug] # this nanorc option is still useful even when using drunc ``` For reference, here are the ideas behind the existing tests: +* `max_file_size_test.py` - verifies that data files are closed when they reach a specified maximum file size (approximately) +* `multiple_data_writers_test.py` - verifies that we can run multiple DataWriters for a single TriggerRecordBuilder +* `hdf5_compression_test.py` - verifies that HDF5 compression is working as expected by writing several data files of known size + * `large_trigger_record_test.py` - verify that TriggerRecords that are close to the size of a whole file get written to disk correctly * `disabled_output_test.py` - verify that the --disable-data-storage option works -* `multi_output_file_test.py` - test that the file size maximum config parameter works * `insufficient_disk_space_test.py` - verify that the appropriate errors and warnings are produced when there isn't enough disk space to write data From fb0e7261312306bd0b1685eb0e26a43a6ade963b Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Tue, 20 May 2025 11:13:49 -0500 Subject: [PATCH 05/18] Renamed the multi_output_file_test.py as max_file_size_test.py to reflect its current purpose, updated some variable names inside it, and added the cleanup of TP Stream files after the test completes. --- integtest/max_file_size_test.py | 273 ++++++++++++++++++++++++++++++++ 1 file changed, 273 insertions(+) create mode 100644 integtest/max_file_size_test.py diff --git a/integtest/max_file_size_test.py b/integtest/max_file_size_test.py new file mode 100644 index 00000000..cd3a9433 --- /dev/null +++ b/integtest/max_file_size_test.py @@ -0,0 +1,273 @@ +import pytest +import os +import re +import copy +import urllib.request + +import integrationtest.data_file_checks as data_file_checks +import integrationtest.log_file_checks as log_file_checks +import integrationtest.data_classes as data_classes + +pytest_plugins = "integrationtest.integrationtest_drunc" + +# Values that help determine the running conditions +number_of_data_producers = 2 +number_of_readout_apps = 3 +data_rate_slowdown_factor = 1 + +# Default values for validation parameters +check_for_logfile_errors = True + +wibeth_frag_params = { + "fragment_type_description": "WIBEth", + "fragment_type": "WIBEth", + "expected_fragment_count": (number_of_data_producers * number_of_readout_apps), + "min_size_bytes": 7272, + "max_size_bytes": 194472, +} +wibeth_tpset_params = { + "fragment_type_description": "TP Stream", + "fragment_type": "Trigger_Primitive", + "expected_fragment_count": number_of_readout_apps * 3, + "frag_counts_by_record_ordinal": {"first": {"min_count": 1, "max_count": number_of_readout_apps * 3}, + "default": {"min_count": number_of_readout_apps * 3, "max_count": number_of_readout_apps * 3} }, + "min_size_bytes": 0, # not checked + "max_size_bytes": 0, # not checked + "debug_mask": 0x0, +} +# sizes: 128 is for one TC with zero TAs inside it (72+56) +# 208 is for one TC with one TA inside it (72+56+80) +# 264 is for two TCs with one TA in one of them (72+56+80+56) +triggercandidate_frag_params = { + "fragment_type_description": "Trigger Candidate", + "fragment_type": "Trigger_Candidate", + "expected_fragment_count": 1, + "min_size_bytes": 128, + "max_size_bytes": 264, +} +# sizes: 72 is for an empty TA fragment +# 632 is for three TAs with one TP in each of them (72+5*(88+24)) +triggeractivity_frag_params = { + "fragment_type_description": "Trigger Activity", + "fragment_type": "Trigger_Activity", + "expected_fragment_count": 1, + "min_size_bytes": 72, + "max_size_bytes": 632, +} +# sizes: 72 is for an empty TP fragment +# 1032 is for a fragment with 40 TPs in it (72+(40*24)) +triggerprimitive_frag_params = { + "fragment_type_description": "Trigger Primitive", + "fragment_type": "Trigger_Primitive", + "expected_fragment_count": (3 * number_of_readout_apps), + "min_size_bytes": 72, + "max_size_bytes": 1032, +} +hsi_frag_params = { + "fragment_type_description": "HSI", + "fragment_type": "Hardware_Signal", + "expected_fragment_count": 1, + "min_size_bytes": 72, + "max_size_bytes": 100, +} +ignored_logfile_problems = { + "-controller": [ + "Worker with pid \\d+ was terminated due to signal 1", + ], + "connectivity-service": [ + "errorlog: -", + ], +} + +# The next three variable declarations *must* be present as globals in the test +# file. They're read by the "fixtures" in conftest.py to determine how +# to run the config generation and nanorc + +object_databases = ["config/daqsystemtest/integrationtest-objects.data.xml"] + +conf_dict = data_classes.drunc_config() +conf_dict.dro_map_config.n_streams = number_of_data_producers +conf_dict.dro_map_config.n_apps = number_of_readout_apps +conf_dict.op_env = "integtest" +conf_dict.session = "maxfilesize" +conf_dict.tpg_enabled = True +conf_dict.fake_hsi_enabled = True +conf_dict.frame_file = ( + "asset://?checksum=dd156b4895f1b06a06b6ff38e37bd798" # WIBEth All Zeros +) + +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_id=conf_dict.session, + obj_class="Session", + updates={"data_rate_slowdown_factor": data_rate_slowdown_factor}, + ) +) +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="LatencyBuffer", updates={"size": 200000} + ) +) + +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="FakeHSIEventGeneratorConf", + updates={"trigger_rate": 10.0}, + ) +) + +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="HSISignalWindow", + updates={ + "time_before": 1000, + "time_after": 500, + }, + ) +) +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="TCReadoutMap", + obj_id = "def-hsi-tc-map", + updates={ + "time_before": 52000, + "time_after": 1000, + }, + ) +) +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="DataStoreConf", + obj_id="default", + updates={"max_file_size": 725000000}, + ) +) +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="DataStoreConf", + obj_id="default_tp_store_conf", + updates={"max_file_size": 275000000}, + ) +) +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="StreamEmulationParameters", + obj_id="stream-emu", + updates={"TP_rate_per_channel": 5}, + ) +) + +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="DFOConf", updates={"busy_threshold": 10, "free_threshold": 7} + ) +) + + +confgen_arguments = { + "WIBEth_TPG_System": conf_dict, +} + +# The commands to run in nanorc, as a list +nanorc_command_list = ( + "boot conf wait 5".split() + + "start --run-number 101 wait 1 enable-triggers wait 178".split() + + "disable-triggers wait 2 drain-dataflow wait 2 stop-trigger-sources stop ".split() + + "start --run-number 102 wait 1 enable-triggers wait 128".split() + + "disable-triggers wait 2 drain-dataflow wait 2 stop-trigger-sources stop ".split() + + " scrap terminate".split() +) + +# The tests themselves + + +def test_nanorc_success(run_nanorc): + current_test = os.environ.get("PYTEST_CURRENT_TEST") + match_obj = re.search(r".*\[(.+)\].*", current_test) + if match_obj: + current_test = match_obj.group(1) + banner_line = re.sub(".", "=", current_test) + print(banner_line) + print(current_test) + print(banner_line) + # Check that nanorc completed correctly + assert run_nanorc.completed_process.returncode == 0 + + +def test_log_files(run_nanorc): + if check_for_logfile_errors: + # Check that there are no warnings or errors in the log files + assert log_file_checks.logs_are_error_free( + run_nanorc.log_files, True, True, ignored_logfile_problems + ) + + +def test_data_files(run_nanorc): + fragment_check_list = [triggercandidate_frag_params, hsi_frag_params, wibeth_frag_params] + fragment_check_list.append(triggerprimitive_frag_params) + fragment_check_list.append(triggeractivity_frag_params) + + # Run some tests on the output data file + assert len(run_nanorc.data_files) == 6 # three for each run + + all_ok = True + for idx in range(len(run_nanorc.data_files)): + data_file = data_file_checks.DataFile(run_nanorc.data_files[idx]) + all_ok &= data_file_checks.sanity_check(data_file) + all_ok &= data_file_checks.check_file_attributes(data_file) + for jdx in range(len(fragment_check_list)): + all_ok &= data_file_checks.check_fragment_count( + data_file, fragment_check_list[jdx] + ) + all_ok &= data_file_checks.check_fragment_sizes( + data_file, fragment_check_list[jdx] + ) + assert all_ok + + +def test_tpstream_files(run_nanorc): + tpstream_files = run_nanorc.tpset_files + fragment_check_list = [wibeth_tpset_params] # WIBEth + + assert len(tpstream_files) == 6 # three for each run + + all_ok = True + for idx in range(len(tpstream_files)): + data_file = data_file_checks.DataFile(tpstream_files[idx]) + all_ok &= data_file_checks.check_file_attributes(data_file) + for jdx in range(len(fragment_check_list)): + all_ok &= data_file_checks.check_fragment_count( + data_file, fragment_check_list[jdx] + ) + assert all_ok + + +def test_cleanup(run_nanorc): + pathlist_string = "" + filelist_string = "" + for data_file in run_nanorc.data_files: + filelist_string += " " + str(data_file) + if str(data_file.parent) not in pathlist_string: + pathlist_string += " " + str(data_file.parent) + for data_file in run_nanorc.tpset_files: + filelist_string += " " + str(data_file) + if str(data_file.parent) not in pathlist_string: + pathlist_string += " " + str(data_file.parent) + + if pathlist_string and filelist_string: + print("============================================") + print("Listing the hdf5 files before deleting them:") + print("============================================") + + os.system(f"df -h {pathlist_string}") + print("--------------------") + os.system(f"ls -alF {filelist_string}") + + for data_file in run_nanorc.data_files: + data_file.unlink() + for data_file in run_nanorc.tpset_files: + data_file.unlink() + + print("--------------------") + os.system(f"df -h {pathlist_string}") + print("============================================") From 49ef02800c18c7ac1c8ed65dfd9632417757b12e Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Tue, 20 May 2025 11:14:46 -0500 Subject: [PATCH 06/18] Renamed the multi_output_file_test.py as max_file_size_test.py to reflect its current purpose, updated some variable names inside it, and added the cleanup of TP Stream files after the test completes. --- integtest/multi_output_file_test.py | 271 ---------------------------- 1 file changed, 271 deletions(-) delete mode 100644 integtest/multi_output_file_test.py diff --git a/integtest/multi_output_file_test.py b/integtest/multi_output_file_test.py deleted file mode 100644 index 5ed5d2e3..00000000 --- a/integtest/multi_output_file_test.py +++ /dev/null @@ -1,271 +0,0 @@ -import pytest -import os -import re -import copy -import urllib.request - -import integrationtest.data_file_checks as data_file_checks -import integrationtest.log_file_checks as log_file_checks -import integrationtest.data_classes as data_classes - -pytest_plugins = "integrationtest.integrationtest_drunc" - -# Values that help determine the running conditions -number_of_data_producers = 2 -number_of_readout_apps = 3 -data_rate_slowdown_factor = 1 - -# Default values for validation parameters -check_for_logfile_errors = True - -wibeth_frag_params = { - "fragment_type_description": "WIBEth", - "fragment_type": "WIBEth", - "expected_fragment_count": (number_of_data_producers * number_of_readout_apps), - "min_size_bytes": 7272, - "max_size_bytes": 194472, -} -wibeth_tpset_params = { - "fragment_type_description": "TP Stream", - "fragment_type": "Trigger_Primitive", - "expected_fragment_count": number_of_readout_apps * 3, - "frag_counts_by_record_ordinal": {"first": {"min_count": 1, "max_count": number_of_readout_apps * 3}, - "default": {"min_count": number_of_readout_apps * 3, "max_count": number_of_readout_apps * 3} }, - "min_size_bytes": 0, # not checked - "max_size_bytes": 0, # not checked - "debug_mask": 0x0, -} -# sizes: 128 is for one TC with zero TAs inside it (72+56) -# 208 is for one TC with one TA inside it (72+56+80) -# 264 is for two TCs with one TA in one of them (72+56+80+56) -triggercandidate_frag_params = { - "fragment_type_description": "Trigger Candidate", - "fragment_type": "Trigger_Candidate", - "expected_fragment_count": 1, - "min_size_bytes": 128, - "max_size_bytes": 264, -} -# sizes: 72 is for an empty TA fragment -# 632 is for three TAs with one TP in each of them (72+5*(88+24)) -triggeractivity_frag_params = { - "fragment_type_description": "Trigger Activity", - "fragment_type": "Trigger_Activity", - "expected_fragment_count": 1, - "min_size_bytes": 72, - "max_size_bytes": 632, -} -# sizes: 72 is for an empty TP fragment -# 1032 is for a fragment with 40 TPs in it (72+(40*24)) -triggerprimitive_frag_params = { - "fragment_type_description": "Trigger Primitive", - "fragment_type": "Trigger_Primitive", - "expected_fragment_count": (3 * number_of_readout_apps), - "min_size_bytes": 72, - "max_size_bytes": 1032, -} -hsi_frag_params = { - "fragment_type_description": "HSI", - "fragment_type": "Hardware_Signal", - "expected_fragment_count": 1, - "min_size_bytes": 72, - "max_size_bytes": 100, -} -ignored_logfile_problems = { - "-controller": [ - "Worker with pid \\d+ was terminated due to signal 1", - ], - "connectivity-service": [ - "errorlog: -", - ], -} - -# The next three variable declarations *must* be present as globals in the test -# file. They're read by the "fixtures" in conftest.py to determine how -# to run the config generation and nanorc - -object_databases = ["config/daqsystemtest/integrationtest-objects.data.xml"] - -wibtpg_conf = data_classes.drunc_config() -wibtpg_conf.dro_map_config.n_streams = number_of_data_producers -wibtpg_conf.dro_map_config.n_apps = number_of_readout_apps -wibtpg_conf.op_env = "integtest" -wibtpg_conf.session = "multioutput" -wibtpg_conf.tpg_enabled = True -wibtpg_conf.fake_hsi_enabled = True -wibtpg_conf.frame_file = ( - "asset://?checksum=dd156b4895f1b06a06b6ff38e37bd798" # WIBEth All Zeros -) - -wibtpg_conf.config_substitutions.append( - data_classes.config_substitution( - obj_id=wibtpg_conf.session, - obj_class="Session", - updates={"data_rate_slowdown_factor": data_rate_slowdown_factor}, - ) -) -wibtpg_conf.config_substitutions.append( - data_classes.config_substitution( - obj_class="LatencyBuffer", updates={"size": 200000} - ) -) - -wibtpg_conf.config_substitutions.append( - data_classes.config_substitution( - obj_class="FakeHSIEventGeneratorConf", - updates={"trigger_rate": 10.0}, - ) -) - -wibtpg_conf.config_substitutions.append( - data_classes.config_substitution( - obj_class="HSISignalWindow", - updates={ - "time_before": 1000, - "time_after": 500, - }, - ) -) -wibtpg_conf.config_substitutions.append( - data_classes.config_substitution( - obj_class="TCReadoutMap", - obj_id = "def-hsi-tc-map", - updates={ - "time_before": 52000, - "time_after": 1000, - }, - ) -) -wibtpg_conf.config_substitutions.append( - data_classes.config_substitution( - obj_class="DataStoreConf", - obj_id="default", - updates={"max_file_size": 725000000}, - ) -) -wibtpg_conf.config_substitutions.append( - data_classes.config_substitution( - obj_class="DataStoreConf", - obj_id="default_tp_store_conf", - updates={"max_file_size": 275000000}, - ) -) -wibtpg_conf.config_substitutions.append( - data_classes.config_substitution( - obj_class="StreamEmulationParameters", - obj_id="stream-emu", - updates={"TP_rate_per_channel": 5}, - ) -) - -wibtpg_conf.config_substitutions.append( - data_classes.config_substitution( - obj_class="DFOConf", updates={"busy_threshold": 10, "free_threshold": 7} - ) -) - - -confgen_arguments = { - "WIBEth_TPG_System": wibtpg_conf, -} - -# The commands to run in nanorc, as a list -nanorc_command_list = ( - "boot conf wait 5".split() - + "start --run-number 101 wait 1 enable-triggers wait 178".split() - + "disable-triggers wait 2 drain-dataflow wait 2 stop-trigger-sources stop ".split() - + "start --run-number 102 wait 1 enable-triggers wait 128".split() - + "disable-triggers wait 2 drain-dataflow wait 2 stop-trigger-sources stop ".split() - + " scrap terminate".split() -) - -# The tests themselves - - -def test_nanorc_success(run_nanorc): - current_test = os.environ.get("PYTEST_CURRENT_TEST") - match_obj = re.search(r".*\[(.+)\].*", current_test) - if match_obj: - current_test = match_obj.group(1) - banner_line = re.sub(".", "=", current_test) - print(banner_line) - print(current_test) - print(banner_line) - # Check that nanorc completed correctly - assert run_nanorc.completed_process.returncode == 0 - - -def test_log_files(run_nanorc): - if check_for_logfile_errors: - # Check that there are no warnings or errors in the log files - assert log_file_checks.logs_are_error_free( - run_nanorc.log_files, True, True, ignored_logfile_problems - ) - - -def test_data_files(run_nanorc): - fragment_check_list = [triggercandidate_frag_params, hsi_frag_params, wibeth_frag_params] - fragment_check_list.append(triggerprimitive_frag_params) - fragment_check_list.append(triggeractivity_frag_params) - - # Run some tests on the output data file - assert len(run_nanorc.data_files) == 6 # three for each run - - all_ok = True - for idx in range(len(run_nanorc.data_files)): - data_file = data_file_checks.DataFile(run_nanorc.data_files[idx]) - all_ok &= data_file_checks.sanity_check(data_file) - all_ok &= data_file_checks.check_file_attributes(data_file) - for jdx in range(len(fragment_check_list)): - all_ok &= data_file_checks.check_fragment_count( - data_file, fragment_check_list[jdx] - ) - all_ok &= data_file_checks.check_fragment_sizes( - data_file, fragment_check_list[jdx] - ) - assert all_ok - - -def test_tpstream_files(run_nanorc): - tpstream_files = run_nanorc.tpset_files - fragment_check_list = [wibeth_tpset_params] # WIBEth - - assert len(tpstream_files) == 6 # three for each run - - all_ok = True - for idx in range(len(tpstream_files)): - data_file = data_file_checks.DataFile(tpstream_files[idx]) - all_ok &= data_file_checks.check_file_attributes(data_file) - for jdx in range(len(fragment_check_list)): - all_ok &= data_file_checks.check_fragment_count( - data_file, fragment_check_list[jdx] - ) - assert all_ok - - -def test_cleanup(run_nanorc): - pathlist_string = "" - filelist_string = "" - for data_file in run_nanorc.data_files: - filelist_string += " " + str(data_file) - if str(data_file.parent) not in pathlist_string: - pathlist_string += " " + str(data_file.parent) - for data_file in run_nanorc.tpset_files: - filelist_string += " " + str(data_file) - if str(data_file.parent) not in pathlist_string: - pathlist_string += " " + str(data_file.parent) - - if pathlist_string and filelist_string: - print("============================================") - print("Listing the hdf5 files before deleting them:") - print("============================================") - - os.system(f"df -h {pathlist_string}") - print("--------------------") - os.system(f"ls -alF {filelist_string}") - - for data_file in run_nanorc.data_files: - data_file.unlink() - - print("--------------------") - os.system(f"df -h {pathlist_string}") - print("============================================") From cc3e806f4dc88b4b662ab0f0ad47c4716f9bf561 Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Tue, 20 May 2025 11:16:29 -0500 Subject: [PATCH 07/18] Created a multiple_data_writers_test.py to test the use of multiple data writers in a single Dataflow App. (This functionality accidentally got lost in the conversion form v4 to v5.) --- integtest/multiple_data_writers_test.py | 174 ++++++++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 integtest/multiple_data_writers_test.py diff --git a/integtest/multiple_data_writers_test.py b/integtest/multiple_data_writers_test.py new file mode 100644 index 00000000..26b8de0f --- /dev/null +++ b/integtest/multiple_data_writers_test.py @@ -0,0 +1,174 @@ +import pytest +import os +import re +import copy +import urllib.request + +import integrationtest.data_file_checks as data_file_checks +import integrationtest.log_file_checks as log_file_checks +import integrationtest.data_classes as data_classes + +pytest_plugins = "integrationtest.integrationtest_drunc" + +# Values that help determine the running conditions +number_of_data_producers = 2 +number_of_readout_apps = 3 + +# Default values for validation parameters +check_for_logfile_errors = True + +wibeth_frag_params = { + "fragment_type_description": "WIBEth", + "fragment_type": "WIBEth", + "expected_fragment_count": (number_of_data_producers * number_of_readout_apps), + "min_size_bytes": 187272, + "max_size_bytes": 194472, +} +# sizes: 128 is for one TC with zero TAs inside it (72+56) +# 208 is for one TC with one TA inside it (72+56+80) +# 264 is for two TCs with one TA in one of them (72+56+80+56) +triggercandidate_frag_params = { + "fragment_type_description": "Trigger Candidate", + "fragment_type": "Trigger_Candidate", + "expected_fragment_count": 1, + "min_size_bytes": 128, + "max_size_bytes": 128, +} +triggeractivity_frag_params = { + "fragment_type_description": "Trigger Activity", + "fragment_type": "Trigger_Activity", + "expected_fragment_count": 0, + "min_size_bytes": 72, + "max_size_bytes": 632, +} +triggerprimitive_frag_params = { + "fragment_type_description": "Trigger Primitive", + "fragment_type": "Trigger_Primitive", + "expected_fragment_count": 0, + "min_size_bytes": 72, + "max_size_bytes": 1032, +} +hsi_frag_params = { + "fragment_type_description": "HSI", + "fragment_type": "Hardware_Signal", + "expected_fragment_count": 1, + "min_size_bytes": 100, + "max_size_bytes": 100, +} +ignored_logfile_problems = { + "-controller": [ + "Worker with pid \\d+ was terminated due to signal 1", + ], + "connectivity-service": [ + "errorlog: -", + ], +} + +# The next three variable declarations *must* be present as globals in the test +# file. They're read by the "fixtures" in conftest.py to determine how +# to run the config generation and nanorc + +object_databases = ["config/daqsystemtest/integrationtest-objects.data.xml"] + +conf_dict = data_classes.drunc_config() +conf_dict.dro_map_config.n_streams = number_of_data_producers +conf_dict.dro_map_config.n_apps = number_of_readout_apps +conf_dict.op_env = "integtest" +conf_dict.session = "multidatawriter" +conf_dict.fake_hsi_enabled = True +conf_dict.n_data_writers = 3 + +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="FakeHSIEventGeneratorConf", + updates={"trigger_rate": 10.0}, + ) +) + +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="HSISignalWindow", + updates={ + "time_before": 1000, + "time_after": 500, + }, + ) +) +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="TCReadoutMap", + obj_id = "def-hsi-tc-map", + updates={ + "time_before": 52000, + "time_after": 1000, + }, + ) +) + +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="QueueDescriptor", + obj_id="trigger-records", + updates={"queue_type": "kFollyMPMCQueue"}, + ) +) + + +confgen_arguments = { + "WIBEth_System": conf_dict, +} + +# The commands to run in nanorc, as a list +nanorc_command_list = ( + "boot conf wait 5".split() + + "start --run-number 101 wait 1 enable-triggers wait 30".split() + + "disable-triggers wait 2 drain-dataflow wait 2 stop-trigger-sources stop ".split() + + "start --run-number 102 wait 1 enable-triggers wait 30".split() + + "disable-triggers wait 2 drain-dataflow wait 2 stop-trigger-sources stop ".split() + + " scrap terminate".split() +) + +# The tests themselves + +def test_nanorc_success(run_nanorc): + current_test = os.environ.get("PYTEST_CURRENT_TEST") + match_obj = re.search(r".*\[(.+)\].*", current_test) + if match_obj: + current_test = match_obj.group(1) + banner_line = re.sub(".", "=", current_test) + print(banner_line) + print(current_test) + print(banner_line) + # Check that nanorc completed correctly + assert run_nanorc.completed_process.returncode == 0 + + +def test_log_files(run_nanorc): + if check_for_logfile_errors: + # Check that there are no warnings or errors in the log files + assert log_file_checks.logs_are_error_free( + run_nanorc.log_files, True, True, ignored_logfile_problems + ) + + +def test_data_files(run_nanorc): + fragment_check_list = [triggercandidate_frag_params, hsi_frag_params, wibeth_frag_params] + fragment_check_list.append(triggerprimitive_frag_params) + fragment_check_list.append(triggeractivity_frag_params) + + # Run some tests on the output data file + assert len(run_nanorc.data_files) == 6 # three for each run + + all_ok = True + for idx in range(len(run_nanorc.data_files)): + data_file = data_file_checks.DataFile(run_nanorc.data_files[idx]) + all_ok &= data_file_checks.sanity_check(data_file) + all_ok &= data_file_checks.check_file_attributes(data_file) + for jdx in range(len(fragment_check_list)): + all_ok &= data_file_checks.check_fragment_count( + data_file, fragment_check_list[jdx] + ) + all_ok &= data_file_checks.check_fragment_sizes( + data_file, fragment_check_list[jdx] + ) + assert all_ok From c726032d9e801cc0b345905a584e40a174011f0e Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Tue, 20 May 2025 11:18:28 -0500 Subject: [PATCH 08/18] Committing a *preliminary* test of HDF5 compression. --- integtest/hdf5_compression_test.py | 322 +++++++++++++++++++++++++++++ 1 file changed, 322 insertions(+) create mode 100644 integtest/hdf5_compression_test.py diff --git a/integtest/hdf5_compression_test.py b/integtest/hdf5_compression_test.py new file mode 100644 index 00000000..a8b09f7a --- /dev/null +++ b/integtest/hdf5_compression_test.py @@ -0,0 +1,322 @@ +import pytest +import os +import re +import copy +import urllib.request + +import integrationtest.data_file_checks as data_file_checks +import integrationtest.log_file_checks as log_file_checks +import integrationtest.data_classes as data_classes + +pytest_plugins = "integrationtest.integrationtest_drunc" + +# Values that help determine the running conditions +number_of_data_producers = 2 +number_of_readout_apps = 3 + +# Default values for validation parameters +check_for_logfile_errors = True + +daphne_frag_params = { + "fragment_type_description": "DAPHNE", + "fragment_type": "DAPHNE", + "expected_fragment_count": number_of_data_producers * number_of_readout_apps, + "min_size_bytes": 1936, + "max_size_bytes": 230000, +# "frag_sizes_by_TC_type": {"kPrescale": {"min_size_bytes": 1936, "max_size_bytes": 30000}, +# "kRandom": {"min_size_bytes": 220000, "max_size_bytes": 230000}, +# "default": {"min_size_bytes": 1936, "max_size_bytes": 230000} } +} +daphne_triggerprimitive_frag_params = { + "fragment_type_description": "Trigger Primitive", + "fragment_type": "Trigger_Primitive", + "expected_fragment_count": number_of_readout_apps, + "min_size_bytes": 72, + "max_size_bytes": 230000, +# "frag_sizes_by_TC_type": {"kPrescale": {"min_size_bytes": 72, "max_size_bytes": 10000}, +# "kRandom": {"min_size_bytes": 220000, "max_size_bytes": 230000}, +# "default": {"min_size_bytes": 72, "max_size_bytes": 230000} } +} +daphne_tpset_params = { + "fragment_type_description": "TP Stream", + "fragment_type": "Trigger_Primitive", + "expected_fragment_count": number_of_readout_apps, + "frag_counts_by_record_ordinal": {"first": {"min_count": 1, "max_count": number_of_readout_apps}, + "default": {"min_count": number_of_readout_apps, "max_count": number_of_readout_apps} }, + "min_size_bytes": 200, + "max_size_bytes": 210, + "debug_mask": 0x0, +# "frag_sizes_by_record_ordinal": { "first": {"min_size_bytes": 96, "max_size_bytes": 120000}, +# "second": {"min_size_bytes": 96, "max_size_bytes": 120000}, +# "last": {"min_size_bytes": 96, "max_size_bytes": 120000}, +# "default": {"min_size_bytes": 80000, "max_size_bytes": 120000} } +} + +#wibeth_frag_params = { +# "fragment_type_description": "WIBEth", +# "fragment_type": "WIBEth", +# "expected_fragment_count": (number_of_data_producers * number_of_readout_apps), +# "min_size_bytes": 7272, +# "max_size_bytes": 194472, +#} +#wibeth_tpset_params = { +# "fragment_type_description": "TP Stream", +# "fragment_type": "Trigger_Primitive", +# "expected_fragment_count": number_of_readout_apps * 3, +# "frag_counts_by_record_ordinal": {"first": {"min_count": 1, "max_count": number_of_readout_apps * 3}, +# "default": {"min_count": number_of_readout_apps * 3, "max_count": number_of_readout_apps * 3} }, +# "min_size_bytes": 0, # not checked +# "max_size_bytes": 0, # not checked +# "debug_mask": 0x0, +#} + +# sizes: 128 is for one TC with zero TAs inside it (72+56) +# 208 is for one TC with one TA inside it (72+56+80) +# 264 is for two TCs with one TA in one of them (72+56+80+56) +triggercandidate_frag_params = { + "fragment_type_description": "Trigger Candidate", + "fragment_type": "Trigger_Candidate", + "expected_fragment_count": 1, + "min_size_bytes": 128, + "max_size_bytes": 264, +} +# sizes: 72 is for an empty TA fragment +# 632 is for three TAs with one TP in each of them (72+5*(88+24)) +triggeractivity_frag_params = { + "fragment_type_description": "Trigger Activity", + "fragment_type": "Trigger_Activity", + "expected_fragment_count": 1, + "min_size_bytes": 72, + "max_size_bytes": 632, +} +# sizes: 72 is for an empty TP fragment +# 1032 is for a fragment with 40 TPs in it (72+(40*24)) +triggerprimitive_frag_params = { + "fragment_type_description": "Trigger Primitive", + "fragment_type": "Trigger_Primitive", + "expected_fragment_count": number_of_readout_apps, + "min_size_bytes": 72, + "max_size_bytes": 1032, +} +hsi_frag_params = { + "fragment_type_description": "HSI", + "fragment_type": "Hardware_Signal", + "expected_fragment_count": 1, + "min_size_bytes": 72, + "max_size_bytes": 100, +} +ignored_logfile_problems = { + "-controller": [ + "Worker with pid \\d+ was terminated due to signal 1", + ], + "connectivity-service": [ + "errorlog: -", + ], +} + +# The next three variable declarations *must* be present as globals in the test +# file. They're read by the "fixtures" in conftest.py to determine how +# to run the config generation and nanorc + +object_databases = ["config/daqsystemtest/integrationtest-objects.data.xml"] + +conf_dict = data_classes.drunc_config() +conf_dict.dro_map_config.n_streams = number_of_data_producers +conf_dict.dro_map_config.n_apps = number_of_readout_apps +conf_dict.op_env = "integtest" +conf_dict.session = "hdf5compression" +conf_dict.tpg_enabled = True +conf_dict.fake_hsi_enabled = True +conf_dict.dro_map_config.det_id = 2 # det_id = 2 for kHD_PDS +conf_dict.frame_file = "asset://?checksum=a8990a9eb3a505d4ded62dfdfa9e2681" # run 36012 +#conf_dict.frame_file = "file:///home/nfs/biery/dunedaq/12MayFDv5.3.2DevInstrUpdate/sourcecode/dfmodules/integtest/np02vdcoldbox_run035227_sample_hd_pds.bin" + +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="TAMakerPrescaleAlgorithm", + obj_id="dummy-ta-maker", + updates={"prescale": 2000}, + ) +) + +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="FakeHSIEventGeneratorConf", + updates={"trigger_rate": 10.0}, + ) +) + +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="HSISignalWindow", + updates={ + "time_before": 1000, + "time_after": 500, + }, + ) +) +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="TCReadoutMap", + obj_id = "def-hsi-tc-map", + updates={ + "time_before": 120000, + "time_after": 1000, + }, + ) +) +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="DataStoreConf", + obj_id="default", + updates={"max_file_size": 130000000}, + ) +) +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="DataStoreConf", + obj_id="default_tp_store_conf", + updates={"max_file_size": 100000000}, + ) +) +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="DataStoreConf", + obj_id="default", + updates={"compression_level": 1}, + ) +) +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="DataStoreConf", + obj_id="default_tp_store_conf", + updates={"compression_level": 1}, + ) +) + +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="DFOConf", updates={"busy_threshold": 10, "free_threshold": 7} + ) +) +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="TCDataProcessor", + obj_id="def-tc-processor", + updates={"merge_overlapping_tcs": 0}, + ) +) +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="LatencyBuffer", updates={"size": 5592000} + ) +) + + +confgen_arguments = { + "DAPHNE_TPG_System": conf_dict, +} + +# The commands to run in nanorc, as a list +nanorc_command_list = ( + "boot conf wait 5".split() + + "start --run-number 101 wait 1 enable-triggers wait 50".split() + + "disable-triggers wait 2 drain-dataflow wait 2 stop-trigger-sources stop ".split() + + "start --run-number 102 wait 1 enable-triggers wait 45".split() + + "disable-triggers wait 2 drain-dataflow wait 2 stop-trigger-sources stop ".split() + + " scrap terminate".split() +) + +# The tests themselves + + +def test_nanorc_success(run_nanorc): + current_test = os.environ.get("PYTEST_CURRENT_TEST") + match_obj = re.search(r".*\[(.+)\].*", current_test) + if match_obj: + current_test = match_obj.group(1) + banner_line = re.sub(".", "=", current_test) + print(banner_line) + print(current_test) + print(banner_line) + # Check that nanorc completed correctly + assert run_nanorc.completed_process.returncode == 0 + + +def test_log_files(run_nanorc): + if check_for_logfile_errors: + # Check that there are no warnings or errors in the log files + assert log_file_checks.logs_are_error_free( + run_nanorc.log_files, True, True, ignored_logfile_problems + ) + + +def test_data_files(run_nanorc): + fragment_check_list = [triggercandidate_frag_params, hsi_frag_params, daphne_frag_params] + fragment_check_list.append(daphne_triggerprimitive_frag_params) + fragment_check_list.append(triggeractivity_frag_params) + + # Run some tests on the output data file + assert len(run_nanorc.data_files) == 10 # three for each run + + all_ok = True + for idx in range(len(run_nanorc.data_files)): + data_file = data_file_checks.DataFile(run_nanorc.data_files[idx]) + all_ok &= data_file_checks.sanity_check(data_file) + all_ok &= data_file_checks.check_file_attributes(data_file) + for jdx in range(len(fragment_check_list)): + all_ok &= data_file_checks.check_fragment_count( + data_file, fragment_check_list[jdx] + ) + all_ok &= data_file_checks.check_fragment_sizes( + data_file, fragment_check_list[jdx] + ) + assert all_ok + + +def test_tpstream_files(run_nanorc): + tpstream_files = run_nanorc.tpset_files + fragment_check_list = [daphne_tpset_params] + + assert len(tpstream_files) == 6 # three for each run + + all_ok = True + for idx in range(len(tpstream_files)): + data_file = data_file_checks.DataFile(tpstream_files[idx]) + all_ok &= data_file_checks.check_file_attributes(data_file) + for jdx in range(len(fragment_check_list)): + all_ok &= data_file_checks.check_fragment_count( + data_file, fragment_check_list[jdx] + ) + assert all_ok + + +def test_cleanup(run_nanorc): + pathlist_string = "" + filelist_string = "" + for data_file in run_nanorc.data_files: + filelist_string += " " + str(data_file) + if str(data_file.parent) not in pathlist_string: + pathlist_string += " " + str(data_file.parent) + for data_file in run_nanorc.tpset_files: + filelist_string += " " + str(data_file) + if str(data_file.parent) not in pathlist_string: + pathlist_string += " " + str(data_file.parent) + + if pathlist_string and filelist_string: + print("============================================", flush=True) + print("Listing the hdf5 files before deleting them:", flush=True) + print("============================================", flush=True) + + os.system(f"df -h {pathlist_string}") + print("--------------------", flush=True) + os.system(f"ls -alF {filelist_string}") + + #for data_file in run_nanorc.data_files: + # data_file.unlink() + #for data_file in run_nanorc.tpset_files: + # data_file.unlink() + + print("--------------------", flush=True) + os.system(f"df -h {pathlist_string}") + print("============================================", flush=True) From 625590d6a301dd31b6137adc60e03785b1187e47 Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Tue, 20 May 2025 11:38:55 -0500 Subject: [PATCH 09/18] Add the 'flush' option to print statements associated with cleaning up data files in various integtests. --- integtest/insufficient_disk_space_test.py | 12 ++++++------ integtest/large_trigger_record_test.py | 12 ++++++------ integtest/max_file_size_test.py | 12 ++++++------ 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/integtest/insufficient_disk_space_test.py b/integtest/insufficient_disk_space_test.py index 7d65f7ae..66369a0f 100644 --- a/integtest/insufficient_disk_space_test.py +++ b/integtest/insufficient_disk_space_test.py @@ -257,17 +257,17 @@ def test_cleanup(run_nanorc): pathlist_string += " " + str(data_file.parent) if pathlist_string and filelist_string: - print("============================================") - print("Listing the hdf5 files before deleting them:") - print("============================================") + print("============================================", flush=True) + print("Listing the hdf5 files before deleting them:", flush=True) + print("============================================", flush=True) os.system(f"df -h {pathlist_string}") - print("--------------------") + print("--------------------", flush=True) os.system(f"ls -alF {filelist_string}") for data_file in run_nanorc.data_files: data_file.unlink() - print("--------------------") + print("--------------------", flush=True) os.system(f"df -h {pathlist_string}") - print("============================================") + print("============================================", flush=True) diff --git a/integtest/large_trigger_record_test.py b/integtest/large_trigger_record_test.py index ada9d309..30a6a7b5 100644 --- a/integtest/large_trigger_record_test.py +++ b/integtest/large_trigger_record_test.py @@ -260,17 +260,17 @@ def test_cleanup(run_nanorc): pathlist_string += " " + str(data_file.parent) if pathlist_string and filelist_string: - print("============================================") - print("Listing the hdf5 files before deleting them:") - print("============================================") + print("============================================", flush=True) + print("Listing the hdf5 files before deleting them:", flush=True) + print("============================================", flush=True) os.system(f"df -h {pathlist_string}") - print("--------------------") + print("--------------------", flush=True) os.system(f"ls -alF {filelist_string}") for data_file in run_nanorc.data_files: data_file.unlink() - print("--------------------") + print("--------------------", flush=True) os.system(f"df -h {pathlist_string}") - print("============================================") + print("============================================", flush=True) diff --git a/integtest/max_file_size_test.py b/integtest/max_file_size_test.py index cd3a9433..c98bd744 100644 --- a/integtest/max_file_size_test.py +++ b/integtest/max_file_size_test.py @@ -255,12 +255,12 @@ def test_cleanup(run_nanorc): pathlist_string += " " + str(data_file.parent) if pathlist_string and filelist_string: - print("============================================") - print("Listing the hdf5 files before deleting them:") - print("============================================") + print("============================================", flush=True) + print("Listing the hdf5 files before deleting them:", flush=True) + print("============================================", flush=True) os.system(f"df -h {pathlist_string}") - print("--------------------") + print("--------------------", flush=True) os.system(f"ls -alF {filelist_string}") for data_file in run_nanorc.data_files: @@ -268,6 +268,6 @@ def test_cleanup(run_nanorc): for data_file in run_nanorc.tpset_files: data_file.unlink() - print("--------------------") + print("--------------------", flush=True) os.system(f"df -h {pathlist_string}") - print("============================================") + print("============================================", flush=True) From de8c491a7cf12133b8841463594bb52350c0c30f Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Tue, 20 May 2025 12:42:18 -0500 Subject: [PATCH 10/18] Tweaked the hdf5_compression_test to provide better feedback when the wrong number of files is produced. --- integtest/hdf5_compression_test.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/integtest/hdf5_compression_test.py b/integtest/hdf5_compression_test.py index a8b09f7a..2b60eaa3 100644 --- a/integtest/hdf5_compression_test.py +++ b/integtest/hdf5_compression_test.py @@ -176,7 +176,7 @@ data_classes.config_substitution( obj_class="DataStoreConf", obj_id="default_tp_store_conf", - updates={"max_file_size": 100000000}, + updates={"max_file_size": 80000000}, ) ) conf_dict.config_substitutions.append( @@ -257,9 +257,12 @@ def test_data_files(run_nanorc): fragment_check_list.append(triggeractivity_frag_params) # Run some tests on the output data file - assert len(run_nanorc.data_files) == 10 # three for each run + all_ok = len(run_nanorc.data_files) == 10 # three for each run - Fix this, KAB, 20-May + if all_ok: + print("\N{WHITE HEAVY CHECK MARK} The correct number of raw data files was found (10)") + else: + print(f"\N{POLICE CARS REVOLVING LIGHT} An incorrect number of raw data files was found, expected 10, found {len(run_nanorc.data_files)} \N{POLICE CARS REVOLVING LIGHT}") - all_ok = True for idx in range(len(run_nanorc.data_files)): data_file = data_file_checks.DataFile(run_nanorc.data_files[idx]) all_ok &= data_file_checks.sanity_check(data_file) @@ -278,9 +281,12 @@ def test_tpstream_files(run_nanorc): tpstream_files = run_nanorc.tpset_files fragment_check_list = [daphne_tpset_params] - assert len(tpstream_files) == 6 # three for each run + all_ok = len(tpstream_files) == 6 # three for each run + if all_ok: + print("\N{WHITE HEAVY CHECK MARK} The correct number of TP-stream data files was found (6)") + else: + print(f"\N{POLICE CARS REVOLVING LIGHT} An incorrect number of TP-stream data files was found, expected 6, found {len(tpstream_files)} \N{POLICE CARS REVOLVING LIGHT}") - all_ok = True for idx in range(len(tpstream_files)): data_file = data_file_checks.DataFile(tpstream_files[idx]) all_ok &= data_file_checks.check_file_attributes(data_file) From cd9af21429d4e2facdddbf6df011197e18661e6d Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Tue, 20 May 2025 17:19:30 -0500 Subject: [PATCH 11/18] Made a few tweaks to hdf5_compression_test.py to get it to run more reliably. --- integtest/hdf5_compression_test.py | 47 +++++++++++++++++------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/integtest/hdf5_compression_test.py b/integtest/hdf5_compression_test.py index 2b60eaa3..d69d207d 100644 --- a/integtest/hdf5_compression_test.py +++ b/integtest/hdf5_compression_test.py @@ -10,6 +10,10 @@ pytest_plugins = "integrationtest.integrationtest_drunc" +# 20-May-2025, KAB: tweak the print() statement default behavior so that it always flushes the output. +import functools +print = functools.partial(print, flush=True) + # Values that help determine the running conditions number_of_data_producers = 2 number_of_readout_apps = 3 @@ -41,7 +45,8 @@ "fragment_type_description": "TP Stream", "fragment_type": "Trigger_Primitive", "expected_fragment_count": number_of_readout_apps, - "frag_counts_by_record_ordinal": {"first": {"min_count": 1, "max_count": number_of_readout_apps}, + "frag_counts_by_record_ordinal": { "first": {"min_count": 1, "max_count": number_of_readout_apps}, + "last": {"min_count": 1, "max_count": number_of_readout_apps}, "default": {"min_count": number_of_readout_apps, "max_count": number_of_readout_apps} }, "min_size_bytes": 200, "max_size_bytes": 210, @@ -169,21 +174,21 @@ data_classes.config_substitution( obj_class="DataStoreConf", obj_id="default", - updates={"max_file_size": 130000000}, + updates={"max_file_size": 400000000}, ) ) conf_dict.config_substitutions.append( data_classes.config_substitution( obj_class="DataStoreConf", obj_id="default_tp_store_conf", - updates={"max_file_size": 80000000}, + updates={"max_file_size": 200000000}, ) ) conf_dict.config_substitutions.append( data_classes.config_substitution( obj_class="DataStoreConf", obj_id="default", - updates={"compression_level": 1}, + updates={"compression_level": 5}, ) ) conf_dict.config_substitutions.append( @@ -196,7 +201,7 @@ conf_dict.config_substitutions.append( data_classes.config_substitution( - obj_class="DFOConf", updates={"busy_threshold": 10, "free_threshold": 7} + obj_class="DFOConf", updates={"busy_threshold": 16, "free_threshold": 12} ) ) conf_dict.config_substitutions.append( @@ -220,9 +225,9 @@ # The commands to run in nanorc, as a list nanorc_command_list = ( "boot conf wait 5".split() - + "start --run-number 101 wait 1 enable-triggers wait 50".split() + + "start --run-number 101 wait 1 enable-triggers wait 100".split() + "disable-triggers wait 2 drain-dataflow wait 2 stop-trigger-sources stop ".split() - + "start --run-number 102 wait 1 enable-triggers wait 45".split() + + "start --run-number 102 wait 1 enable-triggers wait 100".split() + "disable-triggers wait 2 drain-dataflow wait 2 stop-trigger-sources stop ".split() + " scrap terminate".split() ) @@ -257,11 +262,12 @@ def test_data_files(run_nanorc): fragment_check_list.append(triggeractivity_frag_params) # Run some tests on the output data file - all_ok = len(run_nanorc.data_files) == 10 # three for each run - Fix this, KAB, 20-May + all_ok = len(run_nanorc.data_files) == 6 # three for each run + print("") # Clear potential dot from pytest if all_ok: - print("\N{WHITE HEAVY CHECK MARK} The correct number of raw data files was found (10)") + print("\N{WHITE HEAVY CHECK MARK} The correct number of raw data files was found (6)") else: - print(f"\N{POLICE CARS REVOLVING LIGHT} An incorrect number of raw data files was found, expected 10, found {len(run_nanorc.data_files)} \N{POLICE CARS REVOLVING LIGHT}") + print(f"\N{POLICE CARS REVOLVING LIGHT} An incorrect number of raw data files was found, expected 6, found {len(run_nanorc.data_files)} \N{POLICE CARS REVOLVING LIGHT}") for idx in range(len(run_nanorc.data_files)): data_file = data_file_checks.DataFile(run_nanorc.data_files[idx]) @@ -282,6 +288,7 @@ def test_tpstream_files(run_nanorc): fragment_check_list = [daphne_tpset_params] all_ok = len(tpstream_files) == 6 # three for each run + print("") # Clear potential dot from pytest if all_ok: print("\N{WHITE HEAVY CHECK MARK} The correct number of TP-stream data files was found (6)") else: @@ -310,19 +317,19 @@ def test_cleanup(run_nanorc): pathlist_string += " " + str(data_file.parent) if pathlist_string and filelist_string: - print("============================================", flush=True) - print("Listing the hdf5 files before deleting them:", flush=True) - print("============================================", flush=True) + print("============================================") + print("Listing the hdf5 files before deleting them:") + print("============================================") os.system(f"df -h {pathlist_string}") - print("--------------------", flush=True) + print("--------------------") os.system(f"ls -alF {filelist_string}") - #for data_file in run_nanorc.data_files: - # data_file.unlink() - #for data_file in run_nanorc.tpset_files: - # data_file.unlink() + for data_file in run_nanorc.data_files: + data_file.unlink() + for data_file in run_nanorc.tpset_files: + data_file.unlink() - print("--------------------", flush=True) + print("--------------------") os.system(f"df -h {pathlist_string}") - print("============================================", flush=True) + print("============================================") From 827282f224c7c41e4fdf8749f3c547957468051c Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Tue, 20 May 2025 20:58:16 -0500 Subject: [PATCH 12/18] slightly modified the hdf5_compression_test.py TP stream max_file_size --- integtest/hdf5_compression_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integtest/hdf5_compression_test.py b/integtest/hdf5_compression_test.py index d69d207d..2698671f 100644 --- a/integtest/hdf5_compression_test.py +++ b/integtest/hdf5_compression_test.py @@ -181,7 +181,7 @@ data_classes.config_substitution( obj_class="DataStoreConf", obj_id="default_tp_store_conf", - updates={"max_file_size": 200000000}, + updates={"max_file_size": 170000000}, ) ) conf_dict.config_substitutions.append( From 507cb0e4ed81ee2fa4beb57450a3db7093375ef3 Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Tue, 3 Jun 2025 15:07:56 -0500 Subject: [PATCH 13/18] Added the -k option to dfmodules_integtest_bundle.sh script (users can specify tests by name substrings) --- scripts/dfmodules_integtest_bundle.sh | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/scripts/dfmodules_integtest_bundle.sh b/scripts/dfmodules_integtest_bundle.sh index 97d99bcb..45874a70 100755 --- a/scripts/dfmodules_integtest_bundle.sh +++ b/scripts/dfmodules_integtest_bundle.sh @@ -15,6 +15,7 @@ Options: -h, --help : prints out usage information -f -l + -k -n -N --stop-on-failure : causes the script to stop when one of the integtests reports a failure @@ -29,7 +30,7 @@ Options: echo "" } -TEMP=`getopt -o hs:f:l:n:N: --long help,stop-on-failure,stop-on-skip -- "$@"` +TEMP=`getopt -o hs:f:l:k:n:N: --long help,stop-on-failure,stop-on-skip -- "$@"` eval set -- "$TEMP" let first_test_index=0 @@ -37,6 +38,7 @@ let individual_test_requested_iterations=1 let full_set_requested_interations=1 let stop_on_failure=0 let stop_on_skip=0 +requested_test_names= while true; do case "$1" in @@ -52,6 +54,10 @@ while true; do let last_test_index=$2 shift 2 ;; + -k) + requested_test_names=$2 + shift 2 + ;; -n) let individual_test_requested_iterations=$2 shift 2 @@ -90,7 +96,19 @@ fi TIMESTAMP=`date '+%Y%m%d%H%M%S'` mkdir -p /tmp/pytest-of-${USER} ITGRUNNER_LOG_FILE="/tmp/pytest-of-${USER}/dfmodules_integtest_bundle_${TIMESTAMP}.log" -let total_number_of_tests=(1+${last_test_index}-${first_test_index})*${individual_test_requested_iterations}*${full_set_requested_interations} + +let number_of_individual_tests=0 +let test_index=0 +for TEST_NAME in ${integtest_list[@]}; do + if [[ ${test_index} -ge ${first_test_index} && ${test_index} -le ${last_test_index} ]]; then + requested_test=`echo ${TEST_NAME} | egrep -i ${requested_test_names:-${TEST_NAME}}` + if [[ "${requested_test}" != "" ]]; then + let number_of_individual_tests=${number_of_individual_tests}+1 + fi + fi + let test_index=${test_index}+1 +done +let total_number_of_tests=${number_of_individual_tests}*${individual_test_requested_iterations}*${full_set_requested_interations} # run the tests let overall_test_index=0 # this is only used for user feedback @@ -99,6 +117,8 @@ while [[ ${full_set_loop_count} -lt ${full_set_requested_interations} ]]; do let test_index=0 for TEST_NAME in ${integtest_list[@]}; do if [[ ${test_index} -ge ${first_test_index} && ${test_index} -le ${last_test_index} ]]; then + requested_test=`echo ${TEST_NAME} | egrep -i ${requested_test_names:-${TEST_NAME}}` + if [[ "${requested_test}" != "" ]]; then let individual_loop_count=0 while [[ ${individual_loop_count} -lt ${individual_test_requested_iterations} ]]; do @@ -136,6 +156,7 @@ while [[ ${full_set_loop_count} -lt ${full_set_requested_interations} ]]; do fi done + fi fi let test_index=${test_index}+1 done From 74ce26065a74b21c8ae3cfa796d6aabf48fe8d52 Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Tue, 3 Jun 2025 15:09:25 -0500 Subject: [PATCH 14/18] Additional cleanup in various tests --- integtest/disabled_output_test.py | 32 ----------------------- integtest/insufficient_disk_space_test.py | 16 +++++++----- integtest/large_trigger_record_test.py | 20 ++++++++------ 3 files changed, 22 insertions(+), 46 deletions(-) diff --git a/integtest/disabled_output_test.py b/integtest/disabled_output_test.py index fda6688b..79285630 100644 --- a/integtest/disabled_output_test.py +++ b/integtest/disabled_output_test.py @@ -22,34 +22,6 @@ check_for_logfile_errors = True expected_event_count = trigger_rate * run_duration expected_event_count_tolerance = math.ceil(expected_event_count / 10) -wib1_frag_hsi_trig_params = { - "fragment_type_description": "WIB", - "fragment_type": "ProtoWIB", - "expected_fragment_count": number_of_data_producers, - "min_size_bytes": 37656, - "max_size_bytes": 37656, -} -wib1_frag_multi_trig_params = { - "fragment_type_description": "WIB", - "fragment_type": "ProtoWIB", - "expected_fragment_count": number_of_data_producers, - "min_size_bytes": 72, - "max_size_bytes": 54000, -} -wib2_frag_hsi_trig_params = { - "fragment_type_description": "WIB", - "fragment_type": "WIB", - "expected_fragment_count": (number_of_data_producers), - "min_size_bytes": 29808, - "max_size_bytes": 30280, -} -wib2_frag_multi_trig_params = { - "fragment_type_description": "WIB", - "fragment_type": "WIB", - "expected_fragment_count": (number_of_data_producers), - "min_size_bytes": 72, - "max_size_bytes": 54000, -} wibeth_frag_hsi_trig_params = { "fragment_type_description": "WIBEth", "fragment_type": "WIBEth", @@ -212,14 +184,10 @@ def test_data_files(run_nanorc): local_event_count_tolerance += ( 10 * number_of_data_producers * run_duration / 100 ) - # fragment_check_list.append(wib1_frag_multi_trig_params) # ProtoWIB - # fragment_check_list.append(wib2_frag_multi_trig_params) # DuneWIB fragment_check_list.append(wibeth_frag_multi_trig_params) # WIBEth fragment_check_list.append(triggerprimitive_frag_params) fragment_check_list.append(triggeractivity_frag_params) else: - # fragment_check_list.append(wib1_frag_hsi_trig_params) # ProtoWIB - # fragment_check_list.append(wib2_frag_hsi_trig_params) # DuneWIB fragment_check_list.append(wibeth_frag_hsi_trig_params) # WIBEth # Run some tests on the output data file diff --git a/integtest/insufficient_disk_space_test.py b/integtest/insufficient_disk_space_test.py index 66369a0f..3a1f990f 100644 --- a/integtest/insufficient_disk_space_test.py +++ b/integtest/insufficient_disk_space_test.py @@ -10,6 +10,10 @@ pytest_plugins = "integrationtest.integrationtest_drunc" +# 02-Jun-2025, KAB: tweak the print() statement default behavior so that it always flushes the output. +import functools +print = functools.partial(print, flush=True) + # 21-Jul-2022, KAB: # --> changes that are needed in this script include the following: # * add intelligence to verify that the output disk is small enough @@ -257,17 +261,17 @@ def test_cleanup(run_nanorc): pathlist_string += " " + str(data_file.parent) if pathlist_string and filelist_string: - print("============================================", flush=True) - print("Listing the hdf5 files before deleting them:", flush=True) - print("============================================", flush=True) + print("============================================") + print("Listing the hdf5 files before deleting them:") + print("============================================") os.system(f"df -h {pathlist_string}") - print("--------------------", flush=True) + print("--------------------") os.system(f"ls -alF {filelist_string}") for data_file in run_nanorc.data_files: data_file.unlink() - print("--------------------", flush=True) + print("--------------------") os.system(f"df -h {pathlist_string}") - print("============================================", flush=True) + print("============================================") diff --git a/integtest/large_trigger_record_test.py b/integtest/large_trigger_record_test.py index 30a6a7b5..8e238dad 100644 --- a/integtest/large_trigger_record_test.py +++ b/integtest/large_trigger_record_test.py @@ -17,6 +17,10 @@ pytest_plugins = "integrationtest.integrationtest_drunc" +# 02-Jun-2025, KAB: tweak the print() statement default behavior so that it always flushes the output. +import functools +print = functools.partial(print, flush=True) + # Values that help determine the running conditions output_path_parameter = "." number_of_data_producers = 10 @@ -151,10 +155,10 @@ if sufficient_disk_space: nanorc_command_list = ( "boot conf wait 5".split() - + "start 101 wait 1 enable-triggers wait ".split() + + "start --run-number 101 wait 1 enable-triggers wait ".split() + [str(run_duration)] + "disable-triggers wait 2 drain-dataflow wait 2 stop-trigger-sources stop ".split() - + "start 102 wait 1 enable-triggers wait ".split() + + "start --run-number 102 wait 1 enable-triggers wait ".split() + [str(run_duration)] + "disable-triggers wait 2 drain-dataflow wait 2 stop-trigger-sources stop ".split() + " scrap terminate".split() @@ -260,17 +264,17 @@ def test_cleanup(run_nanorc): pathlist_string += " " + str(data_file.parent) if pathlist_string and filelist_string: - print("============================================", flush=True) - print("Listing the hdf5 files before deleting them:", flush=True) - print("============================================", flush=True) + print("============================================") + print("Listing the hdf5 files before deleting them:") + print("============================================") os.system(f"df -h {pathlist_string}") - print("--------------------", flush=True) + print("--------------------") os.system(f"ls -alF {filelist_string}") for data_file in run_nanorc.data_files: data_file.unlink() - print("--------------------", flush=True) + print("--------------------") os.system(f"df -h {pathlist_string}") - print("============================================", flush=True) + print("============================================") From 718c2e72f68400f77d2dedf75464863e297349b0 Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Tue, 3 Jun 2025 15:10:33 -0500 Subject: [PATCH 15/18] Added output information on the number of data files produced in multiple_data_writers_test.py --- integtest/multiple_data_writers_test.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/integtest/multiple_data_writers_test.py b/integtest/multiple_data_writers_test.py index 26b8de0f..5a5173a0 100644 --- a/integtest/multiple_data_writers_test.py +++ b/integtest/multiple_data_writers_test.py @@ -157,9 +157,13 @@ def test_data_files(run_nanorc): fragment_check_list.append(triggeractivity_frag_params) # Run some tests on the output data file - assert len(run_nanorc.data_files) == 6 # three for each run + all_ok = len(run_nanorc.data_files) == 6 # three for each run + print("") # Clear potential dot from pytest + if all_ok: + print("\N{WHITE HEAVY CHECK MARK} The correct number of raw data files was found (6)") + else: + print(f"\N{POLICE CARS REVOLVING LIGHT} An incorrect number of raw data files was found, expected 6, found {len(run_nanorc.data_files)} \N{POLICE CARS REVOLVING LIGHT}") - all_ok = True for idx in range(len(run_nanorc.data_files)): data_file = data_file_checks.DataFile(run_nanorc.data_files[idx]) all_ok &= data_file_checks.sanity_check(data_file) @@ -171,4 +175,4 @@ def test_data_files(run_nanorc): all_ok &= data_file_checks.check_fragment_sizes( data_file, fragment_check_list[jdx] ) - assert all_ok + assert all_ok, "\N{POLICE CARS REVOLVING LIGHT} One or more raw data file checks failed! \N{POLICE CARS REVOLVING LIGHT}" From d28a5e02db47c45147f72bd8415a50ac6b8be2b6 Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Tue, 3 Jun 2025 15:12:52 -0500 Subject: [PATCH 16/18] improved the output created by max_file_size_test.py, and made a few mods to this integtest to get it to run more reliably (e.g. larger tp_input queue). --- integtest/max_file_size_test.py | 55 ++++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 12 deletions(-) diff --git a/integtest/max_file_size_test.py b/integtest/max_file_size_test.py index c98bd744..5f348b35 100644 --- a/integtest/max_file_size_test.py +++ b/integtest/max_file_size_test.py @@ -10,6 +10,10 @@ pytest_plugins = "integrationtest.integrationtest_drunc" +# 02-Jun-2025, KAB: tweak the print() statement default behavior so that it always flushes the output. +import functools +print = functools.partial(print, flush=True) + # Values that help determine the running conditions number_of_data_producers = 2 number_of_readout_apps = 3 @@ -108,6 +112,13 @@ obj_class="LatencyBuffer", updates={"size": 200000} ) ) +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="TCDataProcessor", + obj_id="def-tc-processor", + updates={"merge_overlapping_tcs": 0}, + ) +) conf_dict.config_substitutions.append( data_classes.config_substitution( @@ -163,6 +174,18 @@ ) ) +# 02-Jun-2025, KAB +# With the replay data file and configuration in this test, the rate of TP vectors +# out of each DataLinkHandler is about 3 kHz. Two DLHs per app and a 5-second safety +# factor gives a queue size of 30000. +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="QueueDescriptor", + obj_id="tp-input", + updates={"capacity": 200000}, + ) +) + confgen_arguments = { "WIBEth_TPG_System": conf_dict, @@ -208,9 +231,13 @@ def test_data_files(run_nanorc): fragment_check_list.append(triggeractivity_frag_params) # Run some tests on the output data file - assert len(run_nanorc.data_files) == 6 # three for each run + all_ok = len(run_nanorc.data_files) == 6 # three for each run + print("") # Clear potential dot from pytest + if all_ok: + print("\N{WHITE HEAVY CHECK MARK} The correct number of raw data files was found (6)") + else: + print(f"\N{POLICE CARS REVOLVING LIGHT} An incorrect number of raw data files was found, expected 6, found {len(run_nanorc.data_files)} \N{POLICE CARS REVOLVING LIGHT}") - all_ok = True for idx in range(len(run_nanorc.data_files)): data_file = data_file_checks.DataFile(run_nanorc.data_files[idx]) all_ok &= data_file_checks.sanity_check(data_file) @@ -222,16 +249,20 @@ def test_data_files(run_nanorc): all_ok &= data_file_checks.check_fragment_sizes( data_file, fragment_check_list[jdx] ) - assert all_ok + assert all_ok, "\N{POLICE CARS REVOLVING LIGHT} One or more raw data file checks failed! \N{POLICE CARS REVOLVING LIGHT}" def test_tpstream_files(run_nanorc): tpstream_files = run_nanorc.tpset_files fragment_check_list = [wibeth_tpset_params] # WIBEth - assert len(tpstream_files) == 6 # three for each run + all_ok = len(tpstream_files) == 6 # three for each run + print("") # Clear potential dot from pytest + if all_ok: + print("\N{WHITE HEAVY CHECK MARK} The correct number of TP-stream data files was found (6)") + else: + print(f"\N{POLICE CARS REVOLVING LIGHT} An incorrect number of TP-stream data files was found, expected 6, found {len(tpstream_files)} \N{POLICE CARS REVOLVING LIGHT}") - all_ok = True for idx in range(len(tpstream_files)): data_file = data_file_checks.DataFile(tpstream_files[idx]) all_ok &= data_file_checks.check_file_attributes(data_file) @@ -239,7 +270,7 @@ def test_tpstream_files(run_nanorc): all_ok &= data_file_checks.check_fragment_count( data_file, fragment_check_list[jdx] ) - assert all_ok + assert all_ok, "\N{POLICE CARS REVOLVING LIGHT} One or more TP-stream data file checks failed! \N{POLICE CARS REVOLVING LIGHT}" def test_cleanup(run_nanorc): @@ -255,12 +286,12 @@ def test_cleanup(run_nanorc): pathlist_string += " " + str(data_file.parent) if pathlist_string and filelist_string: - print("============================================", flush=True) - print("Listing the hdf5 files before deleting them:", flush=True) - print("============================================", flush=True) + print("============================================") + print("Listing the hdf5 files before deleting them:") + print("============================================") os.system(f"df -h {pathlist_string}") - print("--------------------", flush=True) + print("--------------------") os.system(f"ls -alF {filelist_string}") for data_file in run_nanorc.data_files: @@ -268,6 +299,6 @@ def test_cleanup(run_nanorc): for data_file in run_nanorc.tpset_files: data_file.unlink() - print("--------------------", flush=True) + print("--------------------") os.system(f"df -h {pathlist_string}") - print("============================================", flush=True) + print("============================================") From bce05a25fa25a7be8c381dd700d8037b599cb9a2 Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Tue, 3 Jun 2025 15:15:14 -0500 Subject: [PATCH 17/18] Increased the capacity of the tp_input queue in hdf5_compression_test.py, to try to avoid occasional failures to push data onto that queue. --- integtest/hdf5_compression_test.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/integtest/hdf5_compression_test.py b/integtest/hdf5_compression_test.py index 2698671f..78c05ce3 100644 --- a/integtest/hdf5_compression_test.py +++ b/integtest/hdf5_compression_test.py @@ -216,6 +216,17 @@ obj_class="LatencyBuffer", updates={"size": 5592000} ) ) +# 03-Jun-2025, KAB +# TP rate is ~67 kHz with the run 36012 replay data file. +# 67 kHz * 2 DataLinkHandlers per RU * safety factor of 3 seconds ~= 400,000 +# However, this doesn't always seem to be large enough, so we'll use 1e6 for now. +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="QueueDescriptor", + obj_id="tp-input", + updates={"capacity": 1000000}, + ) +) confgen_arguments = { @@ -280,7 +291,7 @@ def test_data_files(run_nanorc): all_ok &= data_file_checks.check_fragment_sizes( data_file, fragment_check_list[jdx] ) - assert all_ok + assert all_ok, "\N{POLICE CARS REVOLVING LIGHT} One or more raw data file checks failed! \N{POLICE CARS REVOLVING LIGHT}" def test_tpstream_files(run_nanorc): @@ -301,7 +312,7 @@ def test_tpstream_files(run_nanorc): all_ok &= data_file_checks.check_fragment_count( data_file, fragment_check_list[jdx] ) - assert all_ok + assert all_ok, "\N{POLICE CARS REVOLVING LIGHT} One or more TP-stream data file checks failed! \N{POLICE CARS REVOLVING LIGHT}" def test_cleanup(run_nanorc): From 21910c981984578045f3ddff877499b6bfe75f67 Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Wed, 4 Jun 2025 17:17:24 +0200 Subject: [PATCH 18/18] Cleaned up the heading strings in the integtests a bit, as suggested by Eric. --- integtest/disabled_output_test.py | 2 +- integtest/hdf5_compression_test.py | 2 +- integtest/insufficient_disk_space_test.py | 2 +- integtest/large_trigger_record_test.py | 2 +- integtest/max_file_size_test.py | 2 +- integtest/multiple_data_writers_test.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/integtest/disabled_output_test.py b/integtest/disabled_output_test.py index 79285630..53f30bd4 100644 --- a/integtest/disabled_output_test.py +++ b/integtest/disabled_output_test.py @@ -154,7 +154,7 @@ def test_nanorc_success(run_nanorc): current_test = os.environ.get("PYTEST_CURRENT_TEST") - match_obj = re.search(r".*\[(.+)\].*", current_test) + match_obj = re.search(r".*\[(.+)-run_nanorc0\].*", current_test) if match_obj: current_test = match_obj.group(1) banner_line = re.sub(".", "=", current_test) diff --git a/integtest/hdf5_compression_test.py b/integtest/hdf5_compression_test.py index 78c05ce3..b58133b8 100644 --- a/integtest/hdf5_compression_test.py +++ b/integtest/hdf5_compression_test.py @@ -248,7 +248,7 @@ def test_nanorc_success(run_nanorc): current_test = os.environ.get("PYTEST_CURRENT_TEST") - match_obj = re.search(r".*\[(.+)\].*", current_test) + match_obj = re.search(r".*\[(.+)-run_nanorc0\].*", current_test) if match_obj: current_test = match_obj.group(1) banner_line = re.sub(".", "=", current_test) diff --git a/integtest/insufficient_disk_space_test.py b/integtest/insufficient_disk_space_test.py index 3a1f990f..cd9833fb 100644 --- a/integtest/insufficient_disk_space_test.py +++ b/integtest/insufficient_disk_space_test.py @@ -182,7 +182,7 @@ def test_nanorc_success(run_nanorc): ) current_test = os.environ.get("PYTEST_CURRENT_TEST") - match_obj = re.search(r".*\[(.+)\].*", current_test) + match_obj = re.search(r".*\[(.+)-run_nanorc0\].*", current_test) if match_obj: current_test = match_obj.group(1) banner_line = re.sub(".", "=", current_test) diff --git a/integtest/large_trigger_record_test.py b/integtest/large_trigger_record_test.py index 8e238dad..5c586758 100644 --- a/integtest/large_trigger_record_test.py +++ b/integtest/large_trigger_record_test.py @@ -176,7 +176,7 @@ def test_nanorc_success(run_nanorc): ) current_test = os.environ.get("PYTEST_CURRENT_TEST") - match_obj = re.search(r".*\[(.+)\].*", current_test) + match_obj = re.search(r".*\[(.+)-run_nanorc0\].*", current_test) if match_obj: current_test = match_obj.group(1) banner_line = re.sub(".", "=", current_test) diff --git a/integtest/max_file_size_test.py b/integtest/max_file_size_test.py index 5f348b35..b5573157 100644 --- a/integtest/max_file_size_test.py +++ b/integtest/max_file_size_test.py @@ -206,7 +206,7 @@ def test_nanorc_success(run_nanorc): current_test = os.environ.get("PYTEST_CURRENT_TEST") - match_obj = re.search(r".*\[(.+)\].*", current_test) + match_obj = re.search(r".*\[(.+)-run_nanorc0\].*", current_test) if match_obj: current_test = match_obj.group(1) banner_line = re.sub(".", "=", current_test) diff --git a/integtest/multiple_data_writers_test.py b/integtest/multiple_data_writers_test.py index 5a5173a0..51c48692 100644 --- a/integtest/multiple_data_writers_test.py +++ b/integtest/multiple_data_writers_test.py @@ -132,7 +132,7 @@ def test_nanorc_success(run_nanorc): current_test = os.environ.get("PYTEST_CURRENT_TEST") - match_obj = re.search(r".*\[(.+)\].*", current_test) + match_obj = re.search(r".*\[(.+)-run_nanorc0\].*", current_test) if match_obj: current_test = match_obj.group(1) banner_line = re.sub(".", "=", current_test)