diff --git a/integtest/README.md b/integtest/README.md index 680d8ae7..c4a2dd22 100644 --- a/integtest/README.md +++ b/integtest/README.md @@ -1,4 +1,4 @@ -* 19-Jul-2023, KAB, ELF, and others: notes on existing integtests +* 29-Apr-2025, KAB, ELF, and others: notes on existing integtests "integtests" are intended to be automated integration and/or system tests that make use of the "pytest" framework to validate the operation of the DAQ system in various scenarios. @@ -6,11 +6,14 @@ Here is a sample command for invoking a test (feel free to keep or drop the options in brackets, as you prefer): ``` -pytest -s disabled_output_test.py [--nanorc-option partition-number 2] [--nanorc-option timeout 300] +pytest -s max_file_size_test.py [--nanorc-option log-level debug] # this nanorc option is still useful even when using drunc ``` For reference, here are the ideas behind the existing tests: +* `max_file_size_test.py` - verifies that data files are closed when they reach a specified maximum file size (approximately) +* `multiple_data_writers_test.py` - verifies that we can run multiple DataWriters for a single TriggerRecordBuilder +* `hdf5_compression_test.py` - verifies that HDF5 compression is working as expected by writing several data files of known size + * `large_trigger_record_test.py` - verify that TriggerRecords that are close to the size of a whole file get written to disk correctly * `disabled_output_test.py` - verify that the --disable-data-storage option works -* `multi_output_file_test.py` - test that the file size maximum config parameter works * `insufficient_disk_space_test.py` - verify that the appropriate errors and warnings are produced when there isn't enough disk space to write data diff --git a/integtest/disabled_output_test.py b/integtest/disabled_output_test.py index d06cf386..53f30bd4 100644 --- a/integtest/disabled_output_test.py +++ b/integtest/disabled_output_test.py @@ -22,42 +22,9 @@ check_for_logfile_errors = True expected_event_count = trigger_rate * run_duration expected_event_count_tolerance = math.ceil(expected_event_count / 10) -wib1_frag_hsi_trig_params = { - "fragment_type_description": "WIB", - "fragment_type": "ProtoWIB", - "hdf5_source_subsystem": "Detector_Readout", - "expected_fragment_count": number_of_data_producers, - "min_size_bytes": 37656, - "max_size_bytes": 37656, -} -wib1_frag_multi_trig_params = { - "fragment_type_description": "WIB", - "fragment_type": "ProtoWIB", - "hdf5_source_subsystem": "Detector_Readout", - "expected_fragment_count": number_of_data_producers, - "min_size_bytes": 72, - "max_size_bytes": 54000, -} -wib2_frag_hsi_trig_params = { - "fragment_type_description": "WIB", - "fragment_type": "WIB", - "hdf5_source_subsystem": "Detector_Readout", - "expected_fragment_count": (number_of_data_producers), - "min_size_bytes": 29808, - "max_size_bytes": 30280, -} -wib2_frag_multi_trig_params = { - "fragment_type_description": "WIB", - "fragment_type": "WIB", - "hdf5_source_subsystem": "Detector_Readout", - "expected_fragment_count": (number_of_data_producers), - "min_size_bytes": 72, - "max_size_bytes": 54000, -} wibeth_frag_hsi_trig_params = { "fragment_type_description": "WIBEth", "fragment_type": "WIBEth", - "hdf5_source_subsystem": "Detector_Readout", "expected_fragment_count": (number_of_data_producers), "min_size_bytes": 7272, "max_size_bytes": 14472, @@ -65,7 +32,6 @@ wibeth_frag_multi_trig_params = { "fragment_type_description": "WIBEth", "fragment_type": "WIBEth", - "hdf5_source_subsystem": "Detector_Readout", "expected_fragment_count": (number_of_data_producers), "min_size_bytes": 72, "max_size_bytes": 14472, @@ -73,23 +39,20 @@ triggercandidate_frag_params = { "fragment_type_description": "Trigger Candidate", "fragment_type": "Trigger_Candidate", - "hdf5_source_subsystem": "Trigger", "expected_fragment_count": 1, - "min_size_bytes": 72, + "min_size_bytes": 128, "max_size_bytes": 280, } triggeractivity_frag_params = { "fragment_type_description": "Trigger Activity", "fragment_type": "Trigger_Activity", - "hdf5_source_subsystem": "Trigger", "expected_fragment_count": 1, "min_size_bytes": 72, "max_size_bytes": 216, } -triggertp_frag_params = { - "fragment_type_description": "Trigger with TPs", +triggerprimitive_frag_params = { + "fragment_type_description": "Trigger Primitive", "fragment_type": "Trigger_Primitive", - "hdf5_source_subsystem": "Trigger", "expected_fragment_count": 2, # number of readout apps (1) times 2 "min_size_bytes": 72, "max_size_bytes": 16000, @@ -97,7 +60,6 @@ hsi_frag_params = { "fragment_type_description": "HSI", "fragment_type": "Hardware_Signal", - "hdf5_source_subsystem": "HW_Signals_Interface", "expected_fragment_count": 1, "min_size_bytes": 72, "max_size_bytes": 100, @@ -192,7 +154,7 @@ def test_nanorc_success(run_nanorc): current_test = os.environ.get("PYTEST_CURRENT_TEST") - match_obj = re.search(r".*\[(.+)\].*", current_test) + match_obj = re.search(r".*\[(.+)-run_nanorc0\].*", current_test) if match_obj: current_test = match_obj.group(1) banner_line = re.sub(".", "=", current_test) @@ -222,30 +184,28 @@ def test_data_files(run_nanorc): local_event_count_tolerance += ( 10 * number_of_data_producers * run_duration / 100 ) - # fragment_check_list.append(wib1_frag_multi_trig_params) # ProtoWIB - # fragment_check_list.append(wib2_frag_multi_trig_params) # DuneWIB fragment_check_list.append(wibeth_frag_multi_trig_params) # WIBEth - fragment_check_list.append(triggertp_frag_params) + fragment_check_list.append(triggerprimitive_frag_params) fragment_check_list.append(triggeractivity_frag_params) else: - # fragment_check_list.append(wib1_frag_hsi_trig_params) # ProtoWIB - # fragment_check_list.append(wib2_frag_hsi_trig_params) # DuneWIB fragment_check_list.append(wibeth_frag_hsi_trig_params) # WIBEth # Run some tests on the output data file assert len(run_nanorc.data_files) == expected_number_of_data_files + all_ok = True for idx in range(len(run_nanorc.data_files)): data_file = data_file_checks.DataFile(run_nanorc.data_files[idx]) - assert data_file_checks.sanity_check(data_file) - assert data_file_checks.check_file_attributes(data_file) - assert data_file_checks.check_event_count( + all_ok &= data_file_checks.sanity_check(data_file) + all_ok &= data_file_checks.check_file_attributes(data_file) + all_ok &= data_file_checks.check_event_count( data_file, local_expected_event_count, local_event_count_tolerance ) for jdx in range(len(fragment_check_list)): - assert data_file_checks.check_fragment_count( + all_ok &= data_file_checks.check_fragment_count( data_file, fragment_check_list[jdx] ) - assert data_file_checks.check_fragment_sizes( + all_ok &= data_file_checks.check_fragment_sizes( data_file, fragment_check_list[jdx] ) + assert all_ok diff --git a/integtest/hdf5_compression_test.py b/integtest/hdf5_compression_test.py new file mode 100644 index 00000000..b58133b8 --- /dev/null +++ b/integtest/hdf5_compression_test.py @@ -0,0 +1,346 @@ +import pytest +import os +import re +import copy +import urllib.request + +import integrationtest.data_file_checks as data_file_checks +import integrationtest.log_file_checks as log_file_checks +import integrationtest.data_classes as data_classes + +pytest_plugins = "integrationtest.integrationtest_drunc" + +# 20-May-2025, KAB: tweak the print() statement default behavior so that it always flushes the output. +import functools +print = functools.partial(print, flush=True) + +# Values that help determine the running conditions +number_of_data_producers = 2 +number_of_readout_apps = 3 + +# Default values for validation parameters +check_for_logfile_errors = True + +daphne_frag_params = { + "fragment_type_description": "DAPHNE", + "fragment_type": "DAPHNE", + "expected_fragment_count": number_of_data_producers * number_of_readout_apps, + "min_size_bytes": 1936, + "max_size_bytes": 230000, +# "frag_sizes_by_TC_type": {"kPrescale": {"min_size_bytes": 1936, "max_size_bytes": 30000}, +# "kRandom": {"min_size_bytes": 220000, "max_size_bytes": 230000}, +# "default": {"min_size_bytes": 1936, "max_size_bytes": 230000} } +} +daphne_triggerprimitive_frag_params = { + "fragment_type_description": "Trigger Primitive", + "fragment_type": "Trigger_Primitive", + "expected_fragment_count": number_of_readout_apps, + "min_size_bytes": 72, + "max_size_bytes": 230000, +# "frag_sizes_by_TC_type": {"kPrescale": {"min_size_bytes": 72, "max_size_bytes": 10000}, +# "kRandom": {"min_size_bytes": 220000, "max_size_bytes": 230000}, +# "default": {"min_size_bytes": 72, "max_size_bytes": 230000} } +} +daphne_tpset_params = { + "fragment_type_description": "TP Stream", + "fragment_type": "Trigger_Primitive", + "expected_fragment_count": number_of_readout_apps, + "frag_counts_by_record_ordinal": { "first": {"min_count": 1, "max_count": number_of_readout_apps}, + "last": {"min_count": 1, "max_count": number_of_readout_apps}, + "default": {"min_count": number_of_readout_apps, "max_count": number_of_readout_apps} }, + "min_size_bytes": 200, + "max_size_bytes": 210, + "debug_mask": 0x0, +# "frag_sizes_by_record_ordinal": { "first": {"min_size_bytes": 96, "max_size_bytes": 120000}, +# "second": {"min_size_bytes": 96, "max_size_bytes": 120000}, +# "last": {"min_size_bytes": 96, "max_size_bytes": 120000}, +# "default": {"min_size_bytes": 80000, "max_size_bytes": 120000} } +} + +#wibeth_frag_params = { +# "fragment_type_description": "WIBEth", +# "fragment_type": "WIBEth", +# "expected_fragment_count": (number_of_data_producers * number_of_readout_apps), +# "min_size_bytes": 7272, +# "max_size_bytes": 194472, +#} +#wibeth_tpset_params = { +# "fragment_type_description": "TP Stream", +# "fragment_type": "Trigger_Primitive", +# "expected_fragment_count": number_of_readout_apps * 3, +# "frag_counts_by_record_ordinal": {"first": {"min_count": 1, "max_count": number_of_readout_apps * 3}, +# "default": {"min_count": number_of_readout_apps * 3, "max_count": number_of_readout_apps * 3} }, +# "min_size_bytes": 0, # not checked +# "max_size_bytes": 0, # not checked +# "debug_mask": 0x0, +#} + +# sizes: 128 is for one TC with zero TAs inside it (72+56) +# 208 is for one TC with one TA inside it (72+56+80) +# 264 is for two TCs with one TA in one of them (72+56+80+56) +triggercandidate_frag_params = { + "fragment_type_description": "Trigger Candidate", + "fragment_type": "Trigger_Candidate", + "expected_fragment_count": 1, + "min_size_bytes": 128, + "max_size_bytes": 264, +} +# sizes: 72 is for an empty TA fragment +# 632 is for three TAs with one TP in each of them (72+5*(88+24)) +triggeractivity_frag_params = { + "fragment_type_description": "Trigger Activity", + "fragment_type": "Trigger_Activity", + "expected_fragment_count": 1, + "min_size_bytes": 72, + "max_size_bytes": 632, +} +# sizes: 72 is for an empty TP fragment +# 1032 is for a fragment with 40 TPs in it (72+(40*24)) +triggerprimitive_frag_params = { + "fragment_type_description": "Trigger Primitive", + "fragment_type": "Trigger_Primitive", + "expected_fragment_count": number_of_readout_apps, + "min_size_bytes": 72, + "max_size_bytes": 1032, +} +hsi_frag_params = { + "fragment_type_description": "HSI", + "fragment_type": "Hardware_Signal", + "expected_fragment_count": 1, + "min_size_bytes": 72, + "max_size_bytes": 100, +} +ignored_logfile_problems = { + "-controller": [ + "Worker with pid \\d+ was terminated due to signal 1", + ], + "connectivity-service": [ + "errorlog: -", + ], +} + +# The next three variable declarations *must* be present as globals in the test +# file. They're read by the "fixtures" in conftest.py to determine how +# to run the config generation and nanorc + +object_databases = ["config/daqsystemtest/integrationtest-objects.data.xml"] + +conf_dict = data_classes.drunc_config() +conf_dict.dro_map_config.n_streams = number_of_data_producers +conf_dict.dro_map_config.n_apps = number_of_readout_apps +conf_dict.op_env = "integtest" +conf_dict.session = "hdf5compression" +conf_dict.tpg_enabled = True +conf_dict.fake_hsi_enabled = True +conf_dict.dro_map_config.det_id = 2 # det_id = 2 for kHD_PDS +conf_dict.frame_file = "asset://?checksum=a8990a9eb3a505d4ded62dfdfa9e2681" # run 36012 +#conf_dict.frame_file = "file:///home/nfs/biery/dunedaq/12MayFDv5.3.2DevInstrUpdate/sourcecode/dfmodules/integtest/np02vdcoldbox_run035227_sample_hd_pds.bin" + +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="TAMakerPrescaleAlgorithm", + obj_id="dummy-ta-maker", + updates={"prescale": 2000}, + ) +) + +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="FakeHSIEventGeneratorConf", + updates={"trigger_rate": 10.0}, + ) +) + +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="HSISignalWindow", + updates={ + "time_before": 1000, + "time_after": 500, + }, + ) +) +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="TCReadoutMap", + obj_id = "def-hsi-tc-map", + updates={ + "time_before": 120000, + "time_after": 1000, + }, + ) +) +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="DataStoreConf", + obj_id="default", + updates={"max_file_size": 400000000}, + ) +) +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="DataStoreConf", + obj_id="default_tp_store_conf", + updates={"max_file_size": 170000000}, + ) +) +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="DataStoreConf", + obj_id="default", + updates={"compression_level": 5}, + ) +) +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="DataStoreConf", + obj_id="default_tp_store_conf", + updates={"compression_level": 1}, + ) +) + +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="DFOConf", updates={"busy_threshold": 16, "free_threshold": 12} + ) +) +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="TCDataProcessor", + obj_id="def-tc-processor", + updates={"merge_overlapping_tcs": 0}, + ) +) +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="LatencyBuffer", updates={"size": 5592000} + ) +) +# 03-Jun-2025, KAB +# TP rate is ~67 kHz with the run 36012 replay data file. +# 67 kHz * 2 DataLinkHandlers per RU * safety factor of 3 seconds ~= 400,000 +# However, this doesn't always seem to be large enough, so we'll use 1e6 for now. +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="QueueDescriptor", + obj_id="tp-input", + updates={"capacity": 1000000}, + ) +) + + +confgen_arguments = { + "DAPHNE_TPG_System": conf_dict, +} + +# The commands to run in nanorc, as a list +nanorc_command_list = ( + "boot conf wait 5".split() + + "start --run-number 101 wait 1 enable-triggers wait 100".split() + + "disable-triggers wait 2 drain-dataflow wait 2 stop-trigger-sources stop ".split() + + "start --run-number 102 wait 1 enable-triggers wait 100".split() + + "disable-triggers wait 2 drain-dataflow wait 2 stop-trigger-sources stop ".split() + + " scrap terminate".split() +) + +# The tests themselves + + +def test_nanorc_success(run_nanorc): + current_test = os.environ.get("PYTEST_CURRENT_TEST") + match_obj = re.search(r".*\[(.+)-run_nanorc0\].*", current_test) + if match_obj: + current_test = match_obj.group(1) + banner_line = re.sub(".", "=", current_test) + print(banner_line) + print(current_test) + print(banner_line) + # Check that nanorc completed correctly + assert run_nanorc.completed_process.returncode == 0 + + +def test_log_files(run_nanorc): + if check_for_logfile_errors: + # Check that there are no warnings or errors in the log files + assert log_file_checks.logs_are_error_free( + run_nanorc.log_files, True, True, ignored_logfile_problems + ) + + +def test_data_files(run_nanorc): + fragment_check_list = [triggercandidate_frag_params, hsi_frag_params, daphne_frag_params] + fragment_check_list.append(daphne_triggerprimitive_frag_params) + fragment_check_list.append(triggeractivity_frag_params) + + # Run some tests on the output data file + all_ok = len(run_nanorc.data_files) == 6 # three for each run + print("") # Clear potential dot from pytest + if all_ok: + print("\N{WHITE HEAVY CHECK MARK} The correct number of raw data files was found (6)") + else: + print(f"\N{POLICE CARS REVOLVING LIGHT} An incorrect number of raw data files was found, expected 6, found {len(run_nanorc.data_files)} \N{POLICE CARS REVOLVING LIGHT}") + + for idx in range(len(run_nanorc.data_files)): + data_file = data_file_checks.DataFile(run_nanorc.data_files[idx]) + all_ok &= data_file_checks.sanity_check(data_file) + all_ok &= data_file_checks.check_file_attributes(data_file) + for jdx in range(len(fragment_check_list)): + all_ok &= data_file_checks.check_fragment_count( + data_file, fragment_check_list[jdx] + ) + all_ok &= data_file_checks.check_fragment_sizes( + data_file, fragment_check_list[jdx] + ) + assert all_ok, "\N{POLICE CARS REVOLVING LIGHT} One or more raw data file checks failed! \N{POLICE CARS REVOLVING LIGHT}" + + +def test_tpstream_files(run_nanorc): + tpstream_files = run_nanorc.tpset_files + fragment_check_list = [daphne_tpset_params] + + all_ok = len(tpstream_files) == 6 # three for each run + print("") # Clear potential dot from pytest + if all_ok: + print("\N{WHITE HEAVY CHECK MARK} The correct number of TP-stream data files was found (6)") + else: + print(f"\N{POLICE CARS REVOLVING LIGHT} An incorrect number of TP-stream data files was found, expected 6, found {len(tpstream_files)} \N{POLICE CARS REVOLVING LIGHT}") + + for idx in range(len(tpstream_files)): + data_file = data_file_checks.DataFile(tpstream_files[idx]) + all_ok &= data_file_checks.check_file_attributes(data_file) + for jdx in range(len(fragment_check_list)): + all_ok &= data_file_checks.check_fragment_count( + data_file, fragment_check_list[jdx] + ) + assert all_ok, "\N{POLICE CARS REVOLVING LIGHT} One or more TP-stream data file checks failed! \N{POLICE CARS REVOLVING LIGHT}" + + +def test_cleanup(run_nanorc): + pathlist_string = "" + filelist_string = "" + for data_file in run_nanorc.data_files: + filelist_string += " " + str(data_file) + if str(data_file.parent) not in pathlist_string: + pathlist_string += " " + str(data_file.parent) + for data_file in run_nanorc.tpset_files: + filelist_string += " " + str(data_file) + if str(data_file.parent) not in pathlist_string: + pathlist_string += " " + str(data_file.parent) + + if pathlist_string and filelist_string: + print("============================================") + print("Listing the hdf5 files before deleting them:") + print("============================================") + + os.system(f"df -h {pathlist_string}") + print("--------------------") + os.system(f"ls -alF {filelist_string}") + + for data_file in run_nanorc.data_files: + data_file.unlink() + for data_file in run_nanorc.tpset_files: + data_file.unlink() + + print("--------------------") + os.system(f"df -h {pathlist_string}") + print("============================================") diff --git a/integtest/insufficient_disk_space_test.py b/integtest/insufficient_disk_space_test.py index 3587a71c..cd9833fb 100644 --- a/integtest/insufficient_disk_space_test.py +++ b/integtest/insufficient_disk_space_test.py @@ -10,6 +10,10 @@ pytest_plugins = "integrationtest.integrationtest_drunc" +# 02-Jun-2025, KAB: tweak the print() statement default behavior so that it always flushes the output. +import functools +print = functools.partial(print, flush=True) + # 21-Jul-2022, KAB: # --> changes that are needed in this script include the following: # * add intelligence to verify that the output disk is small enough @@ -53,7 +57,6 @@ wibeth_frag_hsi_trig_params = { "fragment_type_description": "WIBEth", "fragment_type": "WIBEth", - "hdf5_source_subsystem": "Detector_Readout", "expected_fragment_count": (number_of_data_producers * number_of_readout_apps), "min_size_bytes": 35157672, "max_size_bytes": 35157672, @@ -61,31 +64,20 @@ triggercandidate_frag_params = { "fragment_type_description": "Trigger Candidate", "fragment_type": "Trigger_Candidate", - "hdf5_source_subsystem": "Trigger", "expected_fragment_count": 1, - "min_size_bytes": 72, + "min_size_bytes": 128, "max_size_bytes": 280, } triggeractivity_frag_params = { "fragment_type_description": "Trigger Activity", "fragment_type": "Trigger_Activity", - "hdf5_source_subsystem": "Trigger", "expected_fragment_count": number_of_readout_apps, "min_size_bytes": 72, "max_size_bytes": 400, } -triggertp_frag_params = { - "fragment_type_description": "Trigger with TPs", - "fragment_type": "Trigger_Primitive", - "hdf5_source_subsystem": "Trigger", - "expected_fragment_count": ((number_of_data_producers * number_of_readout_apps)), - "min_size_bytes": 72, - "max_size_bytes": 16000, -} hsi_frag_params = { "fragment_type_description": "HSI", "fragment_type": "Hardware_Signal", - "hdf5_source_subsystem": "HW_Signals_Interface", "expected_fragment_count": 1, "min_size_bytes": 72, "max_size_bytes": 100, @@ -190,7 +182,7 @@ def test_nanorc_success(run_nanorc): ) current_test = os.environ.get("PYTEST_CURRENT_TEST") - match_obj = re.search(r".*\[(.+)\].*", current_test) + match_obj = re.search(r".*\[(.+)-run_nanorc0\].*", current_test) if match_obj: current_test = match_obj.group(1) banner_line = re.sub(".", "=", current_test) @@ -237,20 +229,22 @@ def test_data_files(run_nanorc): # Run some tests on the output data file assert len(run_nanorc.data_files) == expected_number_of_data_files + all_ok = True for idx in range(len(run_nanorc.data_files)): data_file = data_file_checks.DataFile(run_nanorc.data_files[idx]) - assert data_file_checks.sanity_check(data_file) - assert data_file_checks.check_file_attributes(data_file) - assert data_file_checks.check_event_count( + all_ok &= data_file_checks.sanity_check(data_file) + all_ok &= data_file_checks.check_file_attributes(data_file) + all_ok &= data_file_checks.check_event_count( data_file, local_expected_event_count, local_event_count_tolerance ) for jdx in range(len(fragment_check_list)): - assert data_file_checks.check_fragment_count( + all_ok &= data_file_checks.check_fragment_count( data_file, fragment_check_list[jdx] ) - assert data_file_checks.check_fragment_sizes( + all_ok &= data_file_checks.check_fragment_sizes( data_file, fragment_check_list[jdx] ) + assert all_ok def test_cleanup(run_nanorc): diff --git a/integtest/large_trigger_record_test.py b/integtest/large_trigger_record_test.py index 5a2a11ca..5c586758 100644 --- a/integtest/large_trigger_record_test.py +++ b/integtest/large_trigger_record_test.py @@ -17,6 +17,10 @@ pytest_plugins = "integrationtest.integrationtest_drunc" +# 02-Jun-2025, KAB: tweak the print() statement default behavior so that it always flushes the output. +import functools +print = functools.partial(print, flush=True) + # Values that help determine the running conditions output_path_parameter = "." number_of_data_producers = 10 @@ -39,7 +43,6 @@ wibeth_frag_55pct_params = { "fragment_type_description": "WIBEth", "fragment_type": "WIBEth", - "hdf5_source_subsystem": "Detector_Readout", "expected_fragment_count": (number_of_data_producers * number_of_readout_apps), "min_size_bytes": 38678472, "max_size_bytes": 38678472, @@ -47,7 +50,6 @@ wibeth_frag_125pct_params = { "fragment_type_description": "WIBEth", "fragment_type": "WIBEth", - "hdf5_source_subsystem": "Detector_Readout", "expected_fragment_count": (number_of_data_producers * number_of_readout_apps), "min_size_bytes": 91411272, "max_size_bytes": 91411272, @@ -55,9 +57,8 @@ triggercandidate_frag_params = { "fragment_type_description": "Trigger Candidate", "fragment_type": "Trigger_Candidate", - "hdf5_source_subsystem": "Trigger", "expected_fragment_count": 1, - "min_size_bytes": 72, + "min_size_bytes": 128, "max_size_bytes": 280, } ignored_logfile_problems = { @@ -154,10 +155,10 @@ if sufficient_disk_space: nanorc_command_list = ( "boot conf wait 5".split() - + "start 101 wait 1 enable-triggers wait ".split() + + "start --run-number 101 wait 1 enable-triggers wait ".split() + [str(run_duration)] + "disable-triggers wait 2 drain-dataflow wait 2 stop-trigger-sources stop ".split() - + "start 102 wait 1 enable-triggers wait ".split() + + "start --run-number 102 wait 1 enable-triggers wait ".split() + [str(run_duration)] + "disable-triggers wait 2 drain-dataflow wait 2 stop-trigger-sources stop ".split() + " scrap terminate".split() @@ -175,7 +176,7 @@ def test_nanorc_success(run_nanorc): ) current_test = os.environ.get("PYTEST_CURRENT_TEST") - match_obj = re.search(r".*\[(.+)\].*", current_test) + match_obj = re.search(r".*\[(.+)-run_nanorc0\].*", current_test) if match_obj: current_test = match_obj.group(1) banner_line = re.sub(".", "=", current_test) diff --git a/integtest/multi_output_file_test.py b/integtest/max_file_size_test.py similarity index 74% rename from integtest/multi_output_file_test.py rename to integtest/max_file_size_test.py index 5ed5d2e3..b5573157 100644 --- a/integtest/multi_output_file_test.py +++ b/integtest/max_file_size_test.py @@ -10,6 +10,10 @@ pytest_plugins = "integrationtest.integrationtest_drunc" +# 02-Jun-2025, KAB: tweak the print() statement default behavior so that it always flushes the output. +import functools +print = functools.partial(print, flush=True) + # Values that help determine the running conditions number_of_data_producers = 2 number_of_readout_apps = 3 @@ -85,38 +89,45 @@ object_databases = ["config/daqsystemtest/integrationtest-objects.data.xml"] -wibtpg_conf = data_classes.drunc_config() -wibtpg_conf.dro_map_config.n_streams = number_of_data_producers -wibtpg_conf.dro_map_config.n_apps = number_of_readout_apps -wibtpg_conf.op_env = "integtest" -wibtpg_conf.session = "multioutput" -wibtpg_conf.tpg_enabled = True -wibtpg_conf.fake_hsi_enabled = True -wibtpg_conf.frame_file = ( +conf_dict = data_classes.drunc_config() +conf_dict.dro_map_config.n_streams = number_of_data_producers +conf_dict.dro_map_config.n_apps = number_of_readout_apps +conf_dict.op_env = "integtest" +conf_dict.session = "maxfilesize" +conf_dict.tpg_enabled = True +conf_dict.fake_hsi_enabled = True +conf_dict.frame_file = ( "asset://?checksum=dd156b4895f1b06a06b6ff38e37bd798" # WIBEth All Zeros ) -wibtpg_conf.config_substitutions.append( +conf_dict.config_substitutions.append( data_classes.config_substitution( - obj_id=wibtpg_conf.session, + obj_id=conf_dict.session, obj_class="Session", updates={"data_rate_slowdown_factor": data_rate_slowdown_factor}, ) ) -wibtpg_conf.config_substitutions.append( +conf_dict.config_substitutions.append( data_classes.config_substitution( obj_class="LatencyBuffer", updates={"size": 200000} ) ) +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="TCDataProcessor", + obj_id="def-tc-processor", + updates={"merge_overlapping_tcs": 0}, + ) +) -wibtpg_conf.config_substitutions.append( +conf_dict.config_substitutions.append( data_classes.config_substitution( obj_class="FakeHSIEventGeneratorConf", updates={"trigger_rate": 10.0}, ) ) -wibtpg_conf.config_substitutions.append( +conf_dict.config_substitutions.append( data_classes.config_substitution( obj_class="HSISignalWindow", updates={ @@ -125,7 +136,7 @@ }, ) ) -wibtpg_conf.config_substitutions.append( +conf_dict.config_substitutions.append( data_classes.config_substitution( obj_class="TCReadoutMap", obj_id = "def-hsi-tc-map", @@ -135,21 +146,21 @@ }, ) ) -wibtpg_conf.config_substitutions.append( +conf_dict.config_substitutions.append( data_classes.config_substitution( obj_class="DataStoreConf", obj_id="default", updates={"max_file_size": 725000000}, ) ) -wibtpg_conf.config_substitutions.append( +conf_dict.config_substitutions.append( data_classes.config_substitution( obj_class="DataStoreConf", obj_id="default_tp_store_conf", updates={"max_file_size": 275000000}, ) ) -wibtpg_conf.config_substitutions.append( +conf_dict.config_substitutions.append( data_classes.config_substitution( obj_class="StreamEmulationParameters", obj_id="stream-emu", @@ -157,15 +168,27 @@ ) ) -wibtpg_conf.config_substitutions.append( +conf_dict.config_substitutions.append( data_classes.config_substitution( obj_class="DFOConf", updates={"busy_threshold": 10, "free_threshold": 7} ) ) +# 02-Jun-2025, KAB +# With the replay data file and configuration in this test, the rate of TP vectors +# out of each DataLinkHandler is about 3 kHz. Two DLHs per app and a 5-second safety +# factor gives a queue size of 30000. +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="QueueDescriptor", + obj_id="tp-input", + updates={"capacity": 200000}, + ) +) + confgen_arguments = { - "WIBEth_TPG_System": wibtpg_conf, + "WIBEth_TPG_System": conf_dict, } # The commands to run in nanorc, as a list @@ -183,7 +206,7 @@ def test_nanorc_success(run_nanorc): current_test = os.environ.get("PYTEST_CURRENT_TEST") - match_obj = re.search(r".*\[(.+)\].*", current_test) + match_obj = re.search(r".*\[(.+)-run_nanorc0\].*", current_test) if match_obj: current_test = match_obj.group(1) banner_line = re.sub(".", "=", current_test) @@ -208,9 +231,13 @@ def test_data_files(run_nanorc): fragment_check_list.append(triggeractivity_frag_params) # Run some tests on the output data file - assert len(run_nanorc.data_files) == 6 # three for each run + all_ok = len(run_nanorc.data_files) == 6 # three for each run + print("") # Clear potential dot from pytest + if all_ok: + print("\N{WHITE HEAVY CHECK MARK} The correct number of raw data files was found (6)") + else: + print(f"\N{POLICE CARS REVOLVING LIGHT} An incorrect number of raw data files was found, expected 6, found {len(run_nanorc.data_files)} \N{POLICE CARS REVOLVING LIGHT}") - all_ok = True for idx in range(len(run_nanorc.data_files)): data_file = data_file_checks.DataFile(run_nanorc.data_files[idx]) all_ok &= data_file_checks.sanity_check(data_file) @@ -222,16 +249,20 @@ def test_data_files(run_nanorc): all_ok &= data_file_checks.check_fragment_sizes( data_file, fragment_check_list[jdx] ) - assert all_ok + assert all_ok, "\N{POLICE CARS REVOLVING LIGHT} One or more raw data file checks failed! \N{POLICE CARS REVOLVING LIGHT}" def test_tpstream_files(run_nanorc): tpstream_files = run_nanorc.tpset_files fragment_check_list = [wibeth_tpset_params] # WIBEth - assert len(tpstream_files) == 6 # three for each run + all_ok = len(tpstream_files) == 6 # three for each run + print("") # Clear potential dot from pytest + if all_ok: + print("\N{WHITE HEAVY CHECK MARK} The correct number of TP-stream data files was found (6)") + else: + print(f"\N{POLICE CARS REVOLVING LIGHT} An incorrect number of TP-stream data files was found, expected 6, found {len(tpstream_files)} \N{POLICE CARS REVOLVING LIGHT}") - all_ok = True for idx in range(len(tpstream_files)): data_file = data_file_checks.DataFile(tpstream_files[idx]) all_ok &= data_file_checks.check_file_attributes(data_file) @@ -239,7 +270,7 @@ def test_tpstream_files(run_nanorc): all_ok &= data_file_checks.check_fragment_count( data_file, fragment_check_list[jdx] ) - assert all_ok + assert all_ok, "\N{POLICE CARS REVOLVING LIGHT} One or more TP-stream data file checks failed! \N{POLICE CARS REVOLVING LIGHT}" def test_cleanup(run_nanorc): @@ -265,6 +296,8 @@ def test_cleanup(run_nanorc): for data_file in run_nanorc.data_files: data_file.unlink() + for data_file in run_nanorc.tpset_files: + data_file.unlink() print("--------------------") os.system(f"df -h {pathlist_string}") diff --git a/integtest/multiple_data_writers_test.py b/integtest/multiple_data_writers_test.py new file mode 100644 index 00000000..51c48692 --- /dev/null +++ b/integtest/multiple_data_writers_test.py @@ -0,0 +1,178 @@ +import pytest +import os +import re +import copy +import urllib.request + +import integrationtest.data_file_checks as data_file_checks +import integrationtest.log_file_checks as log_file_checks +import integrationtest.data_classes as data_classes + +pytest_plugins = "integrationtest.integrationtest_drunc" + +# Values that help determine the running conditions +number_of_data_producers = 2 +number_of_readout_apps = 3 + +# Default values for validation parameters +check_for_logfile_errors = True + +wibeth_frag_params = { + "fragment_type_description": "WIBEth", + "fragment_type": "WIBEth", + "expected_fragment_count": (number_of_data_producers * number_of_readout_apps), + "min_size_bytes": 187272, + "max_size_bytes": 194472, +} +# sizes: 128 is for one TC with zero TAs inside it (72+56) +# 208 is for one TC with one TA inside it (72+56+80) +# 264 is for two TCs with one TA in one of them (72+56+80+56) +triggercandidate_frag_params = { + "fragment_type_description": "Trigger Candidate", + "fragment_type": "Trigger_Candidate", + "expected_fragment_count": 1, + "min_size_bytes": 128, + "max_size_bytes": 128, +} +triggeractivity_frag_params = { + "fragment_type_description": "Trigger Activity", + "fragment_type": "Trigger_Activity", + "expected_fragment_count": 0, + "min_size_bytes": 72, + "max_size_bytes": 632, +} +triggerprimitive_frag_params = { + "fragment_type_description": "Trigger Primitive", + "fragment_type": "Trigger_Primitive", + "expected_fragment_count": 0, + "min_size_bytes": 72, + "max_size_bytes": 1032, +} +hsi_frag_params = { + "fragment_type_description": "HSI", + "fragment_type": "Hardware_Signal", + "expected_fragment_count": 1, + "min_size_bytes": 100, + "max_size_bytes": 100, +} +ignored_logfile_problems = { + "-controller": [ + "Worker with pid \\d+ was terminated due to signal 1", + ], + "connectivity-service": [ + "errorlog: -", + ], +} + +# The next three variable declarations *must* be present as globals in the test +# file. They're read by the "fixtures" in conftest.py to determine how +# to run the config generation and nanorc + +object_databases = ["config/daqsystemtest/integrationtest-objects.data.xml"] + +conf_dict = data_classes.drunc_config() +conf_dict.dro_map_config.n_streams = number_of_data_producers +conf_dict.dro_map_config.n_apps = number_of_readout_apps +conf_dict.op_env = "integtest" +conf_dict.session = "multidatawriter" +conf_dict.fake_hsi_enabled = True +conf_dict.n_data_writers = 3 + +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="FakeHSIEventGeneratorConf", + updates={"trigger_rate": 10.0}, + ) +) + +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="HSISignalWindow", + updates={ + "time_before": 1000, + "time_after": 500, + }, + ) +) +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="TCReadoutMap", + obj_id = "def-hsi-tc-map", + updates={ + "time_before": 52000, + "time_after": 1000, + }, + ) +) + +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="QueueDescriptor", + obj_id="trigger-records", + updates={"queue_type": "kFollyMPMCQueue"}, + ) +) + + +confgen_arguments = { + "WIBEth_System": conf_dict, +} + +# The commands to run in nanorc, as a list +nanorc_command_list = ( + "boot conf wait 5".split() + + "start --run-number 101 wait 1 enable-triggers wait 30".split() + + "disable-triggers wait 2 drain-dataflow wait 2 stop-trigger-sources stop ".split() + + "start --run-number 102 wait 1 enable-triggers wait 30".split() + + "disable-triggers wait 2 drain-dataflow wait 2 stop-trigger-sources stop ".split() + + " scrap terminate".split() +) + +# The tests themselves + +def test_nanorc_success(run_nanorc): + current_test = os.environ.get("PYTEST_CURRENT_TEST") + match_obj = re.search(r".*\[(.+)-run_nanorc0\].*", current_test) + if match_obj: + current_test = match_obj.group(1) + banner_line = re.sub(".", "=", current_test) + print(banner_line) + print(current_test) + print(banner_line) + # Check that nanorc completed correctly + assert run_nanorc.completed_process.returncode == 0 + + +def test_log_files(run_nanorc): + if check_for_logfile_errors: + # Check that there are no warnings or errors in the log files + assert log_file_checks.logs_are_error_free( + run_nanorc.log_files, True, True, ignored_logfile_problems + ) + + +def test_data_files(run_nanorc): + fragment_check_list = [triggercandidate_frag_params, hsi_frag_params, wibeth_frag_params] + fragment_check_list.append(triggerprimitive_frag_params) + fragment_check_list.append(triggeractivity_frag_params) + + # Run some tests on the output data file + all_ok = len(run_nanorc.data_files) == 6 # three for each run + print("") # Clear potential dot from pytest + if all_ok: + print("\N{WHITE HEAVY CHECK MARK} The correct number of raw data files was found (6)") + else: + print(f"\N{POLICE CARS REVOLVING LIGHT} An incorrect number of raw data files was found, expected 6, found {len(run_nanorc.data_files)} \N{POLICE CARS REVOLVING LIGHT}") + + for idx in range(len(run_nanorc.data_files)): + data_file = data_file_checks.DataFile(run_nanorc.data_files[idx]) + all_ok &= data_file_checks.sanity_check(data_file) + all_ok &= data_file_checks.check_file_attributes(data_file) + for jdx in range(len(fragment_check_list)): + all_ok &= data_file_checks.check_fragment_count( + data_file, fragment_check_list[jdx] + ) + all_ok &= data_file_checks.check_fragment_sizes( + data_file, fragment_check_list[jdx] + ) + assert all_ok, "\N{POLICE CARS REVOLVING LIGHT} One or more raw data file checks failed! \N{POLICE CARS REVOLVING LIGHT}" diff --git a/pytest.ini b/pytest.ini index 370e923b..79a333e7 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,3 +2,11 @@ filterwarnings = ignore:.*is a deprecated alias.*:DeprecationWarning ignore:.*invalid escape sequence.*:DeprecationWarning + +tmp_path_retention_count = 10 + +# --tb not given Produces reams of output, with full source code included in tracebacks +# --tb=no Just shows location of failure in the test file: no use for tracking down errors +# --tb=short Just shows vanilla traceback: very useful, but file names are incomplete and relative +# --tb=native Slightly more info than short: still works very well. The full paths may be useful for CI +addopts = --tb=no diff --git a/scripts/dfmodules_integtest_bundle.sh b/scripts/dfmodules_integtest_bundle.sh index 3caef7de..45874a70 100755 --- a/scripts/dfmodules_integtest_bundle.sh +++ b/scripts/dfmodules_integtest_bundle.sh @@ -1,7 +1,9 @@ #!/bin/bash -# 11-Oct-2023, KAB +# 29-Apr-2025, KAB -integtest_list=( "large_trigger_record_test.py" "disabled_output_test.py" "multi_output_file_test.py" "insufficient_disk_space_test.py" ) +integtest_list=( "max_file_size_test.py" "multiple_data_writers_test.py" "hdf5_compression_test.py" ) +#integtest_list=( "large_trigger_record_test.py" "disabled_output_test.py" "multi_output_file_test.py" "insufficient_disk_space_test.py" ) +let last_test_index=${#integtest_list[@]}-1 usage() { declare -r script_name=$(basename "$0") @@ -11,9 +13,9 @@ Usage: Options: -h, --help : prints out usage information - -s -f - -l + -l + -k -n -N --stop-on-failure : causes the script to stop when one of the integtests reports a failure @@ -28,16 +30,15 @@ Options: echo "" } -TEMP=`getopt -o hs:f:l:n:N: --long help,stop-on-failure,stop-on-skip -- "$@"` +TEMP=`getopt -o hs:f:l:k:n:N: --long help,stop-on-failure,stop-on-skip -- "$@"` eval set -- "$TEMP" -let session_number=1 let first_test_index=0 -let last_test_index=999 -let individual_run_count=1 -let overall_run_count=1 +let individual_test_requested_iterations=1 +let full_set_requested_interations=1 let stop_on_failure=0 let stop_on_skip=0 +requested_test_names= while true; do case "$1" in @@ -45,10 +46,6 @@ while true; do usage exit 0 ;; - -s) - let session_number=$2 - shift 2 - ;; -f) let first_test_index=$2 shift 2 @@ -57,12 +54,16 @@ while true; do let last_test_index=$2 shift 2 ;; + -k) + requested_test_names=$2 + shift 2 + ;; -n) - let individual_run_count=$2 + let individual_test_requested_iterations=$2 shift 2 ;; -N) - let overall_run_count=$2 + let full_set_requested_interations=$2 shift 2 ;; --stop-on-failure) @@ -80,37 +81,69 @@ while true; do esac done +# check if the numad daemon is running +numad_grep_output=`ps -ef | grep numad | grep -v grep` +if [[ "${numad_grep_output}" != "" ]]; then + echo "*********************************************************************" + echo "*** DANGER, DANGER, 'numad' appears to be running on this computer!" + echo "*** 'ps' output: ${numad_grep_output}" + echo "*** now if you want to abort this testing." + echo "*********************************************************************" + sleep 3 +fi + # other setup TIMESTAMP=`date '+%Y%m%d%H%M%S'` mkdir -p /tmp/pytest-of-${USER} ITGRUNNER_LOG_FILE="/tmp/pytest-of-${USER}/dfmodules_integtest_bundle_${TIMESTAMP}.log" -# run the tests -let overall_loop_count=0 -while [[ ${overall_loop_count} -lt ${overall_run_count} ]]; do +let number_of_individual_tests=0 +let test_index=0 +for TEST_NAME in ${integtest_list[@]}; do + if [[ ${test_index} -ge ${first_test_index} && ${test_index} -le ${last_test_index} ]]; then + requested_test=`echo ${TEST_NAME} | egrep -i ${requested_test_names:-${TEST_NAME}}` + if [[ "${requested_test}" != "" ]]; then + let number_of_individual_tests=${number_of_individual_tests}+1 + fi + fi + let test_index=${test_index}+1 +done +let total_number_of_tests=${number_of_individual_tests}*${individual_test_requested_iterations}*${full_set_requested_interations} +# run the tests +let overall_test_index=0 # this is only used for user feedback +let full_set_loop_count=0 +while [[ ${full_set_loop_count} -lt ${full_set_requested_interations} ]]; do let test_index=0 for TEST_NAME in ${integtest_list[@]}; do if [[ ${test_index} -ge ${first_test_index} && ${test_index} -le ${last_test_index} ]]; then + requested_test=`echo ${TEST_NAME} | egrep -i ${requested_test_names:-${TEST_NAME}}` + if [[ "${requested_test}" != "" ]]; then let individual_loop_count=0 - while [[ ${individual_loop_count} -lt ${individual_run_count} ]]; do - echo "===== Running ${TEST_NAME}" >> ${ITGRUNNER_LOG_FILE} + while [[ ${individual_loop_count} -lt ${individual_test_requested_iterations} ]]; do + let overall_test_index=${overall_test_index}+1 + echo "" + echo -e "\U0001F535 \033[0;34mStarting test ${overall_test_index} of ${total_number_of_tests}...\033[0m \U0001F535" | tee -a ${ITGRUNNER_LOG_FILE} + + echo -e "\u2B95 \033[0;1mRunning ${TEST_NAME}\033[0m \u2B05" | tee -a ${ITGRUNNER_LOG_FILE} if [[ -e "./${TEST_NAME}" ]]; then - pytest -s ./${TEST_NAME} --nanorc-option partition-number ${session_number} | tee -a ${ITGRUNNER_LOG_FILE} + pytest -s ./${TEST_NAME} | tee -a ${ITGRUNNER_LOG_FILE} elif [[ -e "${DBT_AREA_ROOT}/sourcecode/dfmodules/integtest/${TEST_NAME}" ]]; then - pytest -s ${DBT_AREA_ROOT}/sourcecode/dfmodules/integtest/${TEST_NAME} --nanorc-option partition-number ${session_number} | tee -a ${ITGRUNNER_LOG_FILE} + if [[ -w "${DBT_AREA_ROOT}" ]]; then + pytest -s ${DBT_AREA_ROOT}/sourcecode/dfmodules/integtest/${TEST_NAME} | tee -a ${ITGRUNNER_LOG_FILE} + else + pytest -s -p no:cacheprovider ${DBT_AREA_ROOT}/sourcecode/dfmodules/integtest/${TEST_NAME} | tee -a ${ITGRUNNER_LOG_FILE} + fi else - pytest -s ${DFMODULES_SHARE}/integtest/${TEST_NAME} --nanorc-option partition-number ${session_number} | tee -a ${ITGRUNNER_LOG_FILE} + pytest -s -p no:cacheprovider ${DFMODULES_SHARE}/integtest/${TEST_NAME} | tee -a ${ITGRUNNER_LOG_FILE} fi let pytest_return_code=${PIPESTATUS[0]} let individual_loop_count=${individual_loop_count}+1 if [[ ${stop_on_failure} -gt 0 ]]; then - search_result=`tail -20 ${ITGRUNNER_LOG_FILE} | grep -i fail` - #echo "failure search result is ${search_result}" - if [[ ${search_result} != "" || ${pytest_return_code} -ne 0 ]]; then + if [[ ${pytest_return_code} -ne 0 ]]; then break 3 fi fi @@ -123,11 +156,12 @@ while [[ ${overall_loop_count} -lt ${overall_run_count} ]]; do fi done + fi fi let test_index=${test_index}+1 done - let overall_loop_count=${overall_loop_count}+1 + let full_set_loop_count=${full_set_loop_count}+1 done # print out summary information @@ -140,4 +174,18 @@ echo "" | tee -a ${ITGRUNNER_L date | tee -a ${ITGRUNNER_LOG_FILE} echo "Log file is: ${ITGRUNNER_LOG_FILE}" | tee -a ${ITGRUNNER_LOG_FILE} echo "" | tee -a ${ITGRUNNER_LOG_FILE} -grep '=====' ${ITGRUNNER_LOG_FILE} | egrep ' in |Running' | tee -a ${ITGRUNNER_LOG_FILE} +egrep $'=====|\u2B95' ${ITGRUNNER_LOG_FILE} | egrep ' in |Running' | tee -a ${ITGRUNNER_LOG_FILE} + +# check again if the numad daemon is running +numad_grep_output=`ps -ef | grep numad | grep -v grep` +if [[ "${numad_grep_output}" != "" ]]; then + echo "" | tee -a ${ITGRUNNER_LOG_FILE} + echo "********************************************************************************" | tee -a ${ITGRUNNER_LOG_FILE} + echo "*** WARNING: 'numad' appears to be running on this computer!" | tee -a ${ITGRUNNER_LOG_FILE} + echo "*** 'ps' output: ${numad_grep_output}" | tee -a ${ITGRUNNER_LOG_FILE} + echo "*** This daemon can adversely affect the running of these tests, especially ones" | tee -a ${ITGRUNNER_LOG_FILE} + echo "*** that are resource intensive in the Readout Apps. This is because numad moves" | tee -a ${ITGRUNNER_LOG_FILE} + echo "*** processes (threads?) to different cores/numa nodes periodically, and that" | tee -a ${ITGRUNNER_LOG_FILE} + echo "*** context switch can disrupt the stable running of the DAQ processes." | tee -a ${ITGRUNNER_LOG_FILE} + echo "********************************************************************************" | tee -a ${ITGRUNNER_LOG_FILE} +fi