Added an hdf5_compression_test and restored a couple of the existing tests

integtest/README.md

@@ -1,16 +1,19 @@
-* 19-Jul-2023, KAB, ELF, and others: notes on existing integtests
+* 29-Apr-2025, KAB, ELF, and others: notes on existing integtests
 
 "integtests" are intended to be automated integration and/or system tests that make use of the
 "pytest" framework to validate the operation of the DAQ system in various scenarios.
 
 Here is a sample command for invoking a test (feel free to keep or drop the options in brackets, as you prefer):
 
 ```
-pytest -s disabled_output_test.py [--nanorc-option partition-number 2] [--nanorc-option timeout 300]
+pytest -s max_file_size_test.py [--nanorc-option log-level debug]  # this nanorc option is still useful even when using drunc
 ```
 
 For reference, here are the ideas behind the existing tests:
+* `max_file_size_test.py` - verifies that data files are closed when they reach a specified maximum file size (approximately)
+* `multiple_data_writers_test.py` - verifies that we can run multiple DataWriters for a single TriggerRecordBuilder
+* `hdf5_compression_test.py` - verifies that HDF5 compression is working as expected by writing several data files of known size
+
 * `large_trigger_record_test.py` - verify that TriggerRecords that are close to the size of a whole file get written to disk correctly
 * `disabled_output_test.py` - verify that the --disable-data-storage option works
-* `multi_output_file_test.py` - test that the file size maximum config parameter works
 * `insufficient_disk_space_test.py` - verify that the appropriate errors and warnings are produced when there isn't enough disk space to write data

integtest/disabled_output_test.py

@@ -22,82 +22,44 @@
 check_for_logfile_errors = True
 expected_event_count = trigger_rate * run_duration
 expected_event_count_tolerance = math.ceil(expected_event_count / 10)
-wib1_frag_hsi_trig_params = {
-    "fragment_type_description": "WIB",
-    "fragment_type": "ProtoWIB",
-    "hdf5_source_subsystem": "Detector_Readout",
-    "expected_fragment_count": number_of_data_producers,
-    "min_size_bytes": 37656,
-    "max_size_bytes": 37656,
-}
-wib1_frag_multi_trig_params = {
-    "fragment_type_description": "WIB",
-    "fragment_type": "ProtoWIB",
-    "hdf5_source_subsystem": "Detector_Readout",
-    "expected_fragment_count": number_of_data_producers,
-    "min_size_bytes": 72,
-    "max_size_bytes": 54000,
-}
-wib2_frag_hsi_trig_params = {
-    "fragment_type_description": "WIB",
-    "fragment_type": "WIB",
-    "hdf5_source_subsystem": "Detector_Readout",
-    "expected_fragment_count": (number_of_data_producers),
-    "min_size_bytes": 29808,
-    "max_size_bytes": 30280,
-}
-wib2_frag_multi_trig_params = {
-    "fragment_type_description": "WIB",
-    "fragment_type": "WIB",
-    "hdf5_source_subsystem": "Detector_Readout",
-    "expected_fragment_count": (number_of_data_producers),
-    "min_size_bytes": 72,
-    "max_size_bytes": 54000,
-}
 wibeth_frag_hsi_trig_params = {
     "fragment_type_description": "WIBEth",
     "fragment_type": "WIBEth",
-    "hdf5_source_subsystem": "Detector_Readout",
     "expected_fragment_count": (number_of_data_producers),
     "min_size_bytes": 7272,
     "max_size_bytes": 14472,
 }
 wibeth_frag_multi_trig_params = {
     "fragment_type_description": "WIBEth",
     "fragment_type": "WIBEth",
-    "hdf5_source_subsystem": "Detector_Readout",
     "expected_fragment_count": (number_of_data_producers),
     "min_size_bytes": 72,
     "max_size_bytes": 14472,
 }
 triggercandidate_frag_params = {
     "fragment_type_description": "Trigger Candidate",
     "fragment_type": "Trigger_Candidate",
-    "hdf5_source_subsystem": "Trigger",
     "expected_fragment_count": 1,
-    "min_size_bytes": 72,
+    "min_size_bytes": 128,
     "max_size_bytes": 280,
 }
 triggeractivity_frag_params = {
     "fragment_type_description": "Trigger Activity",
     "fragment_type": "Trigger_Activity",
-    "hdf5_source_subsystem": "Trigger",
     "expected_fragment_count": 1,
     "min_size_bytes": 72,
     "max_size_bytes": 216,
 }
-triggertp_frag_params = {
-    "fragment_type_description": "Trigger with TPs",
+triggerprimitive_frag_params = {
+    "fragment_type_description": "Trigger Primitive",
     "fragment_type": "Trigger_Primitive",
-    "hdf5_source_subsystem": "Trigger",
     "expected_fragment_count": 2,  # number of readout apps (1) times 2
     "min_size_bytes": 72,
     "max_size_bytes": 16000,
 }
 hsi_frag_params = {
     "fragment_type_description": "HSI",
     "fragment_type": "Hardware_Signal",
-    "hdf5_source_subsystem": "HW_Signals_Interface",
     "expected_fragment_count": 1,
     "min_size_bytes": 72,
     "max_size_bytes": 100,
@@ -192,7 +154,7 @@
 
 def test_nanorc_success(run_nanorc):
     current_test = os.environ.get("PYTEST_CURRENT_TEST")
-    match_obj = re.search(r".*\[(.+)\].*", current_test)
+    match_obj = re.search(r".*\[(.+)-run_nanorc0\].*", current_test)
     if match_obj:
         current_test = match_obj.group(1)
     banner_line = re.sub(".", "=", current_test)
@@ -222,30 +184,28 @@ def test_data_files(run_nanorc):
         local_event_count_tolerance += (
             10 * number_of_data_producers * run_duration / 100
         )
-        # fragment_check_list.append(wib1_frag_multi_trig_params) # ProtoWIB
-        # fragment_check_list.append(wib2_frag_multi_trig_params) # DuneWIB
         fragment_check_list.append(wibeth_frag_multi_trig_params)  # WIBEth
-        fragment_check_list.append(triggertp_frag_params)
+        fragment_check_list.append(triggerprimitive_frag_params)
         fragment_check_list.append(triggeractivity_frag_params)
     else:
-        # fragment_check_list.append(wib1_frag_hsi_trig_params) # ProtoWIB
-        # fragment_check_list.append(wib2_frag_hsi_trig_params) # DuneWIB
         fragment_check_list.append(wibeth_frag_hsi_trig_params)  # WIBEth
 
     # Run some tests on the output data file
     assert len(run_nanorc.data_files) == expected_number_of_data_files
 
+    all_ok = True
     for idx in range(len(run_nanorc.data_files)):
         data_file = data_file_checks.DataFile(run_nanorc.data_files[idx])
-        assert data_file_checks.sanity_check(data_file)
-        assert data_file_checks.check_file_attributes(data_file)
-        assert data_file_checks.check_event_count(
+        all_ok &= data_file_checks.sanity_check(data_file)
+        all_ok &= data_file_checks.check_file_attributes(data_file)
+        all_ok &= data_file_checks.check_event_count(
             data_file, local_expected_event_count, local_event_count_tolerance
         )
         for jdx in range(len(fragment_check_list)):
-            assert data_file_checks.check_fragment_count(
+            all_ok &= data_file_checks.check_fragment_count(
                 data_file, fragment_check_list[jdx]
             )
-            assert data_file_checks.check_fragment_sizes(
+            all_ok &= data_file_checks.check_fragment_sizes(
                 data_file, fragment_check_list[jdx]
             )
+    assert all_ok