From b200f553587550bf03bc81b5acc497a4294cc65e Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Fri, 19 Jan 2024 13:05:44 -0500
Subject: [PATCH 001/102] rename Calibration to SensorCalibration

---
 .../{calibration.py => sensor_calibration.py}      |  6 +++---
 scos_actions/hardware/mocks/mock_sigan.py          |  6 +++---
 scos_actions/hardware/sigan_iface.py               | 14 +++++++-------
 3 files changed, 13 insertions(+), 13 deletions(-)
 rename scos_actions/calibration/{calibration.py => sensor_calibration.py} (98%)

diff --git a/scos_actions/calibration/calibration.py b/scos_actions/calibration/sensor_calibration.py
similarity index 98%
rename from scos_actions/calibration/calibration.py
rename to scos_actions/calibration/sensor_calibration.py
index 2506d825..627202dc 100644
--- a/scos_actions/calibration/calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -10,7 +10,7 @@
 
 
 @dataclass
-class Calibration:
+class SensorCalibration:
     last_calibration_datetime: str
     calibration_parameters: List[str]
     calibration_data: dict
@@ -121,7 +121,7 @@ def update(
             outfile.write(json.dumps(cal_dict))
 
 
-def load_from_json(fname: Path, is_default: bool) -> Calibration:
+def load_from_json(fname: Path, is_default: bool) -> SensorCalibration:
     """
     Load a calibration from a JSON file.
 
@@ -155,7 +155,7 @@ def load_from_json(fname: Path, is_default: bool) -> Calibration:
             + f"Required fields: {required_keys}\n"
         )
     # Create and return the Calibration object
-    return Calibration(
+    return SensorCalibration(
         calibration["last_calibration_datetime"],
         calibration["calibration_parameters"],
         calibration["calibration_data"],
diff --git a/scos_actions/hardware/mocks/mock_sigan.py b/scos_actions/hardware/mocks/mock_sigan.py
index 1876851b..71cc71ba 100644
--- a/scos_actions/hardware/mocks/mock_sigan.py
+++ b/scos_actions/hardware/mocks/mock_sigan.py
@@ -4,7 +4,7 @@
 from typing import Optional
 
 import numpy as np
-from scos_actions.calibration.calibration import Calibration
+from scos_actions.calibration.sensor_calibration import SensorCalibration
 from scos_actions.hardware.sigan_iface import SignalAnalyzerInterface
 from scos_actions.utils import get_datetime_str_now
 
@@ -28,8 +28,8 @@ class MockSignalAnalyzer(SignalAnalyzerInterface):
 
     def __init__(
         self,
-        sensor_cal: Optional[Calibration] = None,
-        sigan_cal: Optional[Calibration] = None,
+        sensor_cal: Optional[SensorCalibration] = None,
+        sigan_cal: Optional[SensorCalibration] = None,
         randomize_values: bool = False,
     ):
         super().__init__(sensor_cal, sigan_cal)
diff --git a/scos_actions/hardware/sigan_iface.py b/scos_actions/hardware/sigan_iface.py
index 7ed01d45..f0eccf2c 100644
--- a/scos_actions/hardware/sigan_iface.py
+++ b/scos_actions/hardware/sigan_iface.py
@@ -4,7 +4,7 @@
 from typing import Dict, Optional
 
 from its_preselector.web_relay import WebRelay
-from scos_actions.calibration.calibration import Calibration
+from scos_actions.calibration.sensor_calibration import SensorCalibration
 from scos_actions.hardware.utils import power_cycle_sigan
 from scos_actions.utils import convert_string_to_millisecond_iso_format
 
@@ -25,8 +25,8 @@
 class SignalAnalyzerInterface(ABC):
     def __init__(
         self,
-        sensor_cal: Optional[Calibration] = None,
-        sigan_cal: Optional[Calibration] = None,
+        sensor_cal: Optional[SensorCalibration] = None,
+        sigan_cal: Optional[SensorCalibration] = None,
         switches: Optional[Dict[str, WebRelay]] = None,
     ):
         self.sensor_calibration_data = {}
@@ -166,17 +166,17 @@ def model(self, value: str):
         self._model = value
 
     @property
-    def sensor_calibration(self) -> Calibration:
+    def sensor_calibration(self) -> SensorCalibration:
         return self._sensor_calibration
 
     @sensor_calibration.setter
-    def sensor_calibration(self, cal: Calibration):
+    def sensor_calibration(self, cal: SensorCalibration):
         self._sensor_calibration = cal
 
     @property
-    def sigan_calibration(self) -> Calibration:
+    def sigan_calibration(self) -> SensorCalibration:
         return self._sigan_calibration
 
     @sigan_calibration.setter
-    def sigan_calibration(self, cal: Calibration):
+    def sigan_calibration(self, cal: SensorCalibration):
         self._sigan_calibration = cal

From 367e4db0e4a2975c1bf80b6d1415302840b7344f Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 22 Jan 2024 12:08:31 -0500
Subject: [PATCH 002/102] refactor calibration to use a base class

---
 .../calibration/interfaces/__init__.py        |   0
 .../calibration/interfaces/calibration.py     | 119 ++++++++++++
 .../calibration/sensor_calibration.py         | 174 +++---------------
 .../calibration/tests/test_calibration.py     |  21 +--
 scos_actions/calibration/utils.py             |  57 ++++++
 scos_actions/signal_processing/calibration.py |  13 +-
 6 files changed, 215 insertions(+), 169 deletions(-)
 create mode 100644 scos_actions/calibration/interfaces/__init__.py
 create mode 100644 scos_actions/calibration/interfaces/calibration.py
 create mode 100644 scos_actions/calibration/utils.py

diff --git a/scos_actions/calibration/interfaces/__init__.py b/scos_actions/calibration/interfaces/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/scos_actions/calibration/interfaces/calibration.py b/scos_actions/calibration/interfaces/calibration.py
new file mode 100644
index 00000000..9fe9aac3
--- /dev/null
+++ b/scos_actions/calibration/interfaces/calibration.py
@@ -0,0 +1,119 @@
+import dataclasses
+import json
+import logging
+from abc import abstractmethod
+from pathlib import Path
+from typing import Any, List
+
+from scos_actions.calibration.utils import filter_by_parameter
+
+logger = logging.getLogger(__name__)
+
+
+@dataclasses.dataclass
+class Calibration:
+    calibration_parameters: List[str]
+    calibration_data: dict
+    is_default: bool
+    file_path: Path
+
+    def __post_init__(self):
+        # Convert key names in data to strings
+        # This means that formatting will always match between
+        # native types provided in Python and data loaded from JSON
+        self.calibration_data = json.loads(json.dumps(self.calibration_data))
+
+    def get_calibration_dict(self, cal_params: List[Any]) -> dict:
+        """
+        Get calibration data closest to the specified parameter values.
+
+        :param cal_params: List of calibration parameter values. For example,
+            if ``calibration_parameters`` are ``["sample_rate", "gain"]``,
+            then the input to this method could be ``["15360000.0", "40"]``.
+        :return: The calibration data corresponding to the input parameter values.
+        """
+        cal_data = self.calibration_data
+        for i, setting_value in enumerate(cal_params):
+            setting = self.calibration_parameters[i]
+            logger.debug(f"Looking up calibration for {setting} at {setting_value}")
+            cal_data = filter_by_parameter(cal_data, setting_value)
+        logger.debug(f"Got calibration data: {cal_data}")
+
+        return cal_data
+
+    def _retrieve_data_to_update(self, params: dict) -> dict:
+        """
+        Locate the calibration data entry to update, based on a set
+        of calibration parameters.
+
+        :param params: Parameters used for calibration. This must include
+            entries for all of the ``Calibration.calibration_parameters``
+            Example: ``{"sample_rate": 14000000.0, "attenuation": 10.0}``
+        :return: A dict containing the existing calibration entry at
+            the specified parameter set, which may be empty if none exists.
+        """
+        # Use params keys as calibration_parameters if none exist
+        if len(self.calibration_parameters) == 0:
+            logger.warning(
+                f"Setting required calibration parameters to {list(params.keys())}"
+            )
+            self.calibration_parameters = list(params.keys())
+        elif not set(params.keys()) >= set(self.calibration_parameters):
+            # Otherwise ensure all required parameters were used
+            raise Exception(
+                "Not enough parameters specified to update calibration.\n"
+                + f"Required parameters are {self.calibration_parameters}"
+            )
+
+        # Retrieve the existing calibration data entry based on
+        # the provided parameters and their values
+        data_entry = self.calibration_data
+        for parameter in self.calibration_parameters:
+            value = str(params[parameter]).lower()
+            logger.debug(f"Updating calibration at {parameter} = {value}")
+            try:
+                data_entry = data_entry[value]
+            except KeyError:
+                logger.debug(
+                    f"Creating required calibration data field for {parameter} = {value}"
+                )
+                data_entry[value] = {}
+                data_entry = data_entry[value]
+        return data_entry
+
+    @abstractmethod
+    def update():
+        """Update the calibration data"""
+        pass
+
+    @classmethod
+    def from_json(cls, fname: Path, is_default: bool):
+        """
+        Load a calibration from a JSON file.
+
+        The JSON file must contain top-level fields:
+            ``calibration_parameters``
+            ``calibration_data``
+
+        :param fname: The ``Path`` to the JSON calibration file.
+        :param is_default: If True, the loaded calibration file
+            is treated as the default calibration file.
+        :raises Exception: If the provided file does not include
+            the required keys.
+        :return: The ``Calibration`` object generated from the file.
+        """
+        with open(fname) as file:
+            calibration = json.load(file)
+
+        # Check that the required fields are in the dict
+        required_keys = set(dataclasses.fields(cls).keys())
+
+        if not set(calibration.keys()) >= required_keys:
+            raise Exception(
+                "Loaded calibration dictionary is missing required fields."
+                + f"Existing fields: {set(calibration.keys())}\n"
+                + f"Required fields: {required_keys}\n"
+            )
+
+        # Create and return the Calibration object
+        return cls(is_default=is_default, file_path=fname, **calibration)
diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index 627202dc..3132c9d1 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -4,25 +4,15 @@
 from pathlib import Path
 from typing import Dict, List, Union
 
-from scos_actions.signal_processing.calibration import CalibrationException
+from scos_actions.calibration.interfaces.calibration import Calibration
 
 logger = logging.getLogger(__name__)
 
 
 @dataclass
-class SensorCalibration:
+class SensorCalibration(Calibration):
     last_calibration_datetime: str
-    calibration_parameters: List[str]
-    calibration_data: dict
     clock_rate_lookup_by_sample_rate: List[Dict[str, float]]
-    is_default: bool
-    file_path: Path
-
-    def __post_init__(self):
-        # Convert key names in calibration_data to strings
-        # This means that formatting will always match between
-        # native types provided in Python and data loaded from JSON
-        self.calibration_data = json.loads(json.dumps(self.calibration_data))
 
     def get_clock_rate(self, sample_rate: Union[float, int]) -> Union[float, int]:
         """Find the clock rate (Hz) using the given sample_rate (samples per second)"""
@@ -31,25 +21,6 @@ def get_clock_rate(self, sample_rate: Union[float, int]) -> Union[float, int]:
                 return mapping["clock_frequency"]
         return sample_rate
 
-    def get_calibration_dict(self, cal_params: List[Union[float, int, bool]]) -> dict:
-        """
-        Get calibration data closest to the specified parameter values.
-
-        :param cal_params: List of calibration parameter values. For example,
-            if ``calibration_parameters`` are ``["sample_rate", "gain"]``,
-            then the input to this method could be ``["15360000.0", "40"]``.
-        :return: The calibration data corresponding to the input parameter values.
-        """
-
-        cal_data = self.calibration_data
-        for i, setting_value in enumerate(cal_params):
-            setting = self.calibration_parameters[i]
-            logger.debug(f"Looking up calibration for {setting} at {setting_value}")
-            cal_data = filter_by_parameter(cal_data, setting_value)
-        logger.debug(f"Got calibration data: {cal_data}")
-
-        return cal_data
-
     def update(
         self,
         params: dict,
@@ -76,32 +47,14 @@ def update(
         :param file_path: File path for saving the updated calibration data.
         :raises Exception:
         """
-        cal_data = self.calibration_data
-        self.last_calibration_datetime = calibration_datetime_str
-        if len(self.calibration_parameters) == 0:
-            self.calibration_parameters = list(params.keys())
-        # Ensure all required calibration parameters were used
-        elif not set(params.keys()) >= set(self.calibration_parameters):
-            raise Exception(
-                "Not enough parameters specified to update calibration.\n"
-                + f"Required parameters are {self.calibration_parameters}"
-            )
+        # Get existing calibration data entry which will be updated
+        data_entry = self._retrieve_data_to_update(params)
 
-        # Get calibration entry by parameters used
-        for parameter in self.calibration_parameters:
-            value = str(params[parameter]).lower()
-            logger.debug(f"Updating calibration at {parameter} = {value}")
-            try:
-                cal_data = cal_data[value]
-            except KeyError:
-                logger.debug(
-                    f"Creating required calibration data field for {parameter} = {value}"
-                )
-                cal_data[value] = {}
-                cal_data = cal_data[value]
+        # Update last calibration datetime
+        self.last_calibration_datetime = calibration_datetime_str
 
-        # Update calibration data
-        cal_data.update(
+        # Update calibration data entry (updates entry in self.calibration_data)
+        data_entry.update(
             {
                 "datetime": calibration_datetime_str,
                 "gain": gain_dB,
@@ -120,95 +73,22 @@ def update(
         with open(self.file_path, "w") as outfile:
             outfile.write(json.dumps(cal_dict))
 
-
-def load_from_json(fname: Path, is_default: bool) -> SensorCalibration:
-    """
-    Load a calibration from a JSON file.
-
-    The JSON file must contain top-level fields:
-        ``last_calibration_datetime``
-        ``calibration_parameters``
-        ``calibration_data``
-        ``clock_rate_lookup_by_sample_rate``
-
-    :param fname: The ``Path`` to the JSON calibration file.
-    :param is_default: If True, the loaded calibration file
-        is treated as the default calibration file.
-    :raises Exception: If the provided file does not include
-        the required keys.
-    :return: The ``Calibration`` object generated from the file.
-    """
-    with open(fname) as file:
-        calibration = json.load(file)
-    # Check that the required fields are in the dict
-    required_keys = {
-        "last_calibration_datetime",
-        "calibration_data",
-        "clock_rate_lookup_by_sample_rate",
-        "calibration_parameters",
-    }
-
-    if not calibration.keys() >= required_keys:
-        raise Exception(
-            "Loaded calibration dictionary is missing required fields."
-            + f"Existing fields: {set(calibration.keys())}\n"
-            + f"Required fields: {required_keys}\n"
-        )
-    # Create and return the Calibration object
-    return SensorCalibration(
-        calibration["last_calibration_datetime"],
-        calibration["calibration_parameters"],
-        calibration["calibration_data"],
-        calibration["clock_rate_lookup_by_sample_rate"],
-        is_default,
-        fname,
-    )
-
-
-def filter_by_parameter(calibrations: dict, value: Union[float, int, bool]) -> dict:
-    """
-    Select a certain element by the value of a top-level key in a dictionary.
-
-    This method should be recursively called to select calibration
-    data matching a set of calibration parameters. The ordering of
-    nested dictionaries should match the ordering of the required
-    calibration parameters in the calibration file.
-
-    If ``value`` is a float or bool, ``str(value).lower()`` is used
-    as the dictionary key. If ``value`` is an int, and the previous
-    approach does not work, ``str(float(value))`` is attempted. This
-    allows for value ``1`` to match a key ``"1.0"``.
-
-    :param calibrations: Calibration data dictionary.
-    :param value: The parameter value for filtering. This value should
-        exist as a top-level key in ``calibrations``.
-    :raises CalibrationException: If ``value`` cannot be matched to a
-        top-level key in ``calibrations``, or if ``calibrations`` is not
-        a dict.
-    :return: The value of ``calibrations[value]``, which should be a dict.
-    """
-    try:
-        filtered_data = calibrations.get(str(value).lower(), None)
-        if filtered_data is None and isinstance(value, int):
-            # Try equivalent float for ints, i.e., match "1.0" to 1
-            filtered_data = calibrations.get(str(float(value)), None)
-        if filtered_data is None and isinstance(value, float) and value.is_integer():
-            # Check for, e.g., key '25' if value is '25.0'
-            filtered_data = calibrations.get(str(int(value)), None)
-        if filtered_data is None:
-            raise KeyError
-        else:
-            return filtered_data
-    except AttributeError as e:
-        # calibrations does not have ".get()"
-        # Generally means that calibrations is None or not a dict
-        msg = f"Provided calibration data is not a dict: {calibrations}"
-        raise CalibrationException(msg)
-    except KeyError as e:
-        msg = (
-            f"Could not locate calibration data at {value}"
-            + f"\nAttempted lookup using key '{str(value).lower()}'"
-            + f"{f'and {float(value)}' if isinstance(value, int) else ''}"
-            + f"\nUsing calibration data: {calibrations}"
-        )
-        raise CalibrationException(msg)
+    # @classmethod
+    # def from_json(cls, fname: Path, is_default: bool):
+    #     """
+    #     Load a sensor calibration from a JSON file.
+
+    #     The JSON file must contain top-level fields:
+    #         ``calibration_parameters``
+    #         ``calibration_data``
+    #         ``last_calibration_datetime``
+    #         ``clock_rate_lookup_by_sample_rate``
+
+    #     :param fname: The ``Path`` to the JSON calibration file.
+    #     :param is_default: If True, the loaded calibration file
+    #         is treated as the default calibration file.
+    #     :raises Exception: If the provided file does not include
+    #         the required keys.
+    #     :return: The ``Calibration`` object generated from the file.
+    #     """
+    #     return super().from_json(fname, is_default)
diff --git a/scos_actions/calibration/tests/test_calibration.py b/scos_actions/calibration/tests/test_calibration.py
index 9e503acd..c4484a90 100644
--- a/scos_actions/calibration/tests/test_calibration.py
+++ b/scos_actions/calibration/tests/test_calibration.py
@@ -8,18 +8,13 @@
 from pathlib import Path
 
 import pytest
-
-from scos_actions.calibration.calibration import (
-    Calibration,
-    filter_by_parameter,
-    load_from_json,
-)
-from scos_actions.signal_processing.calibration import CalibrationException
+from scos_actions.calibration.sensor_calibration import SensorCalibration
+from scos_actions.calibration.utils import CalibrationException, filter_by_parameter
 from scos_actions.tests.resources.utils import easy_gain
 from scos_actions.utils import get_datetime_str_now, parse_datetime_iso_format_str
 
 
-class TestCalibrationFile:
+class TestSensorCalibrationFile:
     # Ensure we load the test file
     setup_complete = False
 
@@ -180,7 +175,7 @@ def setup_calibration_file(self, tmpdir):
             json.dump(cal_data, file, indent=4)
 
         # Load the data back in
-        self.sample_cal = load_from_json(self.calibration_file, False)
+        self.sample_cal = SensorCalibration.from_json(self.calibration_file, False)
 
         # Create a list of previous points to ensure that we don't repeat
         self.pytest_points = []
@@ -229,7 +224,7 @@ def test_get_calibration_dict_exact_match_lookup(self):
             200.0: {100.0: {"NF": "NF at 200, 100", "Gain": "Gain at 200, 100"}},
         }
         clock_rate_lookup_by_sample_rate = {}
-        cal = Calibration(
+        cal = SensorCalibration(
             calibration_datetime,
             calibration_params,
             calibration_data,
@@ -249,7 +244,7 @@ def test_get_calibration_dict_within_range(self):
         }
         clock_rate_lookup_by_sample_rate = {}
         test_cal_path = Path("test_calibration.json")
-        cal = Calibration(
+        cal = SensorCalibration(
             calibration_datetime,
             calibration_params,
             calibration_data,
@@ -291,7 +286,7 @@ def test_update(self):
         calibration_data = {100.0: {200.0: {"noise_figure": 0, "gain": 0}}}
         clock_rate_lookup_by_sample_rate = {}
         test_cal_path = Path("test_calibration.json")
-        cal = Calibration(
+        cal = SensorCalibration(
             calibration_datetime,
             calibration_params,
             calibration_data,
@@ -302,7 +297,7 @@ def test_update(self):
         action_params = {"sample_rate": 100.0, "frequency": 200.0}
         update_time = get_datetime_str_now()
         cal.update(action_params, update_time, 30.0, 5.0, 21)
-        cal_from_file = load_from_json(test_cal_path, False)
+        cal_from_file = SensorCalibration.from_json(test_cal_path, False)
         test_cal_path.unlink()
         file_utc_time = parse_datetime_iso_format_str(cal.last_calibration_datetime)
         cal_time_utc = parse_datetime_iso_format_str(update_time)
diff --git a/scos_actions/calibration/utils.py b/scos_actions/calibration/utils.py
new file mode 100644
index 00000000..b3aebdeb
--- /dev/null
+++ b/scos_actions/calibration/utils.py
@@ -0,0 +1,57 @@
+from typing import Union
+
+
+class CalibrationException(Exception):
+    """Basic exception handling for calibration functions."""
+
+    def __init__(self, msg):
+        super().__init__(msg)
+
+
+def filter_by_parameter(calibrations: dict, value: Union[float, int, bool]) -> dict:
+    """
+    Select a certain element by the value of a top-level key in a dictionary.
+
+    This method should be recursively called to select calibration
+    data matching a set of calibration parameters. The ordering of
+    nested dictionaries should match the ordering of the required
+    calibration parameters in the calibration file.
+
+    If ``value`` is a float or bool, ``str(value).lower()`` is used
+    as the dictionary key. If ``value`` is an int, and the previous
+    approach does not work, ``str(float(value))`` is attempted. This
+    allows for value ``1`` to match a key ``"1.0"``.
+
+    :param calibrations: Calibration data dictionary.
+    :param value: The parameter value for filtering. This value should
+        exist as a top-level key in ``calibrations``.
+    :raises CalibrationException: If ``value`` cannot be matched to a
+        top-level key in ``calibrations``, or if ``calibrations`` is not
+        a dict.
+    :return: The value of ``calibrations[value]``, which should be a dict.
+    """
+    try:
+        filtered_data = calibrations.get(str(value).lower(), None)
+        if filtered_data is None and isinstance(value, int):
+            # Try equivalent float for ints, i.e., match "1.0" to 1
+            filtered_data = calibrations.get(str(float(value)), None)
+        if filtered_data is None and isinstance(value, float) and value.is_integer():
+            # Check for, e.g., key '25' if value is '25.0'
+            filtered_data = calibrations.get(str(int(value)), None)
+        if filtered_data is None:
+            raise KeyError
+        else:
+            return filtered_data
+    except AttributeError as e:
+        # calibrations does not have ".get()"
+        # Generally means that calibrations is None or not a dict
+        msg = f"Provided calibration data is not a dict: {calibrations}"
+        raise CalibrationException(msg)
+    except KeyError as e:
+        msg = (
+            f"Could not locate calibration data at {value}"
+            + f"\nAttempted lookup using key '{str(value).lower()}'"
+            + f"{f'and {float(value)}' if isinstance(value, int) else ''}"
+            + f"\nUsing calibration data: {calibrations}"
+        )
+        raise CalibrationException(msg)
diff --git a/scos_actions/signal_processing/calibration.py b/scos_actions/signal_processing/calibration.py
index 41ef5d71..757dfc50 100644
--- a/scos_actions/signal_processing/calibration.py
+++ b/scos_actions/signal_processing/calibration.py
@@ -3,7 +3,9 @@
 
 import numpy as np
 from its_preselector.preselector import Preselector
+from numpy.typing import NDArray
 from scipy.constants import Boltzmann
+from scos_actions.calibration.utils import CalibrationException
 from scos_actions.signal_processing.unit_conversion import (
     convert_celsius_to_fahrenheit,
     convert_celsius_to_kelvins,
@@ -15,16 +17,9 @@
 logger = logging.getLogger(__name__)
 
 
-class CalibrationException(Exception):
-    """Basic exception handling for calibration functions."""
-
-    def __init__(self, msg):
-        super().__init__(msg)
-
-
 def y_factor(
-    pwr_noise_on_watts: np.ndarray,
-    pwr_noise_off_watts: np.ndarray,
+    pwr_noise_on_watts: NDArray,
+    pwr_noise_off_watts: NDArray,
     enr_linear: float,
     enbw_hz: float,
     temp_kelvins: float = 300.0,

From 9f68928781f94c94009fda858d7da4cf0c14111d Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 22 Jan 2024 12:42:35 -0500
Subject: [PATCH 003/102] fix tests, add sensor_uid to SensorCalibration

---
 .../calibration/interfaces/calibration.py     |  33 +-
 .../calibration/sensor_calibration.py         |  22 +-
 .../calibration/tests/test_calibration.py     | 318 +-----------------
 .../tests/test_sensor_calibration.py          | 315 +++++++++++++++++
 4 files changed, 340 insertions(+), 348 deletions(-)
 create mode 100644 scos_actions/calibration/tests/test_sensor_calibration.py

diff --git a/scos_actions/calibration/interfaces/calibration.py b/scos_actions/calibration/interfaces/calibration.py
index 9fe9aac3..debe42ba 100644
--- a/scos_actions/calibration/interfaces/calibration.py
+++ b/scos_actions/calibration/interfaces/calibration.py
@@ -84,16 +84,16 @@ def _retrieve_data_to_update(self, params: dict) -> dict:
     @abstractmethod
     def update():
         """Update the calibration data"""
-        pass
+        raise NotImplementedError
 
     @classmethod
     def from_json(cls, fname: Path, is_default: bool):
         """
         Load a calibration from a JSON file.
 
-        The JSON file must contain top-level fields:
-            ``calibration_parameters``
-            ``calibration_data``
+        The JSON file must contain top-level fields
+        with names identical to the dataclass fields for
+        the class being constructed.
 
         :param fname: The ``Path`` to the JSON calibration file.
         :param is_default: If True, the loaded calibration file
@@ -104,15 +104,26 @@ def from_json(cls, fname: Path, is_default: bool):
         """
         with open(fname) as file:
             calibration = json.load(file)
-
-        # Check that the required fields are in the dict
-        required_keys = set(dataclasses.fields(cls).keys())
-
-        if not set(calibration.keys()) >= required_keys:
+        cal_file_keys = set(calibration.keys())
+
+        # Check that only the required fields are in the dict
+        required_keys = {f.name for f in dataclasses.fields(cls)}
+        required_keys -= {"is_default", "file_path"}  # are not required in JSON
+        if cal_file_keys == required_keys:
+            pass
+        elif cal_file_keys >= required_keys:
+            extra_keys = cal_file_keys - required_keys
+            logger.warning(
+                f"Loaded calibration file contains fields which will be ignored: {extra_keys}"
+            )
+            for k in extra_keys:
+                calibration.pop(k, None)
+        else:
             raise Exception(
-                "Loaded calibration dictionary is missing required fields."
-                + f"Existing fields: {set(calibration.keys())}\n"
+                "Loaded calibration dictionary is missing required fields.\n"
+                + f"Existing fields: {cal_file_keys}\n"
                 + f"Required fields: {required_keys}\n"
+                + f"Missing fields: {required_keys - cal_file_keys}"
             )
 
         # Create and return the Calibration object
diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index 3132c9d1..11865206 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -13,6 +13,7 @@
 class SensorCalibration(Calibration):
     last_calibration_datetime: str
     clock_rate_lookup_by_sample_rate: List[Dict[str, float]]
+    sensor_uid: str
 
     def get_clock_rate(self, sample_rate: Union[float, int]) -> Union[float, int]:
         """Find the clock rate (Hz) using the given sample_rate (samples per second)"""
@@ -66,29 +67,10 @@ def update(
         # Write updated calibration data to file
         cal_dict = {
             "last_calibration_datetime": self.last_calibration_datetime,
+            "sensor_uid": self.sensor_uid,
             "calibration_parameters": self.calibration_parameters,
             "clock_rate_lookup_by_sample_rate": self.clock_rate_lookup_by_sample_rate,
             "calibration_data": self.calibration_data,
         }
         with open(self.file_path, "w") as outfile:
             outfile.write(json.dumps(cal_dict))
-
-    # @classmethod
-    # def from_json(cls, fname: Path, is_default: bool):
-    #     """
-    #     Load a sensor calibration from a JSON file.
-
-    #     The JSON file must contain top-level fields:
-    #         ``calibration_parameters``
-    #         ``calibration_data``
-    #         ``last_calibration_datetime``
-    #         ``clock_rate_lookup_by_sample_rate``
-
-    #     :param fname: The ``Path`` to the JSON calibration file.
-    #     :param is_default: If True, the loaded calibration file
-    #         is treated as the default calibration file.
-    #     :raises Exception: If the provided file does not include
-    #         the required keys.
-    #     :return: The ``Calibration`` object generated from the file.
-    #     """
-    #     return super().from_json(fname, is_default)
diff --git a/scos_actions/calibration/tests/test_calibration.py b/scos_actions/calibration/tests/test_calibration.py
index c4484a90..3d743de1 100644
--- a/scos_actions/calibration/tests/test_calibration.py
+++ b/scos_actions/calibration/tests/test_calibration.py
@@ -1,317 +1 @@
-"""Test aspects of ScaleFactors."""
-
-import datetime
-import json
-import random
-from copy import deepcopy
-from math import isclose
-from pathlib import Path
-
-import pytest
-from scos_actions.calibration.sensor_calibration import SensorCalibration
-from scos_actions.calibration.utils import CalibrationException, filter_by_parameter
-from scos_actions.tests.resources.utils import easy_gain
-from scos_actions.utils import get_datetime_str_now, parse_datetime_iso_format_str
-
-
-class TestSensorCalibrationFile:
-    # Ensure we load the test file
-    setup_complete = False
-
-    def rand_index(self, l):
-        """Get a random index for a list"""
-        return random.randint(0, len(l) - 1)
-
-    def check_duplicate(self, sr, f, g):
-        """Check if a set of points was already tested"""
-        for pt in self.pytest_points:
-            duplicate_f = f == pt["frequency"]
-            duplicate_g = g == pt["setting_value"]
-            duplicate_sr = sr == pt["sample_rate"]
-            if duplicate_f and duplicate_g and duplicate_sr:
-                return True
-
-    def run_pytest_point(self, sr, f, g, reason, sr_m=False, f_m=False, g_m=False):
-        """Test the calculated value against the algorithm
-        Parameters:
-            sr, f, g -> Set values for the mock USRP
-            reason: Test case string for failure reference
-            sr_m, f_m, g_m -> Set values to use when calculating the expected value
-                              May differ in from actual set points in edge cases
-                              such as tuning in divisions or uncalibrated sample rate"""
-        # Check that the setup was completed
-        assert self.setup_complete, "Setup was not completed"
-
-        # If this point was tested before, skip it (triggering a new one)
-        if self.check_duplicate(sr, f, g):
-            return False
-
-        # If the point doesn't have modified inputs, use the algorithm ones
-        if not f_m:
-            f_m = f
-        if not g_m:
-            g_m = g
-        if not sr_m:
-            sr_m = sr
-
-        # Calculate what the scale factor should be
-        calc_gain_sigan = easy_gain(sr_m, f_m, g_m)
-
-        # Get the scale factor from the algorithm
-        interp_cal_data = self.sample_cal.get_calibration_dict([sr, f, g])
-        interp_gain_siggan = interp_cal_data["gain"]
-
-        # Save the point so we don't duplicate
-        self.pytest_points.append(
-            {
-                "sample_rate": int(sr),
-                "frequency": f,
-                "setting_value": g,
-                "gain": calc_gain_sigan,
-                "test": reason,
-            }
-        )
-
-        # Check if the point was calculated correctly
-        tolerance = 1e-5
-        msg = "Scale factor not correctly calculated!\r\n"
-        msg = f"{msg}    Expected value:   {calc_gain_sigan}\r\n"
-        msg = f"{msg}    Calculated value: {interp_gain_siggan}\r\n"
-        msg = f"{msg}    Tolerance: {tolerance}\r\n"
-        msg = f"{msg}    Test: {reason}\r\n"
-        msg = f"{msg}    Sample Rate: {sr / 1e6}({sr_m / 1e6})\r\n"
-        msg = f"{msg}    Frequency: {f / 1e6}({f_m / 1e6})\r\n"
-        msg = f"{msg}    Gain: {g}({g_m})\r\n"
-        msg = (
-            "{}    Formula: -1 * (Gain - Frequency[GHz] - Sample Rate[MHz])\r\n".format(
-                msg
-            )
-        )
-        if not isclose(calc_gain_sigan, interp_gain_siggan, abs_tol=tolerance):
-            interp_cal_data = self.sample_cal.get_calibration_dict([sr, f, g])
-
-        assert isclose(calc_gain_sigan, interp_gain_siggan, abs_tol=tolerance), msg
-        return True
-
-    @pytest.fixture(autouse=True)
-    def setup_calibration_file(self, tmpdir):
-        """Create the dummy calibration file in the pytest temp directory"""
-
-        # Only setup once
-        if self.setup_complete:
-            return
-
-        # Create and save the temp directory and file
-        self.tmpdir = tmpdir.strpath
-        self.calibration_file = "{}".format(tmpdir.join("dummy_cal_file.json"))
-
-        # Setup variables
-        self.dummy_noise_figure = 10
-        self.dummy_compression = -20
-        self.test_repeat_times = 3
-
-        # Sweep variables
-        self.sample_rates = [10e6, 15.36e6, 40e6]
-        self.gain_min = 40
-        self.gain_max = 60
-        self.gain_step = 10
-        gains = list(range(self.gain_min, self.gain_max, self.gain_step)) + [
-            self.gain_max
-        ]
-        self.frequency_min = 1000000000
-        self.frequency_max = 3400000000
-        self.frequency_step = 200000000
-        frequencies = list(
-            range(self.frequency_min, self.frequency_max, self.frequency_step)
-        ) + [self.frequency_max]
-        frequencies = sorted(frequencies)
-
-        # Start with blank cal data dicts
-        cal_data = {}
-
-        # Add the simple stuff to new cal format
-        cal_data["last_calibration_datetime"] = get_datetime_str_now()
-        cal_data["sensor_uid"] = "SAMPLE_CALIBRATION"
-
-        # Add SR/CF lookup table
-        cal_data["clock_rate_lookup_by_sample_rate"] = []
-        for sr in self.sample_rates:
-            cr = sr
-            while cr <= 40e6:
-                cr *= 2
-            cr /= 2
-            cal_data["clock_rate_lookup_by_sample_rate"].append(
-                {"sample_rate": int(sr), "clock_frequency": int(cr)}
-            )
-
-        # Create the JSON architecture for the calibration data
-        cal_data["calibration_data"] = {}
-        cal_data["calibration_parameters"] = ["sample_rate", "frequency", "gain"]
-        for k in range(len(self.sample_rates)):
-            cal_data_f = {}
-            for i in range(len(frequencies)):
-                cal_data_g = {}
-                for j in range(len(gains)):
-                    # Create the scale factor that ensures easy interpolation
-                    gain_sigan = easy_gain(
-                        self.sample_rates[k], frequencies[i], gains[j]
-                    )
-
-                    # Create the data point
-                    cal_data_point = {
-                        "gain": gain_sigan,
-                        "noise_figure": self.dummy_noise_figure,
-                        "1dB_compression_point": self.dummy_compression,
-                    }
-
-                    # Add the generated dicts to the parent lists
-                    cal_data_g[gains[j]] = deepcopy(cal_data_point)
-                cal_data_f[frequencies[i]] = deepcopy(cal_data_g)
-
-            cal_data["calibration_data"][self.sample_rates[k]] = deepcopy(cal_data_f)
-
-        # Write the new json file
-        with open(self.calibration_file, "w+") as file:
-            json.dump(cal_data, file, indent=4)
-
-        # Load the data back in
-        self.sample_cal = SensorCalibration.from_json(self.calibration_file, False)
-
-        # Create a list of previous points to ensure that we don't repeat
-        self.pytest_points = []
-
-        # Create sweep lists for test points
-        self.srs = self.sample_rates
-        self.gi_s = list(range(self.gain_min, self.gain_max, self.gain_step))
-        self.fi_s = list(
-            range(self.frequency_min, self.frequency_max, self.frequency_step)
-        )
-        self.g_s = self.gi_s + [self.gain_max]
-        self.f_s = self.fi_s + [self.frequency_max]
-
-        # Don't repeat test setup
-        self.setup_complete = True
-
-    def test_filter_by_parameter_out_of_range(self):
-        calibrations = {200.0: {"some_cal_data"}, 300.0: {"more cal data"}}
-        with pytest.raises(CalibrationException) as e_info:
-            cal = filter_by_parameter(calibrations, 400.0)
-            assert (
-                e_info.value.args[0]
-                == f"Could not locate calibration data at 400.0"
-                + f"\nAttempted lookup using key '400.0'"
-                + f"\nUsing calibration data: {calibrations}"
-            )
-
-    def test_filter_by_parameter_in_range_requires_match(self):
-        calibrations = {
-            200.0: {"Gain": "Gain at 200.0"},
-            300.0: {"Gain": "Gain at 300.0"},
-        }
-        with pytest.raises(CalibrationException) as e_info:
-            cal = filter_by_parameter(calibrations, 150.0)
-            assert e_info.value.args[0] == (
-                f"Could not locate calibration data at 150.0"
-                + f"\nAttempted lookup using key '150.0'"
-                + f"\nUsing calibration data: {calibrations}"
-            )
-
-    def test_get_calibration_dict_exact_match_lookup(self):
-        calibration_datetime = datetime.datetime.now()
-        calibration_params = ["sample_rate", "frequency"]
-        calibration_data = {
-            100.0: {200.0: {"NF": "NF at 100, 200", "Gain": "Gain at 100, 200"}},
-            200.0: {100.0: {"NF": "NF at 200, 100", "Gain": "Gain at 200, 100"}},
-        }
-        clock_rate_lookup_by_sample_rate = {}
-        cal = SensorCalibration(
-            calibration_datetime,
-            calibration_params,
-            calibration_data,
-            clock_rate_lookup_by_sample_rate,
-            False,
-            Path(""),
-        )
-        cal_data = cal.get_calibration_dict([100.0, 200.0])
-        assert cal_data["NF"] == "NF at 100, 200"
-
-    def test_get_calibration_dict_within_range(self):
-        calibration_datetime = datetime.datetime.now()
-        calibration_params = calibration_params = ["sample_rate", "frequency"]
-        calibration_data = {
-            100.0: {200: {"NF": "NF at 100, 200"}, 300.0: "Cal data at 100,300"},
-            200.0: {100.0: {"NF": "NF at 200, 100"}},
-        }
-        clock_rate_lookup_by_sample_rate = {}
-        test_cal_path = Path("test_calibration.json")
-        cal = SensorCalibration(
-            calibration_datetime,
-            calibration_params,
-            calibration_data,
-            clock_rate_lookup_by_sample_rate,
-            False,
-            test_cal_path,
-        )
-        with pytest.raises(CalibrationException) as e_info:
-            cal_data = cal.get_calibration_dict([100.0, 250.0])
-            assert e_info.value.args[0] == (
-                f"Could not locate calibration data at 250.0"
-                + f"\nAttempted lookup using key '250.0'"
-                + f"\nUsing calibration data: {cal.calibration_data}"
-            )
-
-    def test_sf_bound_points(self):
-        """Test SF determination at boundary points"""
-        self.run_pytest_point(
-            self.srs[0], self.frequency_min, self.gain_min, "Testing boundary points"
-        )
-        self.run_pytest_point(
-            self.srs[0], self.frequency_max, self.gain_max, "Testing boundary points"
-        )
-
-    def test_sf_no_interpolation_points(self):
-        """Test points without interpolation"""
-        for i in range(4 * self.test_repeat_times):
-            while True:
-                g = self.g_s[self.rand_index(self.g_s)]
-                f = self.f_s[self.rand_index(self.f_s)]
-                if self.run_pytest_point(
-                    self.srs[0], f, g, "Testing no interpolation points"
-                ):
-                    break
-
-    def test_update(self):
-        calibration_datetime = get_datetime_str_now()
-        calibration_params = ["sample_rate", "frequency"]
-        calibration_data = {100.0: {200.0: {"noise_figure": 0, "gain": 0}}}
-        clock_rate_lookup_by_sample_rate = {}
-        test_cal_path = Path("test_calibration.json")
-        cal = SensorCalibration(
-            calibration_datetime,
-            calibration_params,
-            calibration_data,
-            clock_rate_lookup_by_sample_rate,
-            False,
-            test_cal_path,
-        )
-        action_params = {"sample_rate": 100.0, "frequency": 200.0}
-        update_time = get_datetime_str_now()
-        cal.update(action_params, update_time, 30.0, 5.0, 21)
-        cal_from_file = SensorCalibration.from_json(test_cal_path, False)
-        test_cal_path.unlink()
-        file_utc_time = parse_datetime_iso_format_str(cal.last_calibration_datetime)
-        cal_time_utc = parse_datetime_iso_format_str(update_time)
-        assert file_utc_time.year == cal_time_utc.year
-        assert file_utc_time.month == cal_time_utc.month
-        assert file_utc_time.day == cal_time_utc.day
-        assert file_utc_time.hour == cal_time_utc.hour
-        assert file_utc_time.minute == cal_time_utc.minute
-        assert cal.calibration_data["100.0"]["200.0"]["gain"] == 30.0
-        assert cal.calibration_data["100.0"]["200.0"]["noise_figure"] == 5.0
-        assert cal_from_file.calibration_data["100.0"]["200.0"]["gain"] == 30.0
-        assert cal_from_file.calibration_data["100.0"]["200.0"]["noise_figure"] == 5.0
-
-    def test_filter_by_paramter_integer(self):
-        calibrations = {"200.0": {"some_cal_data"}, 300.0: {"more cal data"}}
-        filtered_data = filter_by_parameter(calibrations, 200)
-        assert filtered_data is calibrations["200.0"]
+"""Test the Calibration base class."""
diff --git a/scos_actions/calibration/tests/test_sensor_calibration.py b/scos_actions/calibration/tests/test_sensor_calibration.py
new file mode 100644
index 00000000..cf4403fa
--- /dev/null
+++ b/scos_actions/calibration/tests/test_sensor_calibration.py
@@ -0,0 +1,315 @@
+"""Test the SensorCalibration class."""
+
+import json
+import random
+from copy import deepcopy
+from math import isclose
+from pathlib import Path
+
+import pytest
+from scos_actions.calibration.sensor_calibration import SensorCalibration
+from scos_actions.calibration.utils import CalibrationException, filter_by_parameter
+from scos_actions.tests.resources.utils import easy_gain
+from scos_actions.utils import get_datetime_str_now, parse_datetime_iso_format_str
+
+
+class TestSensorCalibrationFile:
+    # Ensure we load the test file
+    setup_complete = False
+
+    def rand_index(self, l):
+        """Get a random index for a list"""
+        return random.randint(0, len(l) - 1)
+
+    def check_duplicate(self, sr, f, g):
+        """Check if a set of points was already tested"""
+        for pt in self.pytest_points:
+            duplicate_f = f == pt["frequency"]
+            duplicate_g = g == pt["setting_value"]
+            duplicate_sr = sr == pt["sample_rate"]
+            if duplicate_f and duplicate_g and duplicate_sr:
+                return True
+
+    def run_pytest_point(self, sr, f, g, reason, sr_m=False, f_m=False, g_m=False):
+        """Test the calculated value against the algorithm
+        Parameters:
+            sr, f, g -> Set values for the mock USRP
+            reason: Test case string for failure reference
+            sr_m, f_m, g_m -> Set values to use when calculating the expected value
+                              May differ in from actual set points in edge cases
+                              such as tuning in divisions or uncalibrated sample rate"""
+        # Check that the setup was completed
+        assert self.setup_complete, "Setup was not completed"
+
+        # If this point was tested before, skip it (triggering a new one)
+        if self.check_duplicate(sr, f, g):
+            return False
+
+        # If the point doesn't have modified inputs, use the algorithm ones
+        if not f_m:
+            f_m = f
+        if not g_m:
+            g_m = g
+        if not sr_m:
+            sr_m = sr
+
+        # Calculate what the scale factor should be
+        calc_gain_sigan = easy_gain(sr_m, f_m, g_m)
+
+        # Get the scale factor from the algorithm
+        interp_cal_data = self.sample_cal.get_calibration_dict([sr, f, g])
+        interp_gain_siggan = interp_cal_data["gain"]
+
+        # Save the point so we don't duplicate
+        self.pytest_points.append(
+            {
+                "sample_rate": int(sr),
+                "frequency": f,
+                "setting_value": g,
+                "gain": calc_gain_sigan,
+                "test": reason,
+            }
+        )
+
+        # Check if the point was calculated correctly
+        tolerance = 1e-5
+        msg = "Scale factor not correctly calculated!\r\n"
+        msg = f"{msg}    Expected value:   {calc_gain_sigan}\r\n"
+        msg = f"{msg}    Calculated value: {interp_gain_siggan}\r\n"
+        msg = f"{msg}    Tolerance: {tolerance}\r\n"
+        msg = f"{msg}    Test: {reason}\r\n"
+        msg = f"{msg}    Sample Rate: {sr / 1e6}({sr_m / 1e6})\r\n"
+        msg = f"{msg}    Frequency: {f / 1e6}({f_m / 1e6})\r\n"
+        msg = f"{msg}    Gain: {g}({g_m})\r\n"
+        msg = (
+            "{}    Formula: -1 * (Gain - Frequency[GHz] - Sample Rate[MHz])\r\n".format(
+                msg
+            )
+        )
+        if not isclose(calc_gain_sigan, interp_gain_siggan, abs_tol=tolerance):
+            interp_cal_data = self.sample_cal.get_calibration_dict([sr, f, g])
+
+        assert isclose(calc_gain_sigan, interp_gain_siggan, abs_tol=tolerance), msg
+        return True
+
+    @pytest.fixture(autouse=True)
+    def setup_calibration_file(self, tmpdir):
+        """Create the dummy calibration file in the pytest temp directory"""
+
+        # Only setup once
+        if self.setup_complete:
+            return
+
+        # Create and save the temp directory and file
+        self.tmpdir = tmpdir.strpath
+        self.calibration_file = "{}".format(tmpdir.join("dummy_cal_file.json"))
+
+        # Setup variables
+        self.dummy_noise_figure = 10
+        self.dummy_compression = -20
+        self.test_repeat_times = 3
+
+        # Sweep variables
+        self.sample_rates = [10e6, 15.36e6, 40e6]
+        self.gain_min = 40
+        self.gain_max = 60
+        self.gain_step = 10
+        gains = list(range(self.gain_min, self.gain_max, self.gain_step)) + [
+            self.gain_max
+        ]
+        self.frequency_min = 1000000000
+        self.frequency_max = 3400000000
+        self.frequency_step = 200000000
+        frequencies = list(
+            range(self.frequency_min, self.frequency_max, self.frequency_step)
+        ) + [self.frequency_max]
+        frequencies = sorted(frequencies)
+
+        # Start with blank cal data dicts
+        cal_data = {}
+
+        # Add the simple stuff to new cal format
+        cal_data["last_calibration_datetime"] = get_datetime_str_now()
+        cal_data["sensor_uid"] = "SAMPLE_CALIBRATION"
+
+        # Add SR/CF lookup table
+        cal_data["clock_rate_lookup_by_sample_rate"] = []
+        for sr in self.sample_rates:
+            cr = sr
+            while cr <= 40e6:
+                cr *= 2
+            cr /= 2
+            cal_data["clock_rate_lookup_by_sample_rate"].append(
+                {"sample_rate": int(sr), "clock_frequency": int(cr)}
+            )
+
+        # Create the JSON architecture for the calibration data
+        cal_data["calibration_data"] = {}
+        cal_data["calibration_parameters"] = ["sample_rate", "frequency", "gain"]
+        for k in range(len(self.sample_rates)):
+            cal_data_f = {}
+            for i in range(len(frequencies)):
+                cal_data_g = {}
+                for j in range(len(gains)):
+                    # Create the scale factor that ensures easy interpolation
+                    gain_sigan = easy_gain(
+                        self.sample_rates[k], frequencies[i], gains[j]
+                    )
+
+                    # Create the data point
+                    cal_data_point = {
+                        "gain": gain_sigan,
+                        "noise_figure": self.dummy_noise_figure,
+                        "1dB_compression_point": self.dummy_compression,
+                    }
+
+                    # Add the generated dicts to the parent lists
+                    cal_data_g[gains[j]] = deepcopy(cal_data_point)
+                cal_data_f[frequencies[i]] = deepcopy(cal_data_g)
+
+            cal_data["calibration_data"][self.sample_rates[k]] = deepcopy(cal_data_f)
+
+        # Write the new json file
+        with open(self.calibration_file, "w+") as file:
+            json.dump(cal_data, file, indent=4)
+
+        # Load the data back in
+        self.sample_cal = SensorCalibration.from_json(self.calibration_file, False)
+
+        # Create a list of previous points to ensure that we don't repeat
+        self.pytest_points = []
+
+        # Create sweep lists for test points
+        self.srs = self.sample_rates
+        self.gi_s = list(range(self.gain_min, self.gain_max, self.gain_step))
+        self.fi_s = list(
+            range(self.frequency_min, self.frequency_max, self.frequency_step)
+        )
+        self.g_s = self.gi_s + [self.gain_max]
+        self.f_s = self.fi_s + [self.frequency_max]
+
+        # Don't repeat test setup
+        self.setup_complete = True
+
+    def test_filter_by_parameter_out_of_range(self):
+        calibrations = {200.0: {"some_cal_data"}, 300.0: {"more cal data"}}
+        with pytest.raises(CalibrationException) as e_info:
+            cal = filter_by_parameter(calibrations, 400.0)
+            assert (
+                e_info.value.args[0]
+                == f"Could not locate calibration data at 400.0"
+                + f"\nAttempted lookup using key '400.0'"
+                + f"\nUsing calibration data: {calibrations}"
+            )
+
+    def test_filter_by_parameter_in_range_requires_match(self):
+        calibrations = {
+            200.0: {"Gain": "Gain at 200.0"},
+            300.0: {"Gain": "Gain at 300.0"},
+        }
+        with pytest.raises(CalibrationException) as e_info:
+            cal = filter_by_parameter(calibrations, 150.0)
+            assert e_info.value.args[0] == (
+                f"Could not locate calibration data at 150.0"
+                + f"\nAttempted lookup using key '150.0'"
+                + f"\nUsing calibration data: {calibrations}"
+            )
+
+    def test_get_calibration_dict_exact_match_lookup(self):
+        calibration_datetime = get_datetime_str_now()
+        calibration_params = ["sample_rate", "frequency"]
+        calibration_data = {
+            100.0: {200.0: {"NF": "NF at 100, 200", "Gain": "Gain at 100, 200"}},
+            200.0: {100.0: {"NF": "NF at 200, 100", "Gain": "Gain at 200, 100"}},
+        }
+        cal = SensorCalibration(
+            calibration_parameters=calibration_params,
+            calibration_data=calibration_data,
+            is_default=False,
+            file_path=Path(""),
+            last_calibration_datetime=calibration_datetime,
+            clock_rate_lookup_by_sample_rate={},
+            sensor_uid="TESTING",
+        )
+        cal_data = cal.get_calibration_dict([100.0, 200.0])
+        assert cal_data["NF"] == "NF at 100, 200"
+
+    def test_get_calibration_dict_within_range(self):
+        calibration_datetime = get_datetime_str_now()
+        calibration_params = calibration_params = ["sample_rate", "frequency"]
+        calibration_data = {
+            100.0: {200: {"NF": "NF at 100, 200"}, 300.0: "Cal data at 100,300"},
+            200.0: {100.0: {"NF": "NF at 200, 100"}},
+        }
+        cal = SensorCalibration(
+            calibration_parameters=calibration_params,
+            calibration_data=calibration_data,
+            is_default=False,
+            file_path=Path("test_calibration.json"),
+            last_calibration_datetime=calibration_datetime,
+            clock_rate_lookup_by_sample_rate={},
+            sensor_uid="TESTING",
+        )
+        with pytest.raises(CalibrationException) as e_info:
+            cal_data = cal.get_calibration_dict([100.0, 250.0])
+            assert e_info.value.args[0] == (
+                f"Could not locate calibration data at 250.0"
+                + f"\nAttempted lookup using key '250.0'"
+                + f"\nUsing calibration data: {cal.calibration_data}"
+            )
+
+    def test_sf_bound_points(self):
+        """Test SF determination at boundary points"""
+        self.run_pytest_point(
+            self.srs[0], self.frequency_min, self.gain_min, "Testing boundary points"
+        )
+        self.run_pytest_point(
+            self.srs[0], self.frequency_max, self.gain_max, "Testing boundary points"
+        )
+
+    def test_sf_no_interpolation_points(self):
+        """Test points without interpolation"""
+        for i in range(4 * self.test_repeat_times):
+            while True:
+                g = self.g_s[self.rand_index(self.g_s)]
+                f = self.f_s[self.rand_index(self.f_s)]
+                if self.run_pytest_point(
+                    self.srs[0], f, g, "Testing no interpolation points"
+                ):
+                    break
+
+    def test_update(self):
+        calibration_datetime = get_datetime_str_now()
+        calibration_params = ["sample_rate", "frequency"]
+        calibration_data = {100.0: {200.0: {"noise_figure": 0, "gain": 0}}}
+        test_cal_path = Path("test_calibration.json")
+        cal = SensorCalibration(
+            calibration_parameters=calibration_params,
+            calibration_data=calibration_data,
+            is_default=False,
+            file_path=test_cal_path,
+            last_calibration_datetime=calibration_datetime,
+            clock_rate_lookup_by_sample_rate={},
+            sensor_uid="TESTING",
+        )
+        action_params = {"sample_rate": 100.0, "frequency": 200.0}
+        update_time = get_datetime_str_now()
+        cal.update(action_params, update_time, 30.0, 5.0, 21)
+        cal_from_file = SensorCalibration.from_json(test_cal_path, False)
+        test_cal_path.unlink()
+        file_utc_time = parse_datetime_iso_format_str(cal.last_calibration_datetime)
+        cal_time_utc = parse_datetime_iso_format_str(update_time)
+        assert file_utc_time.year == cal_time_utc.year
+        assert file_utc_time.month == cal_time_utc.month
+        assert file_utc_time.day == cal_time_utc.day
+        assert file_utc_time.hour == cal_time_utc.hour
+        assert file_utc_time.minute == cal_time_utc.minute
+        assert cal.calibration_data["100.0"]["200.0"]["gain"] == 30.0
+        assert cal.calibration_data["100.0"]["200.0"]["noise_figure"] == 5.0
+        assert cal_from_file.calibration_data["100.0"]["200.0"]["gain"] == 30.0
+        assert cal_from_file.calibration_data["100.0"]["200.0"]["noise_figure"] == 5.0
+
+    def test_filter_by_paramter_integer(self):
+        calibrations = {"200.0": {"some_cal_data"}, 300.0: {"more cal data"}}
+        filtered_data = filter_by_parameter(calibrations, 200)
+        assert filtered_data is calibrations["200.0"]

From e4234a9840b5b2534c2dc91574ed5f7a85747bf8 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 22 Jan 2024 12:49:29 -0500
Subject: [PATCH 004/102] remove sigan calibrations

---
 .../acquire_stepped_freq_tdomain_iq.py        |  7 ------
 .../actions/interfaces/measurement_action.py  |  5 ----
 scos_actions/hardware/mocks/mock_sigan.py     | 24 ++-----------------
 scos_actions/hardware/sigan_iface.py          | 22 +----------------
 scos_actions/metadata/structs/capture.py      |  3 ++-
 5 files changed, 5 insertions(+), 56 deletions(-)

diff --git a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
index 495be7e2..bd114e57 100644
--- a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
+++ b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
@@ -117,14 +117,7 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
                 overload=measurement_result["overload"],
                 sigan_settings=sigan_settings,
             )
-            sigan_cal = self.sensor.signal_analyzer.sigan_calibration_data
             sensor_cal = self.sensor.signal_analyzer.sensor_calibration_data
-            if sigan_cal is not None:
-                if "1db_compression_point" in sigan_cal:
-                    sigan_cal["compression_point"] = sigan_cal.pop(
-                        "1db_compression_point"
-                    )
-                capture_segment.sigan_calibration = ntia_sensor.Calibration(**sigan_cal)
             if sensor_cal is not None:
                 if "1db_compression_point" in sensor_cal:
                     sensor_cal["compression_point"] = sensor_cal.pop(
diff --git a/scos_actions/actions/interfaces/measurement_action.py b/scos_actions/actions/interfaces/measurement_action.py
index f0b06e54..d3299ca4 100644
--- a/scos_actions/actions/interfaces/measurement_action.py
+++ b/scos_actions/actions/interfaces/measurement_action.py
@@ -51,7 +51,6 @@ def create_capture_segment(
             overload=overload,
             sigan_settings=sigan_settings,
         )
-        sigan_cal = self.sensor.signal_analyzer.sigan_calibration_data
         sensor_cal = self.sensor.signal_analyzer.sensor_calibration_data
         # Rename compression point keys if they exist
         # then set calibration metadata if it exists
@@ -61,10 +60,6 @@ def create_capture_segment(
                     "1db_compression_point"
                 )
             capture_segment.sensor_calibration = ntia_sensor.Calibration(**sensor_cal)
-        if sigan_cal is not None:
-            if "1db_compression_point" in sigan_cal:
-                sigan_cal["compression_point"] = sigan_cal.pop("1db_compression_point")
-            capture_segment.sigan_calibration = ntia_sensor.Calibration(**sigan_cal)
         return capture_segment
 
     def create_metadata(
diff --git a/scos_actions/hardware/mocks/mock_sigan.py b/scos_actions/hardware/mocks/mock_sigan.py
index 71cc71ba..35078918 100644
--- a/scos_actions/hardware/mocks/mock_sigan.py
+++ b/scos_actions/hardware/mocks/mock_sigan.py
@@ -29,20 +29,10 @@ class MockSignalAnalyzer(SignalAnalyzerInterface):
     def __init__(
         self,
         sensor_cal: Optional[SensorCalibration] = None,
-        sigan_cal: Optional[SensorCalibration] = None,
         randomize_values: bool = False,
     ):
-        super().__init__(sensor_cal, sigan_cal)
-        # Define the default calibration dicts
-        self.DEFAULT_SIGAN_CALIBRATION = {
-            "datetime": get_datetime_str_now(),
-            "gain": 0,  # Defaults to gain setting
-            "enbw": None,  # Defaults to sample rate
-            "noise_figure": 0,
-            "1db_compression_point": 100,
-            "temperature": 26.85,
-        }
-
+        super().__init__(sensor_cal)
+        # Define the default calibration dict
         self.DEFAULT_SENSOR_CALIBRATION = {
             "datetime": get_datetime_str_now(),
             "gain": 0,  # Defaults to sigan gain
@@ -73,7 +63,6 @@ def __init__(
 
         self.randomize_values = randomize_values
         self.sensor_calibration_data = self.DEFAULT_SENSOR_CALIBRATION
-        self.sigan_calibration_data = self.DEFAULT_SIGAN_CALIBRATION
 
     @property
     def is_available(self):
@@ -215,12 +204,3 @@ def recompute_sensor_calibration_data(self, cal_args: list) -> None:
             )
         else:
             logger.warning("Sensor calibration does not exist.")
-
-    def recompute_sigan_calibration_data(self, cal_args: list) -> None:
-        """Set the sigan calibration data based on the current tuning"""
-        if self.sigan_calibration is not None:
-            self.sigan_calibration_data.update(
-                self.sigan_calibration.get_calibration_dict(cal_args)
-            )
-        else:
-            logger.warning("Sigan calibration does not exist.")
diff --git a/scos_actions/hardware/sigan_iface.py b/scos_actions/hardware/sigan_iface.py
index f0eccf2c..68ed0463 100644
--- a/scos_actions/hardware/sigan_iface.py
+++ b/scos_actions/hardware/sigan_iface.py
@@ -26,13 +26,10 @@ class SignalAnalyzerInterface(ABC):
     def __init__(
         self,
         sensor_cal: Optional[SensorCalibration] = None,
-        sigan_cal: Optional[SensorCalibration] = None,
         switches: Optional[Dict[str, WebRelay]] = None,
     ):
         self.sensor_calibration_data = {}
-        self.sigan_calibration_data = {}
         self._sensor_calibration = sensor_cal
-        self._sigan_calibration = sigan_cal
         self._model = "Unknown"
         self.switches = switches
 
@@ -136,6 +133,7 @@ def power_cycle_and_connect(self, sleep_time: float = 2.0) -> None:
         return
 
     def recompute_sensor_calibration_data(self, cal_args: list) -> None:
+        """Set the sensor calibration data based on the current tuning."""
         self.sensor_calibration_data = {}
         if self.sensor_calibration is not None:
             self.sensor_calibration_data.update(
@@ -144,16 +142,6 @@ def recompute_sensor_calibration_data(self, cal_args: list) -> None:
         else:
             logger.warning("Sensor calibration does not exist.")
 
-    def recompute_sigan_calibration_data(self, cal_args: list) -> None:
-        self.sigan_calibration_data = {}
-        """Set the sigan calibration data based on the current tuning"""
-        if self.sigan_calibration is not None:
-            self.sigan_calibration_data.update(
-                self.sigan_calibration.get_calibration_dict(cal_args)
-            )
-        else:
-            logger.warning("Sigan calibration does not exist.")
-
     def get_status(self) -> dict:
         return {"model": self._model, "healthy": self.healthy()}
 
@@ -172,11 +160,3 @@ def sensor_calibration(self) -> SensorCalibration:
     @sensor_calibration.setter
     def sensor_calibration(self, cal: SensorCalibration):
         self._sensor_calibration = cal
-
-    @property
-    def sigan_calibration(self) -> SensorCalibration:
-        return self._sigan_calibration
-
-    @sigan_calibration.setter
-    def sigan_calibration(self, cal: SensorCalibration):
-        self._sigan_calibration = cal
diff --git a/scos_actions/metadata/structs/capture.py b/scos_actions/metadata/structs/capture.py
index 16f12280..59fe6128 100644
--- a/scos_actions/metadata/structs/capture.py
+++ b/scos_actions/metadata/structs/capture.py
@@ -1,7 +1,6 @@
 from typing import Optional
 
 import msgspec
-
 from scos_actions.metadata.structs.ntia_sensor import Calibration, SiganSettings
 from scos_actions.metadata.utils import SIGMF_OBJECT_KWARGS
 
@@ -14,6 +13,8 @@
     "duration": "ntia-sensor:duration",
     "overload": "ntia-sensor:overload",
     "sensor_calibration": "ntia-sensor:sensor_calibration",
+    # sigan_calibration is unused by SCOS Sensor but still defined
+    # in the ntia-sensor extension as of v2.0.0
     "sigan_calibration": "ntia-sensor:sigan_calibration",
     "sigan_settings": "ntia-sensor:sigan_settings",
 }

From 3be9f69bee592cd5f3a75345ce4240eb6939cb2f Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 22 Jan 2024 12:51:35 -0500
Subject: [PATCH 005/102] update docstring for 'update' method

---
 scos_actions/calibration/sensor_calibration.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index 11865206..d341bcab 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -33,7 +33,7 @@ def update(
         """
         Update the calibration data by overwriting or adding an entry.
 
-        This method updates the instance variables of the ``Calibration``
+        This updates the instance variables of the ``SensorCalibration``
         object and additionally writes these changes to the specified
         output file.
 
@@ -46,7 +46,6 @@ def update(
         :param noise_figure_dB: Noise figure value for calibration, in dB.
         :param temp_degC: Temperature at calibration time, in degrees Celsius.
         :param file_path: File path for saving the updated calibration data.
-        :raises Exception:
         """
         # Get existing calibration data entry which will be updated
         data_entry = self._retrieve_data_to_update(params)

From 50b596f99252b0e8fd5d11887691e8c6bcffa0f8 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 22 Jan 2024 13:04:29 -0500
Subject: [PATCH 006/102] add differential calibration dataclass

---
 .../calibration/differential_calibration.py   | 38 +++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100644 scos_actions/calibration/differential_calibration.py

diff --git a/scos_actions/calibration/differential_calibration.py b/scos_actions/calibration/differential_calibration.py
new file mode 100644
index 00000000..cc013c6a
--- /dev/null
+++ b/scos_actions/calibration/differential_calibration.py
@@ -0,0 +1,38 @@
+"""
+Dataclass implementation for "differential calibration" handling.
+
+A differential calibration provides loss values at different frequencies
+which represent excess loss between the calibration terminal and the antenna
+port. At present, this is measured manually using a calibration probe consisting
+of a calibrated noise source and a programmable attenuator.
+
+The ``reference_point`` top-level key defines the point to which measurements
+are referenced after using the correction factors included in the file.
+
+The ``calibration_data`` entries are expected to include these correction factors,
+with the key name ``"differential_loss"`` and values in decibels (dB). These correction
+factors represent the differential loss between the calibration terminal used by onboard
+``SensorCalibration`` results and the reference point defined by ``reference_point``.
+"""
+
+from dataclasses import dataclass
+
+from scos_actions.calibration.interfaces.calibration import Calibration
+
+
+@dataclass
+class DifferentialCalibration(Calibration):
+    reference_point: str
+
+    def update(self):
+        """
+        SCOS Sensor should not update differential calibration files.
+
+        Instead, these should be generated through an external calibration
+        process. This class should only be used to read JSON files, and never
+        to update their entries. Therefore, no ``update`` method is implemented.
+
+        If, at some point in the future, functionality is added to automate these
+        calibrations, this function may be implemented.
+        """
+        raise NotImplementedError

From b8ad5d022cdee1f044901170e7484cc6590e78b8 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 22 Jan 2024 13:58:37 -0500
Subject: [PATCH 007/102] add field input validator to base class

---
 .../calibration/interfaces/calibration.py         | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/scos_actions/calibration/interfaces/calibration.py b/scos_actions/calibration/interfaces/calibration.py
index debe42ba..a17bc899 100644
--- a/scos_actions/calibration/interfaces/calibration.py
+++ b/scos_actions/calibration/interfaces/calibration.py
@@ -3,7 +3,7 @@
 import logging
 from abc import abstractmethod
 from pathlib import Path
-from typing import Any, List
+from typing import Any, List, get_origin
 
 from scos_actions.calibration.utils import filter_by_parameter
 
@@ -18,11 +18,24 @@ class Calibration:
     file_path: Path
 
     def __post_init__(self):
+        self._validate_fields()
         # Convert key names in data to strings
         # This means that formatting will always match between
         # native types provided in Python and data loaded from JSON
         self.calibration_data = json.loads(json.dumps(self.calibration_data))
 
+    def _validate_fields(self) -> None:
+        """Loosely check that the input types are as expected."""
+        for f_name, f_def in self.__dataclass_fields__.items():
+            f_type = get_origin(f_def.type) or f_def.type
+            actual_value = getattr(self, f_name)
+            if not isinstance(actual_value, f_type):
+                c_name = self.__class__.__name__
+                actual_type = type(actual_value)
+                raise TypeError(
+                    f"{c_name} field {f_name} must be {f_type}, not {actual_type}"
+                )
+
     def get_calibration_dict(self, cal_params: List[Any]) -> dict:
         """
         Get calibration data closest to the specified parameter values.

From 971409b33fbaf35c86019d9bc35095a871d2b372 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 22 Jan 2024 15:21:26 -0500
Subject: [PATCH 008/102] add to_json method

---
 .../calibration/interfaces/calibration.py     | 22 +++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/scos_actions/calibration/interfaces/calibration.py b/scos_actions/calibration/interfaces/calibration.py
index a17bc899..2e339966 100644
--- a/scos_actions/calibration/interfaces/calibration.py
+++ b/scos_actions/calibration/interfaces/calibration.py
@@ -64,6 +64,8 @@ def _retrieve_data_to_update(self, params: dict) -> dict:
             Example: ``{"sample_rate": 14000000.0, "attenuation": 10.0}``
         :return: A dict containing the existing calibration entry at
             the specified parameter set, which may be empty if none exists.
+        :raises ValueError: If not all calibration parameters exist as keys
+            in ``params``.
         """
         # Use params keys as calibration_parameters if none exist
         if len(self.calibration_parameters) == 0:
@@ -73,7 +75,7 @@ def _retrieve_data_to_update(self, params: dict) -> dict:
             self.calibration_parameters = list(params.keys())
         elif not set(params.keys()) >= set(self.calibration_parameters):
             # Otherwise ensure all required parameters were used
-            raise Exception(
+            raise ValueError(
                 "Not enough parameters specified to update calibration.\n"
                 + f"Required parameters are {self.calibration_parameters}"
             )
@@ -95,7 +97,7 @@ def _retrieve_data_to_update(self, params: dict) -> dict:
         return data_entry
 
     @abstractmethod
-    def update():
+    def update(self):
         """Update the calibration data"""
         raise NotImplementedError
 
@@ -141,3 +143,19 @@ def from_json(cls, fname: Path, is_default: bool):
 
         # Create and return the Calibration object
         return cls(is_default=is_default, file_path=fname, **calibration)
+
+    def to_json(self) -> None:
+        """
+        Save the calibration to a JSON file.
+
+        The JSON file will be located at ``self.file_path`` and will
+        contain a copy of ``self.__dict__``, except for the ``file_path``
+        and ``is_default`` key/value pairs. This includes all dataclass
+        fields, with their parameter names as JSON key names.
+        """
+        dict_to_json = self.__dict__.copy()
+        # Remove keys which should not save to JSON
+        dict_to_json.pop("file_path", None)
+        dict_to_json.pop("is_default", None)
+        with open(self.file_path, "w") as outfile:
+            outfile.write(json.dumps(dict_to_json))

From 96488c65a9f8e650a83633272871af1aa63c5844 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 22 Jan 2024 15:21:55 -0500
Subject: [PATCH 009/102] use to_json in update

---
 scos_actions/calibration/sensor_calibration.py | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index d341bcab..57604c62 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -64,12 +64,4 @@ def update(
         )
 
         # Write updated calibration data to file
-        cal_dict = {
-            "last_calibration_datetime": self.last_calibration_datetime,
-            "sensor_uid": self.sensor_uid,
-            "calibration_parameters": self.calibration_parameters,
-            "clock_rate_lookup_by_sample_rate": self.clock_rate_lookup_by_sample_rate,
-            "calibration_data": self.calibration_data,
-        }
-        with open(self.file_path, "w") as outfile:
-            outfile.write(json.dumps(cal_dict))
+        self.to_json()

From c1a5f7d6cc1ea4c975611451419b0a5d46d829a8 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 22 Jan 2024 15:22:38 -0500
Subject: [PATCH 010/102] update calibration-related unit tests

---
 .../calibration/tests/test_calibration.py     | 152 +++++++++++++++++-
 .../tests/test_differential_calibration.py    |  42 +++++
 .../tests/test_sensor_calibration.py          |  91 ++++++-----
 scos_actions/calibration/tests/test_utils.py  |  41 +++++
 scos_actions/calibration/tests/utils.py       |  10 ++
 5 files changed, 295 insertions(+), 41 deletions(-)
 create mode 100644 scos_actions/calibration/tests/test_differential_calibration.py
 create mode 100644 scos_actions/calibration/tests/test_utils.py
 create mode 100644 scos_actions/calibration/tests/utils.py

diff --git a/scos_actions/calibration/tests/test_calibration.py b/scos_actions/calibration/tests/test_calibration.py
index 3d743de1..a335d86f 100644
--- a/scos_actions/calibration/tests/test_calibration.py
+++ b/scos_actions/calibration/tests/test_calibration.py
@@ -1 +1,151 @@
-"""Test the Calibration base class."""
+"""Test the Calibration base dataclass."""
+import dataclasses
+import json
+from pathlib import Path
+from typing import List
+
+import pytest
+from scos_actions.calibration.interfaces.calibration import Calibration
+from scos_actions.calibration.sensor_calibration import SensorCalibration
+from scos_actions.calibration.tests.utils import recursive_check_keys
+
+
+class TestBaseCalibration:
+    @pytest.fixture(autouse=True)
+    def setup_calibration_file(self, tmp_path: Path):
+        """Create a dummy calibration file in the pytest temp directory."""
+        # Create some dummy calibration data
+        self.cal_params = ["frequency", "gain"]
+        self.frequencies = [3555e9, 3565e9, 3575e9]
+        self.gains = [10.0, 20.0, 30.0]
+        cal_data = {}
+        for frequency in self.frequencies:
+            cal_data[frequency] = {}
+            for gain in self.gains:
+                cal_data[frequency][gain] = {
+                    "gain": gain * 1.1,
+                    "noise_figure": gain / 5.0,
+                    "1dB_compression_point": -50 + gain,
+                }
+        self.cal_data = cal_data
+        self.dummy_file_path = tmp_path / "dummy_cal.json"
+        self.dummy_default_file_path = tmp_path / "dummy_default_cal.json"
+
+        self.sample_cal = Calibration(
+            calibration_parameters=self.cal_params,
+            calibration_data=self.cal_data,
+            is_default=False,
+            file_path=self.dummy_file_path,
+        )
+
+        self.sample_default_cal = Calibration(
+            calibration_parameters=self.cal_params,
+            calibration_data=self.cal_data,
+            is_default=True,
+            file_path=self.dummy_default_file_path,
+        )
+
+    def test_calibration_data_key_name_conversion(self):
+        """On post-init, all calibration_data key names should be converted to strings."""
+        recursive_check_keys(self.sample_cal.calibration_data)
+        recursive_check_keys(self.sample_default_cal.calibration_data)
+
+    def test_calibration_dataclass_fields(self):
+        """Check that the dataclass is set up as expected."""
+        fields = {f.name: f.type for f in dataclasses.fields(Calibration)}
+        # Note: does not check field order
+        assert fields == {
+            "calibration_parameters": List[str],
+            "is_default": bool,
+            "calibration_data": dict,
+            "file_path": Path,
+        }, "Calibration class fields have changed"
+
+    def test_field_validator(self):
+        """Check that the input field type validator works."""
+        with pytest.raises(TypeError):
+            _ = Calibration([], {}, False, False)
+        with pytest.raises(TypeError):
+            _ = Calibration([], {}, 100, Path(""))
+        with pytest.raises(TypeError):
+            _ = Calibration([], [10, 20], False, Path(""))
+        with pytest.raises(TypeError):
+            _ = Calibration({"test": 1}, {}, False, Path(""))
+
+    def test_get_calibration_dict(self):
+        """Check the get_calibration_dict method with all dummy data."""
+        for f in self.frequencies:
+            assert json.loads(
+                json.dumps(self.cal_data[f])
+            ) == self.sample_cal.get_calibration_dict([f])
+            for g in self.gains:
+                assert json.loads(
+                    json.dumps(self.cal_data[f][g])
+                ) == self.sample_cal.get_calibration_dict([f, g])
+
+    def test_retrieve_data_to_update(self):
+        """Check that the calibration data entry is correctly returned."""
+        for f in self.frequencies:
+            for g in self.gains:
+                params = {"frequency": f, "gain": g}
+                # Use the "is" keyword since this must not be a copy/identical dict
+                assert self.sample_cal.calibration_data[str(f)][
+                    str(g)
+                ] is self.sample_cal._retrieve_data_to_update(params)
+        # Method should work with len=0 calibration parameters
+        test_cal = Calibration([], {}, False, Path(""))
+        _ = test_cal._retrieve_data_to_update({"frequency": 3555e9, "gain": 10.0})
+        # And should fail if calibration parameters are not all supplied
+        with pytest.raises(ValueError):
+            _ = self.sample_cal._retrieve_data_to_update(
+                {"frequency": self.frequencies[0]}
+            )
+
+    def test_to_and_from_json(self, tmp_path: Path):
+        """Test the ``from_json`` factory method."""
+        # First save the calibration data to temporary files
+        self.sample_cal.to_json()
+        self.sample_default_cal.to_json()
+        # Then load and compare
+        assert self.sample_cal == Calibration.from_json(self.dummy_file_path, False)
+        assert self.sample_default_cal == Calibration.from_json(
+            self.dummy_default_file_path, True
+        )
+
+        # These should fail: the is_default parameter is different
+        # even though the other contents are identical.
+        with pytest.raises(AssertionError):
+            assert self.sample_cal == Calibration.from_json(self.dummy_file_path, True)
+        with pytest.raises(AssertionError):
+            assert self.sample_default_cal == Calibration.from_json(
+                self.dummy_default_file_path, False
+            )
+
+        # from_json should ignore extra keys in the loaded file, but not fail
+        # Test this by trying to load a SensorCalibration as a Calibration
+        sensor_cal = SensorCalibration(
+            self.sample_cal.calibration_parameters,
+            self.sample_cal.calibration_data,
+            False,
+            tmp_path / "testing.json",
+            "dt_str",
+            [],
+            "uid",
+        )
+        sensor_cal.to_json()
+        loaded_cal = Calibration.from_json(tmp_path / "testing.json", False)
+        loaded_cal.file_path = self.sample_cal.file_path  # Force these to be the same
+        assert loaded_cal == self.sample_cal
+
+        # from_json should fail if required fields are missing
+        # Create an incorrect JSON file
+        almost_a_cal = {"calibration_parameters": []}
+        with open(tmp_path / "almost_a_cal.json", "w") as outfile:
+            outfile.write(json.dumps(almost_a_cal))
+        with pytest.raises(Exception):
+            almost = Calibration.from_json(tmp_path / "almost_a_cal.json", False)
+
+    def test_update_not_implemented(self):
+        """Ensure the update abstract method is not implemented in the base class"""
+        with pytest.raises(NotImplementedError):
+            self.sample_cal.update()
diff --git a/scos_actions/calibration/tests/test_differential_calibration.py b/scos_actions/calibration/tests/test_differential_calibration.py
new file mode 100644
index 00000000..8f1b5594
--- /dev/null
+++ b/scos_actions/calibration/tests/test_differential_calibration.py
@@ -0,0 +1,42 @@
+"""Test the DifferentialCalibration dataclass."""
+import json
+from pathlib import Path
+
+import pytest
+from scos_actions.calibration.differential_calibration import DifferentialCalibration
+
+
+class TestDifferentialCalibration:
+    @pytest.fixture(autouse=True)
+    def setup_differential_calibration_file(self, tmp_path: Path):
+        dict_to_json = {
+            "calibration_parameters": ["frequency"],
+            "reference_point": "antenna input",
+            "calibration_data": {3555e6: 11.5},
+        }
+        self.valid_file_path = tmp_path / "sample_diff_cal.json"
+        self.invalid_file_path = tmp_path / "sample_diff_cal_invalid.json"
+
+        self.sample_diff_cal = DifferentialCalibration(
+            is_default=False, file_path=self.valid_file_path, **dict_to_json
+        )
+
+        with open(self.valid_file_path, "w") as f:
+            f.write(json.dumps(dict_to_json))
+
+        dict_to_json.pop("reference_point", None)
+
+        with open(self.invalid_file_path, "w") as f:
+            f.write(json.dumps(dict_to_json))
+
+    def test_from_json(self):
+        """Check from_json functionality with valid and invalid dummy data."""
+        diff_cal = DifferentialCalibration.from_json(self.valid_file_path, False)
+        assert diff_cal == self.sample_diff_cal
+        with pytest.raises(Exception):
+            _ = DifferentialCalibration.from_json(self.invalid_file_path, False)
+
+    def test_update_not_implemented(self):
+        """Check that the update method is not implemented."""
+        with pytest.raises(NotImplementedError):
+            self.sample_diff_cal.update()
diff --git a/scos_actions/calibration/tests/test_sensor_calibration.py b/scos_actions/calibration/tests/test_sensor_calibration.py
index cf4403fa..6348c409 100644
--- a/scos_actions/calibration/tests/test_sensor_calibration.py
+++ b/scos_actions/calibration/tests/test_sensor_calibration.py
@@ -1,14 +1,18 @@
-"""Test the SensorCalibration class."""
-
+"""Test the SensorCalibration dataclass."""
+import dataclasses
+import datetime
 import json
 import random
 from copy import deepcopy
 from math import isclose
 from pathlib import Path
+from typing import Dict, List
 
 import pytest
+from scos_actions.calibration.interfaces.calibration import Calibration
 from scos_actions.calibration.sensor_calibration import SensorCalibration
-from scos_actions.calibration.utils import CalibrationException, filter_by_parameter
+from scos_actions.calibration.tests.utils import recursive_check_keys
+from scos_actions.calibration.utils import CalibrationException
 from scos_actions.tests.resources.utils import easy_gain
 from scos_actions.utils import get_datetime_str_now, parse_datetime_iso_format_str
 
@@ -93,7 +97,7 @@ def run_pytest_point(self, sr, f, g, reason, sr_m=False, f_m=False, g_m=False):
         return True
 
     @pytest.fixture(autouse=True)
-    def setup_calibration_file(self, tmpdir):
+    def setup_calibration_file(self, tmp_path: Path):
         """Create the dummy calibration file in the pytest temp directory"""
 
         # Only setup once
@@ -101,8 +105,7 @@ def setup_calibration_file(self, tmpdir):
             return
 
         # Create and save the temp directory and file
-        self.tmpdir = tmpdir.strpath
-        self.calibration_file = "{}".format(tmpdir.join("dummy_cal_file.json"))
+        self.calibration_file = tmp_path / "dummy_cal_file.json"
 
         # Setup variables
         self.dummy_noise_figure = 10
@@ -191,30 +194,43 @@ def setup_calibration_file(self, tmpdir):
         # Don't repeat test setup
         self.setup_complete = True
 
-    def test_filter_by_parameter_out_of_range(self):
-        calibrations = {200.0: {"some_cal_data"}, 300.0: {"more cal data"}}
-        with pytest.raises(CalibrationException) as e_info:
-            cal = filter_by_parameter(calibrations, 400.0)
-            assert (
-                e_info.value.args[0]
-                == f"Could not locate calibration data at 400.0"
-                + f"\nAttempted lookup using key '400.0'"
-                + f"\nUsing calibration data: {calibrations}"
-            )
+    def test_calibration_data_key_name_conversion(self):
+        """On post-init, all calibration_data key names should be converted to strings."""
+        recursive_check_keys(self.sample_cal.calibration_data)
 
-    def test_filter_by_parameter_in_range_requires_match(self):
-        calibrations = {
-            200.0: {"Gain": "Gain at 200.0"},
-            300.0: {"Gain": "Gain at 300.0"},
+    def test_sensor_calibration_dataclass_fields(self):
+        """Check that the dataclass fields are as expected."""
+        fields = {
+            f.name: f.type
+            for f in dataclasses.fields(SensorCalibration)
+            if f not in dataclasses.fields(Calibration)
         }
-        with pytest.raises(CalibrationException) as e_info:
-            cal = filter_by_parameter(calibrations, 150.0)
-            assert e_info.value.args[0] == (
-                f"Could not locate calibration data at 150.0"
-                + f"\nAttempted lookup using key '150.0'"
-                + f"\nUsing calibration data: {calibrations}"
+        # Note: does not check field order
+        assert fields == {
+            "last_calibration_datetime": str,
+            "clock_rate_lookup_by_sample_rate": List[Dict[str, float]],
+            "sensor_uid": str,
+        }
+
+    def test_field_validator(self):
+        """Check that the input field type validator works."""
+        # only check fields not inherited from Calibration base class
+        with pytest.raises(TypeError):
+            _ = SensorCalibration([], {}, False, Path(""), "dt_str", [], 10)
+        with pytest.raises(TypeError):
+            _ = SensorCalibration([], {}, False, Path(""), "dt_str", {}, "uid")
+        with pytest.raises(TypeError):
+            _ = SensorCalibration(
+                [], {}, False, Path(""), datetime.datetime.now(), [], "uid"
             )
 
+    def test_get_clock_rate(self):
+        """Test the get_clock_rate method"""
+        # Test getting a clock rate by sample rate
+        assert self.sample_cal.get_clock_rate(10e6) == 40e6
+        # If there isn't an entry, the sample rate should be returned
+        assert self.sample_cal.get_clock_rate(-999) == -999
+
     def test_get_calibration_dict_exact_match_lookup(self):
         calibration_datetime = get_datetime_str_now()
         calibration_params = ["sample_rate", "frequency"]
@@ -228,7 +244,7 @@ def test_get_calibration_dict_exact_match_lookup(self):
             is_default=False,
             file_path=Path(""),
             last_calibration_datetime=calibration_datetime,
-            clock_rate_lookup_by_sample_rate={},
+            clock_rate_lookup_by_sample_rate=[],
             sensor_uid="TESTING",
         )
         cal_data = cal.get_calibration_dict([100.0, 200.0])
@@ -247,16 +263,16 @@ def test_get_calibration_dict_within_range(self):
             is_default=False,
             file_path=Path("test_calibration.json"),
             last_calibration_datetime=calibration_datetime,
-            clock_rate_lookup_by_sample_rate={},
+            clock_rate_lookup_by_sample_rate=[],
             sensor_uid="TESTING",
         )
         with pytest.raises(CalibrationException) as e_info:
-            cal_data = cal.get_calibration_dict([100.0, 250.0])
-            assert e_info.value.args[0] == (
-                f"Could not locate calibration data at 250.0"
-                + f"\nAttempted lookup using key '250.0'"
-                + f"\nUsing calibration data: {cal.calibration_data}"
-            )
+            _ = cal.get_calibration_dict([100.0, 250.0])
+        assert e_info.value.args[0] == (
+            f"Could not locate calibration data at 250.0"
+            + f"\nAttempted lookup using key '250.0'"
+            + f"\nUsing calibration data: {cal.calibration_data['100.0']}"
+        )
 
     def test_sf_bound_points(self):
         """Test SF determination at boundary points"""
@@ -289,7 +305,7 @@ def test_update(self):
             is_default=False,
             file_path=test_cal_path,
             last_calibration_datetime=calibration_datetime,
-            clock_rate_lookup_by_sample_rate={},
+            clock_rate_lookup_by_sample_rate=[],
             sensor_uid="TESTING",
         )
         action_params = {"sample_rate": 100.0, "frequency": 200.0}
@@ -308,8 +324,3 @@ def test_update(self):
         assert cal.calibration_data["100.0"]["200.0"]["noise_figure"] == 5.0
         assert cal_from_file.calibration_data["100.0"]["200.0"]["gain"] == 30.0
         assert cal_from_file.calibration_data["100.0"]["200.0"]["noise_figure"] == 5.0
-
-    def test_filter_by_paramter_integer(self):
-        calibrations = {"200.0": {"some_cal_data"}, 300.0: {"more cal data"}}
-        filtered_data = filter_by_parameter(calibrations, 200)
-        assert filtered_data is calibrations["200.0"]
diff --git a/scos_actions/calibration/tests/test_utils.py b/scos_actions/calibration/tests/test_utils.py
new file mode 100644
index 00000000..2a751831
--- /dev/null
+++ b/scos_actions/calibration/tests/test_utils.py
@@ -0,0 +1,41 @@
+import pytest
+from scos_actions.calibration.utils import CalibrationException, filter_by_parameter
+
+
+class TestCalibrationUtils:
+    def test_filter_by_parameter_out_of_range(self):
+        calibrations = {200.0: {"some_cal_data"}, 300.0: {"more cal data"}}
+        with pytest.raises(CalibrationException) as e_info:
+            _ = filter_by_parameter(calibrations, 400.0)
+        assert (
+            e_info.value.args[0]
+            == f"Could not locate calibration data at 400.0"
+            + f"\nAttempted lookup using key '400.0'"
+            + f"\nUsing calibration data: {calibrations}"
+        )
+
+    def test_filter_by_parameter_in_range_requires_match(self):
+        calibrations = {
+            200.0: {"Gain": "Gain at 200.0"},
+            300.0: {"Gain": "Gain at 300.0"},
+        }
+        with pytest.raises(CalibrationException) as e_info:
+            _ = filter_by_parameter(calibrations, 150.0)
+        assert e_info.value.args[0] == (
+            f"Could not locate calibration data at 150.0"
+            + f"\nAttempted lookup using key '150.0'"
+            + f"\nUsing calibration data: {calibrations}"
+        )
+
+    def test_filter_by_paramter_integer(self):
+        calibrations = {"200.0": {"some_cal_data"}, 300.0: {"more cal data"}}
+        filtered_data = filter_by_parameter(calibrations, 200)
+        assert filtered_data is calibrations["200.0"]
+
+    def test_filter_by_parameter_type_error(self):
+        calibrations = [300.0, 400.0]
+        with pytest.raises(CalibrationException) as e_info:
+            _ = filter_by_parameter(calibrations, 300.0)
+        assert e_info.value.args[0] == (
+            f"Provided calibration data is not a dict: {calibrations}"
+        )
diff --git a/scos_actions/calibration/tests/utils.py b/scos_actions/calibration/tests/utils.py
new file mode 100644
index 00000000..a2ad6e51
--- /dev/null
+++ b/scos_actions/calibration/tests/utils.py
@@ -0,0 +1,10 @@
+"""Utility functions used for scos_sensor.calibration unit tests."""
+
+
+def recursive_check_keys(d: dict):
+    """Recursively checks a dict to check that all keys are strings"""
+    for k, v in d.items():
+        if isinstance(v, dict):
+            recursive_check_keys(v)
+        else:
+            assert isinstance(k, str)

From c304e0207c412141e0d81682382d7c7a21891bff Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 22 Jan 2024 15:33:32 -0500
Subject: [PATCH 011/102] bump version number

---
 scos_actions/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scos_actions/__init__.py b/scos_actions/__init__.py
index 73d4c8be..6dcf770e 100644
--- a/scos_actions/__init__.py
+++ b/scos_actions/__init__.py
@@ -1 +1 @@
-__version__ = "8.0.0"
+__version__ = "9.0.0"

From fc2d450b6adba1a494023c9918a56484fb6f9d76 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 22 Jan 2024 16:07:11 -0500
Subject: [PATCH 012/102] remove unused imports

---
 scos_actions/calibration/sensor_calibration.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index 57604c62..12997f22 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -1,7 +1,5 @@
-import json
 import logging
 from dataclasses import dataclass
-from pathlib import Path
 from typing import Dict, List, Union
 
 from scos_actions.calibration.interfaces.calibration import Calibration

From 013b791e7e1d9db666bdcc7c8d6a762625cc83b7 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Tue, 23 Jan 2024 10:37:04 -0500
Subject: [PATCH 013/102] remove unused imports

---
 scos_actions/hardware/utils.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/scos_actions/hardware/utils.py b/scos_actions/hardware/utils.py
index d9f6fbf2..b479f2a5 100644
--- a/scos_actions/hardware/utils.py
+++ b/scos_actions/hardware/utils.py
@@ -1,14 +1,9 @@
 import logging
 import subprocess
-from pathlib import Path
 from typing import Dict
 
 import psutil
-from its_preselector.configuration_exception import ConfigurationException
-from its_preselector.controlbyweb_web_relay import ControlByWebWebRelay
 from its_preselector.web_relay import WebRelay
-
-from scos_actions import utils
 from scos_actions.hardware.hardware_configuration_exception import (
     HardwareConfigurationException,
 )

From ddac9c5ce322b26da55d38467c6a8698ea1bba4a Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Tue, 23 Jan 2024 20:05:53 -0500
Subject: [PATCH 014/102] simplify expressions in Y-factor cal

Avoids some redundant additions and subtractions by working internally in dBW instead of dBm
---
 scos_actions/signal_processing/calibration.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/scos_actions/signal_processing/calibration.py b/scos_actions/signal_processing/calibration.py
index 757dfc50..3f98ade8 100644
--- a/scos_actions/signal_processing/calibration.py
+++ b/scos_actions/signal_processing/calibration.py
@@ -11,7 +11,6 @@
     convert_celsius_to_kelvins,
     convert_dB_to_linear,
     convert_linear_to_dB,
-    convert_watts_to_dBm,
 )
 
 logger = logging.getLogger(__name__)
@@ -43,16 +42,16 @@ def y_factor(
     :return: A tuple (noise_figure, gain) containing the calculated
         noise figure and gain, both in dB, from the Y-factor method.
     """
-    mean_on_dBm = convert_watts_to_dBm(np.mean(pwr_noise_on_watts))
-    mean_off_dBm = convert_watts_to_dBm(np.mean(pwr_noise_off_watts))
+    mean_on_dBW = convert_linear_to_dB(np.mean(pwr_noise_on_watts))
+    mean_off_dBW = convert_linear_to_dB(np.mean(pwr_noise_off_watts))
     if logger.isEnabledFor(logging.DEBUG):
         logger.debug(f"ENR: {convert_linear_to_dB(enr_linear)} dB")
         logger.debug(f"ENBW: {enbw_hz} Hz")
-        logger.debug(f"Mean power on: {mean_on_dBm:.2f} dBm")
-        logger.debug(f"Mean power off: {mean_off_dBm:.2f} dBm")
-    y = convert_dB_to_linear(mean_on_dBm - mean_off_dBm)
+        logger.debug(f"Mean power on: {mean_on_dBW+30:.2f} dBm")
+        logger.debug(f"Mean power off: {mean_off_dBW+30:.2f} dBm")
+    y = convert_dB_to_linear(mean_on_dBW - mean_off_dBW)
     noise_factor = enr_linear / (y - 1.0)
-    gain_dB = mean_on_dBm - convert_watts_to_dBm(
+    gain_dB = mean_on_dBW - convert_linear_to_dB(
         Boltzmann * temp_kelvins * enbw_hz * (enr_linear + noise_factor)
     )
     noise_figure_dB = convert_linear_to_dB(noise_factor)

From f978515e097bafa6f0771dbff7f79bd4c08ed199 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Tue, 23 Jan 2024 20:23:48 -0500
Subject: [PATCH 015/102] Use "loss" as value key instead of
 "differential_loss"

---
 scos_actions/calibration/differential_calibration.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scos_actions/calibration/differential_calibration.py b/scos_actions/calibration/differential_calibration.py
index cc013c6a..4cf40322 100644
--- a/scos_actions/calibration/differential_calibration.py
+++ b/scos_actions/calibration/differential_calibration.py
@@ -10,9 +10,11 @@
 are referenced after using the correction factors included in the file.
 
 The ``calibration_data`` entries are expected to include these correction factors,
-with the key name ``"differential_loss"`` and values in decibels (dB). These correction
-factors represent the differential loss between the calibration terminal used by onboard
+with the key name ``"loss"`` and values in decibels (dB). These correction factors
+represent the differential loss between the calibration terminal used by onboard
 ``SensorCalibration`` results and the reference point defined by ``reference_point``.
+A positive value of ``"loss"`` indicates a LOSS going FROM ``reference_point`` TO
+the calibration terminal used by the ``SensorCalibration``.
 """
 
 from dataclasses import dataclass

From 0832db6e2babe647d0ecc2b2b42d6d4e47f3ee12 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Tue, 23 Jan 2024 20:24:51 -0500
Subject: [PATCH 016/102] clarify functionality of _validate_fields

---
 scos_actions/calibration/interfaces/calibration.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scos_actions/calibration/interfaces/calibration.py b/scos_actions/calibration/interfaces/calibration.py
index 2e339966..406e325e 100644
--- a/scos_actions/calibration/interfaces/calibration.py
+++ b/scos_actions/calibration/interfaces/calibration.py
@@ -27,6 +27,8 @@ def __post_init__(self):
     def _validate_fields(self) -> None:
         """Loosely check that the input types are as expected."""
         for f_name, f_def in self.__dataclass_fields__.items():
+            # Note that nested types are not checked: i.e., "List[str]"
+            # will surely be a list, but may not be filled with strings.
             f_type = get_origin(f_def.type) or f_def.type
             actual_value = getattr(self, f_name)
             if not isinstance(actual_value, f_type):

From 49503f71903ed23fa2231fd78e41375b17ffffda Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Tue, 23 Jan 2024 21:15:26 -0500
Subject: [PATCH 017/102] implement sensor-level acquire_samples

- fully remove calibration from sigan interfaces. all calibration handling happens at the sensor object level
- alter the sigan interface: retry logic and calibration scaling now happens in the sensor acquire_samples method instead of sigan versions.
- dynamically set data reference point as part the measurement result based on which calibration scaling was applied
---
 scos_actions/hardware/mocks/mock_sigan.py |  19 +-
 scos_actions/hardware/sensor.py           | 214 +++++++++++++++++++---
 scos_actions/hardware/sigan_iface.py      |  41 +----
 3 files changed, 197 insertions(+), 77 deletions(-)

diff --git a/scos_actions/hardware/mocks/mock_sigan.py b/scos_actions/hardware/mocks/mock_sigan.py
index 2fdc04df..03a9e42b 100644
--- a/scos_actions/hardware/mocks/mock_sigan.py
+++ b/scos_actions/hardware/mocks/mock_sigan.py
@@ -4,7 +4,6 @@
 from typing import Optional
 
 import numpy as np
-from scos_actions.calibration.sensor_calibration import SensorCalibration
 from scos_actions.hardware.sigan_iface import SignalAnalyzerInterface
 from scos_actions.utils import get_datetime_str_now
 
@@ -28,11 +27,10 @@ class MockSignalAnalyzer(SignalAnalyzerInterface):
 
     def __init__(
         self,
-        sensor_cal: Optional[SensorCalibration] = None,
         switches: Optional[dict] = None,
         randomize_values: bool = False,
     ):
-        super().__init__(sensor_cal)
+        super().__init__(switches)
         # Define the default calibration dict
         self.DEFAULT_SENSOR_CALIBRATION = {
             "datetime": get_datetime_str_now(),
@@ -136,8 +134,8 @@ def connect(self):
         pass
 
     def acquire_time_domain_samples(
-        self, num_samples, num_samples_skip=0, retries=5, cal_adjust=True
-    ):
+        self, num_samples: int, num_samples_skip: int = 0, retries: int = 5
+    ) -> dict:
         logger.warning("Using mock signal analyzer!")
         self.sigan_overload = False
         self._capture_time = None
@@ -194,14 +192,3 @@ def set_times_to_fail_recv(self, n):
     @property
     def last_calibration_time(self):
         return get_datetime_str_now()
-
-    def update_calibration(self, params):
-        pass
-
-    def recompute_sensor_calibration_data(self, cal_args: list) -> None:
-        if self.sensor_calibration is not None:
-            self.sensor_calibration_data.update(
-                self._sensor_calibration.get_calibration_dict(cal_args)
-            )
-        else:
-            logger.warning("Sensor calibration does not exist.")
diff --git a/scos_actions/hardware/sensor.py b/scos_actions/hardware/sensor.py
index c4120a33..91f9a273 100644
--- a/scos_actions/hardware/sensor.py
+++ b/scos_actions/hardware/sensor.py
@@ -1,25 +1,35 @@
 import datetime
 import hashlib
 import json
-from typing import Dict
+import logging
+from typing import Dict, Optional
 
 from its_preselector.preselector import Preselector
 from its_preselector.web_relay import WebRelay
+from scos_actions.calibration.differential_calibration import DifferentialCalibration
+from scos_actions.calibration.interfaces.calibration import Calibration
+from scos_actions.calibration.sensor_calibration import SensorCalibration
+from scos_actions.hardware.gps_iface import GPSInterface
+from scos_actions.hardware.sigan_iface import (
+    SIGAN_SETTINGS_KEYS,
+    SignalAnalyzerInterface,
+)
+from scos_actions.utils import convert_string_to_millisecond_iso_format
 
-from .gps_iface import GPSInterface
-from .mocks.mock_sigan import MockSignalAnalyzer
-from .sigan_iface import SignalAnalyzerInterface
+logger = logging.getLogger(__name__)
 
 
 class Sensor:
     def __init__(
         self,
-        signal_analyzer: SignalAnalyzerInterface = None,
-        gps: GPSInterface = None,
-        preselector: Preselector = None,
+        signal_analyzer: Optional[SignalAnalyzerInterface] = None,
+        gps: Optional[GPSInterface] = None,
+        preselector: Optional[Preselector] = None,
         switches: Dict[str, WebRelay] = {},
-        location: dict = None,
-        capabilities: dict = None,
+        location: Optional[dict] = None,
+        capabilities: Optional[dict] = None,
+        sensor_cal: Optional[SensorCalibration] = None,
+        differential_cal: Optional[DifferentialCalibration] = None,
     ):
         self.signal_analyzer = signal_analyzer
         self.gps = gps
@@ -27,31 +37,34 @@ def __init__(
         self.switches = switches
         self.location = location
         self.capabilities = capabilities
+        self.sensor_calibration_data = {}
+        self._sensor_calibration = sensor_cal
+        self._differential_calibration = differential_cal
         # There is no setter for start_time property
         self._start_time = datetime.datetime.utcnow()
 
     @property
-    def signal_analyzer(self) -> SignalAnalyzerInterface:
+    def signal_analyzer(self) -> Optional[SignalAnalyzerInterface]:
         return self._signal_analyzer
 
     @signal_analyzer.setter
-    def signal_analyzer(self, sigan: SignalAnalyzerInterface):
+    def signal_analyzer(self, sigan: Optional[SignalAnalyzerInterface]):
         self._signal_analyzer = sigan
 
     @property
-    def gps(self) -> GPSInterface:
+    def gps(self) -> Optional[GPSInterface]:
         return self._gps
 
     @gps.setter
-    def gps(self, gps: GPSInterface):
+    def gps(self, gps: Optional[GPSInterface]):
         self._gps = gps
 
     @property
-    def preselector(self) -> Preselector:
+    def preselector(self) -> Optional[Preselector]:
         return self._preselector
 
     @preselector.setter
-    def preselector(self, preselector: Preselector):
+    def preselector(self, preselector: Optional[Preselector]):
         self._preselector = preselector
 
     @property
@@ -63,19 +76,19 @@ def switches(self, switches: Dict[str, WebRelay]):
         self._switches = switches
 
     @property
-    def location(self) -> dict:
+    def location(self) -> Optional[dict]:
         return self._location
 
     @location.setter
-    def location(self, loc: dict):
+    def location(self, loc: Optional[dict]):
         self._location = loc
 
     @property
-    def capabilities(self) -> dict:
+    def capabilities(self) -> Optional[dict]:
         return self._capabilities
 
     @capabilities.setter
-    def capabilities(self, capabilities: dict):
+    def capabilities(self, capabilities: Optional[dict]):
         if capabilities is not None:
             if "sensor_sha512" not in capabilities["sensor"]:
                 sensor_def = json.dumps(capabilities["sensor"], sort_keys=True)
@@ -83,9 +96,7 @@ def capabilities(self, capabilities: dict):
                     sensor_def.encode("UTF-8")
                 ).hexdigest()
                 capabilities["sensor"]["sensor_sha512"] = SENSOR_DEFINITION_HASH
-            self._capabilities = capabilities
-        else:
-            self._capabilities = None
+        self._capabilities = capabilities
 
     @property
     def has_configurable_preselector(self) -> bool:
@@ -102,5 +113,162 @@ def has_configurable_preselector(self) -> bool:
                 return False
 
     @property
-    def start_time(self):
+    def start_time(self) -> datetime.datetime:
         return self._start_time
+
+    @property
+    def sensor_calibration(self) -> Optional[SensorCalibration]:
+        return self._sensor_calibration
+
+    @sensor_calibration.setter
+    def sensor_calibration(self, cal: Optional[SensorCalibration]):
+        self._sensor_calibration = cal
+
+    @property
+    def differential_calibration(self) -> Optional[DifferentialCalibration]:
+        return self._differential_calibration
+
+    @differential_calibration.setter
+    def differential_calibration(self, cal: Optional[DifferentialCalibration]):
+        self._differential_calibration = cal
+
+    @property
+    def last_calibration_time(self) -> str:
+        """Get a datetime string for the most recent sensor calibration."""
+        return convert_string_to_millisecond_iso_format(
+            self.sensor_calibration.last_calibration_datetime
+        )
+
+    def _get_calibration_args_from_sigan(self, calibration: Calibration) -> list:
+        """Get current values of signal analyzer settings which are calibration parameters."""
+        cal_params = [
+            p for p in calibration.calibration_parameters if p in SIGAN_SETTINGS_KEYS
+        ]
+        if set(cal_params) <= set(calibration.calibration_parameters):
+            msg = (
+                "One or more required calibration parameters is not a valid signal "
+                + f"analyzer property.\nRequired parameters: {calibration.calibration_parameters}"
+                + f"\nSignal analyzer properties: {list(vars(self.signal_analyzer).keys())}"
+            )
+            logger.error(msg)
+            raise KeyError
+        cal_args = [vars(self.signal_analyzer)[f"_{p}"] for p in cal_params]
+        logger.debug(f"Matched calibration params: {cal_args}")
+        return cal_args  # Order matches calibration.calibration_parameters
+
+    def recompute_differential_calibration_data(self) -> None:
+        """Set the differential calibration data based on the current tuning."""
+        self.differential_calibration_data = {}
+        if self.differential_calibration is not None:
+            cal_args = self._get_calibration_args_from_sigan(
+                self.differential_calibration
+            )
+            self.differential_calibration_data.update(
+                self.differential_calibration.get_calibration_dict(cal_args)
+            )
+        else:
+            logger.warning("Differential calibration does not exist.")
+
+    def recompute_sensor_calibration_data(self) -> None:
+        """Set the sensor calibration data based on the current tuning."""
+        self.sensor_calibration_data = {}
+        if self.sensor_calibration is not None:
+            cal_args = self._get_calibration_args_from_sigan(self.sensor_calibration)
+            self.sensor_calibration_data.update(
+                self.sensor_calibration.get_calibration_dict(cal_args)
+            )
+        else:
+            logger.warning("Sensor calibration does not exist.")
+
+    def acquire_time_domain_samples(
+        self,
+        num_samples: int,
+        num_samples_skip: int = 0,
+        retries: int = 5,
+        cal_adjust: bool = True,
+    ) -> dict:
+        """
+        Acquire time-domain IQ samples from the signal analyzer.
+
+        Signal analyzer settings, preselector state, etc. should already be
+        set before calling this function.
+
+        Gain adjustment can be applied to acquired samples using ``cal_adjust``.
+        If ``True``, the samples acquired from the signal analyzer will be
+        scaled based on the calibrated ``gain`` value in the ``SensorCalibration``,
+        if one exists for this sensor, and "calibration terminal" will be the value
+        of the "reference" key in the returned dict.
+
+        :param num_samples: Number of samples to acquire
+        :param num_samples_skip: Number of samples to skip
+        :param retries: Maximum number of retries on failure
+        :param cal_adjust: If True, use available calibration data to scale the samples.
+        :return: dictionary containing data, sample_rate, frequency, capture_time, etc
+        :raises Exception: If the sample acquisition fails, or the sensor has
+            no signal analyzer.
+        """
+        logger.debug("Sensor.acquire_time_domain_samples starting")
+        logger.debug(f"Number of retries = {retries}")
+        max_retries = retries
+        # TODO: Include RF path as a sensor cal argument?
+        # Acquire samples from signal analyzer
+        if self.signal_analyzer is not None:
+            while True:
+                try:
+                    measurement_result = (
+                        self.signal_analyzer.acquire_time_domain_samples(
+                            num_samples, num_samples_skip
+                        )
+                    )
+                    break
+                except Exception as e:
+                    retries -= 1
+                    logger.info("Error while acquiring samples from signal analyzer.")
+                    if retries == 0:
+                        logger.exception(
+                            "Failed to acquire samples from signal analyzer. "
+                            + f"Tried {max_retries} times."
+                        )
+                        raise e
+        else:
+            msg = "Failed to acquire samples: sensor has no signal analyzer"
+            logger.error(msg)
+            raise Exception(msg)
+
+        # Apply gain adjustment based on calibration
+        if cal_adjust:
+            if self.sensor_calibration is not None:
+                logger.debug("Scaling samples using calibration data")
+                calibrated_gain__db = 0.0
+                self.recompute_sensor_calibration_data()
+                sensor_gain = self.sensor_calibration_data["gain"]
+                logger.debug(f"Using sensor gain: {sensor_gain} dB")
+                calibrated_gain__db += sensor_gain
+                if self.differential_calibration is not None:
+                    # Also apply differential calibration correction
+                    # TODO recompute functions match to current signal analyzer
+                    # settings. should they use the measurement_result instead?
+                    self.recompute_differential_calibration_data()
+                    differential_loss = self.differential_calibration_data["loss"]
+                    logger.debug(f"Using differential loss: {differential_loss} dB")
+                    calibrated_gain__db -= differential_loss
+                    measurement_result[
+                        "reference"
+                    ] = self.differential_calibration.reference_point
+                else:
+                    # No differential calibration exists
+                    logger.debug("No differential calibration was applied")
+                    measurement_result["reference"] = "calibration terminal"
+                linear_gain = 10.0 ** (calibrated_gain__db / 20.0)
+                logger.debug(f"Applying gain of {linear_gain}")
+                measurement_result["data"] /= linear_gain
+            else:
+                # No sensor calibration exists
+                msg = "Unable to scale samples without sensor calibration data"
+                logger.error(msg)
+                raise Exception(msg)
+        else:
+            # Set the data reference in the measurement_result
+            measurement_result["reference"] = "signal analyzer input"
+
+        return measurement_result
diff --git a/scos_actions/hardware/sigan_iface.py b/scos_actions/hardware/sigan_iface.py
index 68ed0463..f7fb0bed 100644
--- a/scos_actions/hardware/sigan_iface.py
+++ b/scos_actions/hardware/sigan_iface.py
@@ -4,9 +4,7 @@
 from typing import Dict, Optional
 
 from its_preselector.web_relay import WebRelay
-from scos_actions.calibration.sensor_calibration import SensorCalibration
 from scos_actions.hardware.utils import power_cycle_sigan
-from scos_actions.utils import convert_string_to_millisecond_iso_format
 
 logger = logging.getLogger(__name__)
 
@@ -25,21 +23,11 @@
 class SignalAnalyzerInterface(ABC):
     def __init__(
         self,
-        sensor_cal: Optional[SensorCalibration] = None,
         switches: Optional[Dict[str, WebRelay]] = None,
     ):
-        self.sensor_calibration_data = {}
-        self._sensor_calibration = sensor_cal
         self._model = "Unknown"
         self.switches = switches
 
-    @property
-    def last_calibration_time(self) -> str:
-        """Returns the last calibration time from calibration data."""
-        return convert_string_to_millisecond_iso_format(
-            self.sensor_calibration.last_calibration_datetime
-        )
-
     @property
     @abstractmethod
     def is_available(self) -> bool:
@@ -67,16 +55,13 @@ def acquire_time_domain_samples(
         self,
         num_samples: int,
         num_samples_skip: int = 0,
-        retries: int = 5,
-        cal_adjust: bool = True,
     ) -> dict:
         """
-        Acquire time domain IQ samples
+        Acquire time domain IQ samples, scaled to Volts at
+        the signal analyzer input.
 
         :param num_samples: Number of samples to acquire
         :param num_samples_skip: Number of samples to skip
-        :param retries: Maximum number of retries on failure
-        :param cal_adjust: If True, scale IQ samples based on calibration data.
         :return: dictionary containing data, sample_rate, frequency, capture_time, etc
         """
         pass
@@ -94,9 +79,7 @@ def healthy(self, num_samples: int = 56000) -> bool:
         if not self.is_available:
             return False
         try:
-            measurement_result = self.acquire_time_domain_samples(
-                num_samples, cal_adjust=False
-            )
+            measurement_result = self.acquire_time_domain_samples(num_samples)
             data = measurement_result["data"]
         except Exception as e:
             logger.exception("Unable to acquire samples from device.")
@@ -132,16 +115,6 @@ def power_cycle_and_connect(self, sleep_time: float = 2.0) -> None:
             )
         return
 
-    def recompute_sensor_calibration_data(self, cal_args: list) -> None:
-        """Set the sensor calibration data based on the current tuning."""
-        self.sensor_calibration_data = {}
-        if self.sensor_calibration is not None:
-            self.sensor_calibration_data.update(
-                self.sensor_calibration.get_calibration_dict(cal_args)
-            )
-        else:
-            logger.warning("Sensor calibration does not exist.")
-
     def get_status(self) -> dict:
         return {"model": self._model, "healthy": self.healthy()}
 
@@ -152,11 +125,3 @@ def model(self) -> str:
     @model.setter
     def model(self, value: str):
         self._model = value
-
-    @property
-    def sensor_calibration(self) -> SensorCalibration:
-        return self._sensor_calibration
-
-    @sensor_calibration.setter
-    def sensor_calibration(self, cal: SensorCalibration):
-        self._sensor_calibration = cal

From 4a81a2ad9e39e748e94a177078122a6f523807a2 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Tue, 23 Jan 2024 21:18:12 -0500
Subject: [PATCH 018/102] remove unused import

---
 scos_actions/settings.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scos_actions/settings.py b/scos_actions/settings.py
index fa63aaa1..7d45d179 100644
--- a/scos_actions/settings.py
+++ b/scos_actions/settings.py
@@ -1,5 +1,4 @@
 import logging
-from os import path
 from pathlib import Path
 
 from environs import Env

From 43ac4be377d9141c9a60b789d4dd870df7fbc7f9 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Tue, 23 Jan 2024 21:50:07 -0500
Subject: [PATCH 019/102] update actions for sensor cal handling changes

---
 .../actions/acquire_sea_data_product.py       | 33 ++++++++++---------
 .../actions/acquire_single_freq_fft.py        |  8 ++---
 .../actions/acquire_single_freq_tdomain_iq.py |  2 +-
 .../acquire_stepped_freq_tdomain_iq.py        | 11 ++++---
 scos_actions/actions/calibrate_y_factor.py    | 26 ++++++---------
 .../actions/interfaces/measurement_action.py  |  4 +--
 scos_actions/hardware/sensor.py               | 19 ++++++++---
 7 files changed, 53 insertions(+), 50 deletions(-)

diff --git a/scos_actions/actions/acquire_sea_data_product.py b/scos_actions/actions/acquire_sea_data_product.py
index 3409699a..003db3b8 100644
--- a/scos_actions/actions/acquire_sea_data_product.py
+++ b/scos_actions/actions/acquire_sea_data_product.py
@@ -34,9 +34,6 @@
 from environs import Env
 from its_preselector import __version__ as PRESELECTOR_API_VERSION
 from scipy.signal import sos2tf, sosfilt
-
-from scos_actions import __version__ as SCOS_ACTIONS_VERSION
-from scos_actions import utils
 from scos_actions.actions.interfaces.action import Action
 from scos_actions.hardware.sensor import Sensor
 from scos_actions.hardware.utils import (
@@ -76,6 +73,9 @@
 from scos_actions.signals import measurement_action_completed, trigger_api_restart
 from scos_actions.utils import convert_datetime_to_millisecond_iso_format, get_days_up
 
+from scos_actions import __version__ as SCOS_ACTIONS_VERSION
+from scos_actions import utils
+
 env = Env()
 logger = logging.getLogger(__name__)
 
@@ -112,7 +112,7 @@
 FFT_WINDOW = get_fft_window(FFT_WINDOW_TYPE, FFT_SIZE)
 FFT_WINDOW_ECF = get_fft_window_correction(FFT_WINDOW, "energy")
 IMPEDANCE_OHMS = 50.0
-DATA_REFERENCE_POINT = "noise source output"
+DATA_REFERENCE_POINT = "noise source output"  # TODO delete
 NUM_ACTORS = 3  # Number of ray actors to initialize
 
 # Create power detectors
@@ -626,14 +626,13 @@ def capture_iq(self, params: dict) -> dict:
         nskip = utils.get_parameter(NUM_SKIP, params)
         num_samples = int(params[SAMPLE_RATE] * duration_ms * 1e-3)
         # Collect IQ data
-        measurement_result = self.sensor.signal_analyzer.acquire_time_domain_samples(
-            num_samples, nskip
-        )
+        measurement_result = self.sensor.acquire_time_domain_samples(num_samples, nskip)
         # Store some metadata with the IQ
         measurement_result.update(params)
+        measurement_result["sensor_cal"] = self.sensor.sensor_calibration_data
         measurement_result[
-            "sensor_cal"
-        ] = self.sensor.signal_analyzer.sensor_calibration_data
+            "differential_cal"
+        ] = self.sensor.differential_calibration_data
         toc = perf_counter()
         logger.debug(
             f"IQ Capture ({duration_ms} ms @ {(params[FREQUENCY]/1e6):.1f} MHz) completed in {toc-tic:.2f} s."
@@ -1019,7 +1018,7 @@ def create_global_data_product_metadata(self) -> None:
             x_step=[p[SAMPLE_RATE] / FFT_SIZE],
             y_units="dBm/Hz",
             processing=[dft_obj.id],
-            reference=DATA_REFERENCE_POINT,
+            reference=DATA_REFERENCE_POINT,  # TODO update
             description=(
                 "Results of statistical detectors (max, mean, median, 25th_percentile, 75th_percentile, "
                 + "90th_percentile, 95th_percentile, 99th_percentile, 99.9th_percentile, 99.99th_percentile) "
@@ -1039,7 +1038,7 @@ def create_global_data_product_metadata(self) -> None:
             x_stop=[pvt_x_axis__s[-1]],
             x_step=[pvt_x_axis__s[1] - pvt_x_axis__s[0]],
             y_units="dBm",
-            reference=DATA_REFERENCE_POINT,
+            reference=DATA_REFERENCE_POINT,  # TODO update
             description=(
                 "Max- and mean-detected channel power vs. time, with "
                 + f"an integration time of {p[TD_BIN_SIZE_MS]} ms. "
@@ -1066,7 +1065,7 @@ def create_global_data_product_metadata(self) -> None:
             x_stop=[pfp_x_axis__s[-1]],
             x_step=[pfp_x_axis__s[1] - pfp_x_axis__s[0]],
             y_units="dBm",
-            reference=DATA_REFERENCE_POINT,
+            reference=DATA_REFERENCE_POINT,  # TODO update
             description=(
                 "Channelized periodic frame power statistics reported over"
                 + f" a {p[PFP_FRAME_PERIOD_MS]} ms frame period, with frame resolution"
@@ -1122,11 +1121,13 @@ def create_capture_segment(
             duration=measurement_result[DURATION_MS],
             overload=measurement_result["overload"],
             sensor_calibration=ntia_sensor.Calibration(
-                datetime=measurement_result["sensor_cal"]["datetime"],
-                gain=round(measurement_result["sensor_cal"]["gain"], 3),
-                noise_figure=round(measurement_result["sensor_cal"]["noise_figure"], 3),
+                datetime=self.sensor.sensor_calibration_data["datetime"],
+                gain=round(measurement_result["applied_calibration"]["gain"], 3),
+                noise_figure=round(
+                    measurement_result["applied_calibration"]["noise_figure"], 3
+                ),
                 temperature=round(measurement_result["sensor_cal"]["temperature"], 1),
-                reference=DATA_REFERENCE_POINT,
+                reference=measurement_result["reference"],
             ),
             sigan_settings=ntia_sensor.SiganSettings(
                 reference_level=self.sensor.signal_analyzer.reference_level,
diff --git a/scos_actions/actions/acquire_single_freq_fft.py b/scos_actions/actions/acquire_single_freq_fft.py
index aa6f16ba..a54cb785 100644
--- a/scos_actions/actions/acquire_single_freq_fft.py
+++ b/scos_actions/actions/acquire_single_freq_fft.py
@@ -152,10 +152,6 @@ def __init__(self, parameters: dict):
         self.classification = get_parameter(CLASSIFICATION, self.parameters)
         self.cal_adjust = get_parameter(CAL_ADJUST, self.parameters)
         assert isinstance(self.cal_adjust, bool)
-        if self.cal_adjust:
-            self.data_reference = "calibration terminal"
-        else:
-            self.data_reference = "signal analyzer input"
         # FFT setup
         self.fft_detector = create_statistical_detector(
             "M4sDetector", ["min", "max", "mean", "median", "sample"]
@@ -179,7 +175,7 @@ def execute(self, schedule_entry: dict, task_id: int) -> dict:
         measurement_result.update(self.parameters)
         measurement_result[
             "calibration_datetime"
-        ] = self.sensor.signal_analyzer.sensor_calibration_data["datetime"]
+        ] = self.sensor.sensor_calibration_data["datetime"]
         measurement_result["task_id"] = task_id
         measurement_result["classification"] = self.classification
 
@@ -269,7 +265,7 @@ def create_metadata(self, measurement_result: dict, recording: int = None) -> No
             x_stop=[frequencies[-1]],
             x_step=[frequencies[1] - frequencies[0]],
             y_units="dBm",
-            reference=self.data_reference,
+            reference=measurement_result["reference"],
             description=(
                 "Results of min, max, mean, and median statistical detectors, "
                 + f"along with a random sampling, from a set of {self.nffts} "
diff --git a/scos_actions/actions/acquire_single_freq_tdomain_iq.py b/scos_actions/actions/acquire_single_freq_tdomain_iq.py
index 0eb55655..a2261fbf 100644
--- a/scos_actions/actions/acquire_single_freq_tdomain_iq.py
+++ b/scos_actions/actions/acquire_single_freq_tdomain_iq.py
@@ -91,7 +91,7 @@ def execute(self, schedule_entry: dict, task_id: int) -> dict:
         measurement_result["task_id"] = task_id
         measurement_result[
             "calibration_datetime"
-        ] = self.sensor.signal_analyzer.sensor_calibration_data["datetime"]
+        ] = self.sensor.sensor_calibration_data["datetime"]
         measurement_result["classification"] = self.classification
         sigan_settings = self.get_sigan_settings(measurement_result)
         logger.debug(f"sigan settings:{sigan_settings}")
diff --git a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
index bd114e57..72995f14 100644
--- a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
+++ b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
@@ -117,15 +117,18 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
                 overload=measurement_result["overload"],
                 sigan_settings=sigan_settings,
             )
-            sensor_cal = self.sensor.signal_analyzer.sensor_calibration_data
+            sensor_cal = self.sensor.sensor_calibration_data
             if sensor_cal is not None:
                 if "1db_compression_point" in sensor_cal:
                     sensor_cal["compression_point"] = sensor_cal.pop(
                         "1db_compression_point"
                     )
-                    capture_segment.sensor_calibration = ntia_sensor.Calibration(
-                        **sensor_cal
-                    )
+                if "reference" not in sensor_cal:
+                    # If the calibration data already includes this, don't overwrite
+                    sensor_cal["reference"] = measurement_result["reference"]
+                capture_segment.sensor_calibration = ntia_sensor.Calibration(
+                    **sensor_cal
+                )
             measurement_result["capture_segment"] = capture_segment
 
             self.create_metadata(measurement_result, recording_id)
diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py
index d26bb743..bf1099e7 100644
--- a/scos_actions/actions/calibrate_y_factor.py
+++ b/scos_actions/actions/calibrate_y_factor.py
@@ -228,10 +228,8 @@ def calibrate(self, params: dict):
 
         # Get noise diode on IQ
         logger.debug("Acquiring IQ samples with noise diode ON")
-        noise_on_measurement_result = (
-            self.sensor.signal_analyzer.acquire_time_domain_samples(
-                num_samples, num_samples_skip=nskip, cal_adjust=False
-            )
+        noise_on_measurement_result = self.sensor.acquire_time_domain_samples(
+            num_samples, nskip, cal_adjust=False
         )
         sample_rate = noise_on_measurement_result["sample_rate"]
 
@@ -242,10 +240,8 @@ def calibrate(self, params: dict):
 
         # Get noise diode off IQ
         logger.debug("Acquiring IQ samples with noise diode OFF")
-        noise_off_measurement_result = (
-            self.sensor.signal_analyzer.acquire_time_domain_samples(
-                num_samples, num_samples_skip=nskip, cal_adjust=False
-            )
+        noise_off_measurement_result = self.sensor.acquire_time_domain_samples(
+            num_samples, nskip, cal_adjust=False
         )
         assert (
             sample_rate == noise_off_measurement_result["sample_rate"]
@@ -259,24 +255,22 @@ def calibrate(self, params: dict):
             noise_on_data = sosfilt(self.iir_sos, noise_on_measurement_result["data"])
             noise_off_data = sosfilt(self.iir_sos, noise_off_measurement_result["data"])
         else:
-            if self.sensor.signal_analyzer.sensor_calibration.is_default:
+            if self.sensor.sensor_calibration.is_default:
                 raise Exception(
                     "Calibrations without IIR filter cannot be performed with default calibration."
                 )
 
             logger.debug("Skipping IIR filtering")
             # Get ENBW from sensor calibration
-            assert set(
-                self.sensor.signal_analyzer.sensor_calibration.calibration_parameters
-            ) <= set(
+            assert set(self.sensor.sensor_calibration.calibration_parameters) <= set(
                 sigan_params.keys()
             ), f"Action parameters do not include all required calibration parameters"
             cal_args = [
                 sigan_params[k]
-                for k in self.sensor.signal_analyzer.sensor_calibration.calibration_parameters
+                for k in self.sensor.sensor_calibration.calibration_parameters
             ]
-            self.sensor.signal_analyzer.recompute_sensor_calibration_data(cal_args)
-            enbw_hz = self.sensor.signal_analyzer.sensor_calibration_data["enbw"]
+            self.sensor.recompute_sensor_calibration_data(cal_args)
+            enbw_hz = self.sensor.sensor_calibration_data["enbw"]
             noise_on_data = noise_on_measurement_result["data"]
             noise_off_data = noise_off_measurement_result["data"]
 
@@ -294,7 +288,7 @@ def calibrate(self, params: dict):
         )
 
         # Update sensor calibration with results
-        self.sensor.signal_analyzer.sensor_calibration.update(
+        self.sensor.sensor_calibration.update(
             sigan_params, utils.get_datetime_str_now(), gain, noise_figure, temp_c
         )
 
diff --git a/scos_actions/actions/interfaces/measurement_action.py b/scos_actions/actions/interfaces/measurement_action.py
index d3299ca4..8d16d763 100644
--- a/scos_actions/actions/interfaces/measurement_action.py
+++ b/scos_actions/actions/interfaces/measurement_action.py
@@ -51,7 +51,7 @@ def create_capture_segment(
             overload=overload,
             sigan_settings=sigan_settings,
         )
-        sensor_cal = self.sensor.signal_analyzer.sensor_calibration_data
+        sensor_cal = self.sensor.sensor_calibration_data
         # Rename compression point keys if they exist
         # then set calibration metadata if it exists
         if sensor_cal is not None:
@@ -161,7 +161,7 @@ def acquire_data(
             + f" and {'' if cal_adjust else 'not '}applying gain adjustment based"
             + " on calibration data"
         )
-        measurement_result = self.sensor.signal_analyzer.acquire_time_domain_samples(
+        measurement_result = self.sensor.acquire_time_domain_samples(
             num_samples,
             num_samples_skip=nskip,
             cal_adjust=cal_adjust,
diff --git a/scos_actions/hardware/sensor.py b/scos_actions/hardware/sensor.py
index 91f9a273..cbd3e351 100644
--- a/scos_actions/hardware/sensor.py
+++ b/scos_actions/hardware/sensor.py
@@ -239,11 +239,10 @@ def acquire_time_domain_samples(
         if cal_adjust:
             if self.sensor_calibration is not None:
                 logger.debug("Scaling samples using calibration data")
-                calibrated_gain__db = 0.0
                 self.recompute_sensor_calibration_data()
-                sensor_gain = self.sensor_calibration_data["gain"]
-                logger.debug(f"Using sensor gain: {sensor_gain} dB")
-                calibrated_gain__db += sensor_gain
+                calibrated_gain__db = self.sensor_calibration_data["gain"]
+                calibrated_nf__db = self.sensor_calibration_data["noise_figure"]
+                logger.debug(f"Using sensor gain: {calibrated_gain__db} dB")
                 if self.differential_calibration is not None:
                     # Also apply differential calibration correction
                     # TODO recompute functions match to current signal analyzer
@@ -252,6 +251,7 @@ def acquire_time_domain_samples(
                     differential_loss = self.differential_calibration_data["loss"]
                     logger.debug(f"Using differential loss: {differential_loss} dB")
                     calibrated_gain__db -= differential_loss
+                    calibrated_nf__db += differential_loss
                     measurement_result[
                         "reference"
                     ] = self.differential_calibration.reference_point
@@ -259,9 +259,17 @@ def acquire_time_domain_samples(
                     # No differential calibration exists
                     logger.debug("No differential calibration was applied")
                     measurement_result["reference"] = "calibration terminal"
+
                 linear_gain = 10.0 ** (calibrated_gain__db / 20.0)
-                logger.debug(f"Applying gain of {linear_gain}")
+                logger.debug(f"Applying total gain of {calibrated_gain__db}")
                 measurement_result["data"] /= linear_gain
+
+                # Metadata: record the gain and noise figure based on the actual
+                # scaling which was used.
+                measurement_result["applied_calibration"] = {
+                    "gain": calibrated_gain__db,
+                    "noise_figure": calibrated_nf__db,
+                }
             else:
                 # No sensor calibration exists
                 msg = "Unable to scale samples without sensor calibration data"
@@ -270,5 +278,6 @@ def acquire_time_domain_samples(
         else:
             # Set the data reference in the measurement_result
             measurement_result["reference"] = "signal analyzer input"
+            measurement_result["calibration"] = None
 
         return measurement_result

From eba790a42c81ecf6cd075c7a8cd7b17c3938744d Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Wed, 24 Jan 2024 13:33:34 -0500
Subject: [PATCH 020/102] generalize differential calibration module docstring

---
 .../calibration/differential_calibration.py    | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/scos_actions/calibration/differential_calibration.py b/scos_actions/calibration/differential_calibration.py
index 4cf40322..e25ad71d 100644
--- a/scos_actions/calibration/differential_calibration.py
+++ b/scos_actions/calibration/differential_calibration.py
@@ -1,20 +1,20 @@
 """
 Dataclass implementation for "differential calibration" handling.
 
-A differential calibration provides loss values at different frequencies
-which represent excess loss between the calibration terminal and the antenna
-port. At present, this is measured manually using a calibration probe consisting
-of a calibrated noise source and a programmable attenuator.
+A differential calibration provides loss values which represent excess loss
+between the `SensorCalibration` reference point and another reference point.
+A typical usage would be for calibrating out measured cable losses which exist
+between the antenna and the Y-factor calibration terminal. At present, this is
+measured manually using a calibration probe consisting of a calibrated noise
+source and a programmable attenuator.
 
 The ``reference_point`` top-level key defines the point to which measurements
 are referenced after using the correction factors included in the file.
 
 The ``calibration_data`` entries are expected to include these correction factors,
-with the key name ``"loss"`` and values in decibels (dB). These correction factors
-represent the differential loss between the calibration terminal used by onboard
-``SensorCalibration`` results and the reference point defined by ``reference_point``.
-A positive value of ``"loss"`` indicates a LOSS going FROM ``reference_point`` TO
-the calibration terminal used by the ``SensorCalibration``.
+with the key name ``"loss"`` and values in decibels (dB). A positive value of
+``"loss"`` indicates a LOSS going FROM ``reference_point`` TO the calibration
+terminal used by the ``SensorCalibration``.
 """
 
 from dataclasses import dataclass

From fdea1a7aa875560468a69fcc8b444128461c4f51 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Wed, 24 Jan 2024 18:42:02 -0500
Subject: [PATCH 021/102] make calibration data a sensor property

automatically recompute calibration data in the getter method
---
 scos_actions/hardware/sensor.py | 61 +++++++++++++++++++++------------
 1 file changed, 40 insertions(+), 21 deletions(-)

diff --git a/scos_actions/hardware/sensor.py b/scos_actions/hardware/sensor.py
index cbd3e351..6143b0a5 100644
--- a/scos_actions/hardware/sensor.py
+++ b/scos_actions/hardware/sensor.py
@@ -2,7 +2,7 @@
 import hashlib
 import json
 import logging
-from typing import Dict, Optional
+from typing import Any, Dict, Optional
 
 from its_preselector.preselector import Preselector
 from its_preselector.web_relay import WebRelay
@@ -37,8 +37,9 @@ def __init__(
         self.switches = switches
         self.location = location
         self.capabilities = capabilities
-        self.sensor_calibration_data = {}
+        self._sensor_calibration_data = {}
         self._sensor_calibration = sensor_cal
+        self._differential_calibration_data = {}
         self._differential_calibration = differential_cal
         # There is no setter for start_time property
         self._start_time = datetime.datetime.utcnow()
@@ -134,47 +135,67 @@ def differential_calibration(self, cal: Optional[DifferentialCalibration]):
 
     @property
     def last_calibration_time(self) -> str:
-        """Get a datetime string for the most recent sensor calibration."""
+        """A datetime string for the most recent sensor calibration."""
         return convert_string_to_millisecond_iso_format(
             self.sensor_calibration.last_calibration_datetime
         )
 
+    @property
+    def sensor_calibration_data(self) -> Dict[str, Any]:
+        """Sensor calibration data for the current sensor settings."""
+        self._recompute_sensor_calibration_data()
+        return self._sensor_calibration_data
+
+    @property
+    def differential_calibration_data(self) -> Dict[str, float]:
+        """Differential calibration data for the current sensor settings."""
+        self._recompute_differential_calibration_data()
+        return self._differential_calibration_data
+
     def _get_calibration_args_from_sigan(self, calibration: Calibration) -> list:
         """Get current values of signal analyzer settings which are calibration parameters."""
-        cal_params = [
-            p for p in calibration.calibration_parameters if p in SIGAN_SETTINGS_KEYS
-        ]
-        if set(cal_params) <= set(calibration.calibration_parameters):
+        try:
+            # Get calibration parameters which are valid settings for signal analyzers
+            cal_params = [
+                p
+                for p in calibration.calibration_parameters
+                if p in SIGAN_SETTINGS_KEYS
+            ]
+            if len(cal_params) == 0:
+                err_text = "any"  # Formats error message below
+                raise ValueError
+            else:
+                err_text = "this"  # Formats error message below
+            cal_args = [vars(self.signal_analyzer)[f"_{p}"] for p in cal_params]
+        except Exception as e:
             msg = (
-                "One or more required calibration parameters is not a valid signal "
-                + f"analyzer property.\nRequired parameters: {calibration.calibration_parameters}"
-                + f"\nSignal analyzer properties: {list(vars(self.signal_analyzer).keys())}"
+                f"One or more calibration parameters is not a valid setting for {err_text} "
+                + f"signal analyzer.\nRequired parameters: {calibration.calibration_parameters}"
             )
-            logger.error(msg)
-            raise KeyError
-        cal_args = [vars(self.signal_analyzer)[f"_{p}"] for p in cal_params]
+            logger.exception(msg)
+            raise e
         logger.debug(f"Matched calibration params: {cal_args}")
         return cal_args  # Order matches calibration.calibration_parameters
 
-    def recompute_differential_calibration_data(self) -> None:
+    def _recompute_differential_calibration_data(self) -> None:
         """Set the differential calibration data based on the current tuning."""
-        self.differential_calibration_data = {}
+        self._differential_calibration_data = {}
         if self.differential_calibration is not None:
             cal_args = self._get_calibration_args_from_sigan(
                 self.differential_calibration
             )
-            self.differential_calibration_data.update(
+            self._differential_calibration_data.update(
                 self.differential_calibration.get_calibration_dict(cal_args)
             )
         else:
             logger.warning("Differential calibration does not exist.")
 
-    def recompute_sensor_calibration_data(self) -> None:
+    def _recompute_sensor_calibration_data(self) -> None:
         """Set the sensor calibration data based on the current tuning."""
-        self.sensor_calibration_data = {}
+        self._sensor_calibration_data = {}
         if self.sensor_calibration is not None:
             cal_args = self._get_calibration_args_from_sigan(self.sensor_calibration)
-            self.sensor_calibration_data.update(
+            self._sensor_calibration_data.update(
                 self.sensor_calibration.get_calibration_dict(cal_args)
             )
         else:
@@ -239,7 +260,6 @@ def acquire_time_domain_samples(
         if cal_adjust:
             if self.sensor_calibration is not None:
                 logger.debug("Scaling samples using calibration data")
-                self.recompute_sensor_calibration_data()
                 calibrated_gain__db = self.sensor_calibration_data["gain"]
                 calibrated_nf__db = self.sensor_calibration_data["noise_figure"]
                 logger.debug(f"Using sensor gain: {calibrated_gain__db} dB")
@@ -247,7 +267,6 @@ def acquire_time_domain_samples(
                     # Also apply differential calibration correction
                     # TODO recompute functions match to current signal analyzer
                     # settings. should they use the measurement_result instead?
-                    self.recompute_differential_calibration_data()
                     differential_loss = self.differential_calibration_data["loss"]
                     logger.debug(f"Using differential loss: {differential_loss} dB")
                     calibrated_gain__db -= differential_loss

From 3dfdb522182d4496564a022555b729395ab93ad2 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Thu, 25 Jan 2024 10:14:29 -0500
Subject: [PATCH 022/102] implement mock sensor for testing

---
 .../tests/test_acquire_single_freq_fft.py     |  8 +--
 .../actions/tests/test_monitor_sigan.py       | 10 ++--
 .../tests/test_single_freq_tdomain_iq.py      | 10 ++--
 .../tests/test_stepped_freq_tdomain_iq.py     |  9 +--
 scos_actions/actions/tests/test_sync_gps.py   |  6 +-
 scos_actions/hardware/mocks/mock_sensor.py    | 59 +++++++++++++++++++
 scos_actions/hardware/mocks/mock_sigan.py     | 14 -----
 scos_actions/hardware/tests/test_sensor.py    | 26 ++++++--
 scos_actions/hardware/tests/test_sigan.py     | 12 ++--
 9 files changed, 105 insertions(+), 49 deletions(-)
 create mode 100644 scos_actions/hardware/mocks/mock_sensor.py

diff --git a/scos_actions/actions/tests/test_acquire_single_freq_fft.py b/scos_actions/actions/tests/test_acquire_single_freq_fft.py
index a172f36f..8a4bdf3e 100644
--- a/scos_actions/actions/tests/test_acquire_single_freq_fft.py
+++ b/scos_actions/actions/tests/test_acquire_single_freq_fft.py
@@ -1,7 +1,6 @@
 from scos_actions.actions.tests.utils import check_metadata_fields
 from scos_actions.discover import test_actions as actions
-from scos_actions.hardware.mocks.mock_sigan import MockSignalAnalyzer
-from scos_actions.hardware.sensor import Sensor
+from scos_actions.hardware.mocks.mock_sensor import MockSensor
 from scos_actions.signals import measurement_action_completed
 
 SINGLE_FREQUENCY_FFT_ACQUISITION = {
@@ -29,9 +28,8 @@ def callback(sender, **kwargs):
     measurement_action_completed.connect(callback)
     action = actions["test_single_frequency_m4s_action"]
     assert action.description
-    sensor = Sensor(signal_analyzer=MockSignalAnalyzer())
     action(
-        sensor=sensor,
+        sensor=MockSensor(),
         schedule_entry=SINGLE_FREQUENCY_FFT_ACQUISITION,
         task_id=1,
     )
@@ -89,6 +87,6 @@ def callback(sender, **kwargs):
 def test_num_samples_skip():
     action = actions["test_single_frequency_m4s_action"]
     assert action.description
-    sensor = Sensor(signal_analyzer=MockSignalAnalyzer())
+    sensor = MockSensor()
     action(sensor, SINGLE_FREQUENCY_FFT_ACQUISITION, 1)
     assert action.sensor.signal_analyzer._num_samples_skip == action.parameters["nskip"]
diff --git a/scos_actions/actions/tests/test_monitor_sigan.py b/scos_actions/actions/tests/test_monitor_sigan.py
index 0767ea84..38085efd 100644
--- a/scos_actions/actions/tests/test_monitor_sigan.py
+++ b/scos_actions/actions/tests/test_monitor_sigan.py
@@ -1,6 +1,6 @@
 from scos_actions.discover import test_actions as actions
+from scos_actions.hardware.mocks.mock_sensor import MockSensor
 from scos_actions.hardware.mocks.mock_sigan import MockSignalAnalyzer
-from scos_actions.hardware.sensor import Sensor
 from scos_actions.signals import trigger_api_restart
 
 MONITOR_SIGAN_SCHEDULE = {
@@ -23,10 +23,10 @@ def callback(sender, **kwargs):
     action = actions["test_monitor_sigan"]
     mock_sigan = MockSignalAnalyzer()
     mock_sigan._is_available = False
-    sensor = Sensor(signal_analyzer=mock_sigan)
+    sensor = MockSensor(signal_analyzer=mock_sigan)
     action(sensor, MONITOR_SIGAN_SCHEDULE, 1)
     assert _api_restart_triggered == True  # signal sent
-    mock_sigan._is_available = True
+    sensor.signal_analyzer._is_available = True
 
 
 def test_monitor_sigan_not_healthy():
@@ -40,7 +40,7 @@ def callback(sender, **kwargs):
     action = actions["test_monitor_sigan"]
     mock_sigan = MockSignalAnalyzer()
     mock_sigan.times_to_fail_recv = 6
-    sensor = Sensor(signal_analyzer=mock_sigan)
+    sensor = MockSensor(signal_analyzer=mock_sigan)
     action(sensor, MONITOR_SIGAN_SCHEDULE, 1)
     assert _api_restart_triggered == True  # signal sent
 
@@ -57,6 +57,6 @@ def callback(sender, **kwargs):
     mock_sigan = MockSignalAnalyzer()
     mock_sigan._is_available = True
     mock_sigan.set_times_to_fail_recv(0)
-    sensor = Sensor(signal_analyzer=mock_sigan)
+    sensor = MockSensor(signal_analyzer=mock_sigan)
     action(sensor, MONITOR_SIGAN_SCHEDULE, 1)
     assert _api_restart_triggered == False  # signal not sent
diff --git a/scos_actions/actions/tests/test_single_freq_tdomain_iq.py b/scos_actions/actions/tests/test_single_freq_tdomain_iq.py
index fa5a1ad4..c5d4eacd 100644
--- a/scos_actions/actions/tests/test_single_freq_tdomain_iq.py
+++ b/scos_actions/actions/tests/test_single_freq_tdomain_iq.py
@@ -1,9 +1,8 @@
 import pytest
-
 from scos_actions.actions.tests.utils import check_metadata_fields
 from scos_actions.discover import test_actions as actions
+from scos_actions.hardware.mocks.mock_sensor import MockSensor
 from scos_actions.hardware.mocks.mock_sigan import MockSignalAnalyzer
-from scos_actions.hardware.sensor import Sensor
 from scos_actions.signals import measurement_action_completed
 
 SINGLE_TIMEDOMAIN_IQ_ACQUISITION = {
@@ -31,7 +30,7 @@ def callback(sender, **kwargs):
     measurement_action_completed.connect(callback)
     action = actions["test_single_frequency_iq_action"]
     assert action.description
-    sensor = Sensor(signal_analyzer=MockSignalAnalyzer())
+    sensor = MockSensor()
     action(sensor, SINGLE_TIMEDOMAIN_IQ_ACQUISITION, 1)
     assert _data.any()
     assert _metadata
@@ -62,7 +61,7 @@ def test_required_components():
     action = actions["test_single_frequency_m4s_action"]
     mock_sigan = MockSignalAnalyzer()
     mock_sigan._is_available = False
-    sensor = Sensor(signal_analyzer=mock_sigan)
+    sensor = MockSensor(signal_analyzer=mock_sigan)
     with pytest.raises(RuntimeError):
         action(sensor, SINGLE_TIMEDOMAIN_IQ_ACQUISITION, 1)
     mock_sigan._is_available = True
@@ -71,7 +70,6 @@ def test_required_components():
 def test_num_samples_skip():
     action = actions["test_single_frequency_iq_action"]
     assert action.description
-    mock_sigan = MockSignalAnalyzer()
-    sensor = Sensor(signal_analyzer=mock_sigan)
+    sensor = MockSensor()
     action(sensor, SINGLE_TIMEDOMAIN_IQ_ACQUISITION, 1)
     assert action.sensor.signal_analyzer._num_samples_skip == action.parameters["nskip"]
diff --git a/scos_actions/actions/tests/test_stepped_freq_tdomain_iq.py b/scos_actions/actions/tests/test_stepped_freq_tdomain_iq.py
index 40c44141..d389e024 100644
--- a/scos_actions/actions/tests/test_stepped_freq_tdomain_iq.py
+++ b/scos_actions/actions/tests/test_stepped_freq_tdomain_iq.py
@@ -1,6 +1,5 @@
 from scos_actions.discover import test_actions as actions
-from scos_actions.hardware.mocks.mock_sigan import MockSignalAnalyzer
-from scos_actions.hardware.sensor import Sensor
+from scos_actions.hardware.mocks.mock_sensor import MockSensor
 from scos_actions.signals import measurement_action_completed
 
 SINGLE_TIMEDOMAIN_IQ_MULTI_RECORDING_ACQUISITION = {
@@ -34,8 +33,7 @@ def callback(sender, **kwargs):
     measurement_action_completed.connect(callback)
     action = actions["test_multi_frequency_iq_action"]
     assert action.description
-    mock_sigan = MockSignalAnalyzer()
-    sensor = Sensor(signal_analyzer=mock_sigan)
+    sensor = MockSensor()
     action(sensor, SINGLE_TIMEDOMAIN_IQ_MULTI_RECORDING_ACQUISITION, 1)
     for i in range(_count):
         assert _datas[i].any()
@@ -48,8 +46,7 @@ def callback(sender, **kwargs):
 def test_num_samples_skip():
     action = actions["test_multi_frequency_iq_action"]
     assert action.description
-    mock_sigan = MockSignalAnalyzer()
-    sensor = Sensor(signal_analyzer=mock_sigan)
+    sensor = MockSensor()
     action(sensor, SINGLE_TIMEDOMAIN_IQ_MULTI_RECORDING_ACQUISITION, 1)
     if isinstance(action.parameters["nskip"], list):
         assert (
diff --git a/scos_actions/actions/tests/test_sync_gps.py b/scos_actions/actions/tests/test_sync_gps.py
index 5b8a9311..694c4647 100644
--- a/scos_actions/actions/tests/test_sync_gps.py
+++ b/scos_actions/actions/tests/test_sync_gps.py
@@ -2,10 +2,8 @@
 import sys
 
 import pytest
-
 from scos_actions.discover import test_actions
-from scos_actions.hardware.mocks.mock_gps import MockGPS
-from scos_actions.hardware.sensor import Sensor
+from scos_actions.hardware.mocks.mock_sensor import MockSensor
 from scos_actions.signals import location_action_completed
 
 SYNC_GPS = {
@@ -29,7 +27,7 @@ def callback(sender, **kwargs):
 
     location_action_completed.connect(callback)
     action = test_actions["test_sync_gps"]
-    sensor = Sensor(gps=MockGPS())
+    sensor = MockSensor()
     if sys.platform == "linux":
         action(sensor, SYNC_GPS, 1)
         assert _latitude
diff --git a/scos_actions/hardware/mocks/mock_sensor.py b/scos_actions/hardware/mocks/mock_sensor.py
new file mode 100644
index 00000000..4e432f9f
--- /dev/null
+++ b/scos_actions/hardware/mocks/mock_sensor.py
@@ -0,0 +1,59 @@
+import logging
+
+from scos_actions.hardware.mocks.mock_gps import MockGPS
+from scos_actions.hardware.mocks.mock_sigan import MockSignalAnalyzer
+from scos_actions.hardware.sensor import Sensor
+from scos_actions.utils import get_datetime_str_now
+
+_mock_sensor_cal_data = {
+    "datetime": get_datetime_str_now(),
+    "gain": 0,
+    "enbw": None,
+    "noise_figure": None,
+    "1db_compression_point": None,
+    "temperature": 26.85,
+}
+
+_mock_differential_cal_data = {"loss": 10.0}
+
+_mock_capabilities = {"sensor": {}}
+
+_mock_location = {"x": -999, "y": -999, "z": -999, "description": "Testing"}
+
+logger = logging.getLogger(__name__)
+
+
+class MockSensor(Sensor):
+    def __init__(
+        self,
+        signal_analyzer=MockSignalAnalyzer(),
+        gps=MockGPS(),
+        preselector=None,
+        switches={},
+        location=_mock_location,
+        capabilities=_mock_capabilities,
+        sensor_cal=None,
+        differential_cal=None,
+    ):
+        if (sensor_cal is not None) or (differential_cal is not None):
+            logger.warning(
+                "Calibration object provided to mock sensor will not be used to query calibration data."
+            )
+        super().__init__(
+            signal_analyzer,
+            gps,
+            preselector,
+            switches,
+            location,
+            capabilities,
+            sensor_cal,
+            differential_cal,
+        )
+
+    @property
+    def sensor_calibration_data(self):
+        return _mock_sensor_cal_data
+
+    @property
+    def differential_calibration_data(self):
+        return _mock_differential_cal_data
diff --git a/scos_actions/hardware/mocks/mock_sigan.py b/scos_actions/hardware/mocks/mock_sigan.py
index 03a9e42b..00a31804 100644
--- a/scos_actions/hardware/mocks/mock_sigan.py
+++ b/scos_actions/hardware/mocks/mock_sigan.py
@@ -31,15 +31,6 @@ def __init__(
         randomize_values: bool = False,
     ):
         super().__init__(switches)
-        # Define the default calibration dict
-        self.DEFAULT_SENSOR_CALIBRATION = {
-            "datetime": get_datetime_str_now(),
-            "gain": 0,  # Defaults to sigan gain
-            "enbw": None,  # Defaults to sigan enbw
-            "noise_figure": None,  # Defaults to sigan noise figure
-            "1db_compression_point": None,  # Defaults to sigan compression + preselector gain
-            "temperature": 26.85,
-        }
         self.auto_dc_offset = False
         self._frequency = 700e6
         self._sample_rate = 10e6
@@ -61,7 +52,6 @@ def __init__(
         self.times_failed_recv = 0
 
         self.randomize_values = randomize_values
-        self.sensor_calibration_data = self.DEFAULT_SENSOR_CALIBRATION
 
     @property
     def is_available(self):
@@ -188,7 +178,3 @@ def acquire_time_domain_samples(
     def set_times_to_fail_recv(self, n):
         self.times_to_fail_recv = n
         self.times_failed_recv = 0
-
-    @property
-    def last_calibration_time(self):
-        return get_datetime_str_now()
diff --git a/scos_actions/hardware/tests/test_sensor.py b/scos_actions/hardware/tests/test_sensor.py
index 1d86de81..25e66ebb 100644
--- a/scos_actions/hardware/tests/test_sensor.py
+++ b/scos_actions/hardware/tests/test_sensor.py
@@ -1,10 +1,26 @@
-from scos_actions.hardware.mocks.mock_sigan import MockSignalAnalyzer
-from scos_actions.hardware.mocks.mock_gps import MockGPS
-from scos_actions.hardware.sensor import Sensor
+import datetime
 
+from scos_actions.hardware.mocks.mock_sensor import (
+    MockSensor,
+    _mock_capabilities,
+    _mock_differential_cal_data,
+    _mock_location,
+    _mock_sensor_cal_data,
+)
 
-def test_sensor():
-    sensor = Sensor(signal_analyzer=MockSignalAnalyzer(), gps=MockGPS())
+
+def test_mock_sensor():
+    sensor = MockSensor()
     assert sensor is not None
     assert sensor.signal_analyzer is not None
     assert sensor.gps is not None
+    assert sensor.preselector is None
+    assert sensor.switches == {}
+    assert sensor.location == _mock_location
+    assert sensor.capabilities == _mock_capabilities
+    assert sensor.sensor_calibration is None
+    assert sensor.differential_calibration is None
+    assert sensor.has_configurable_preselector is False
+    assert sensor.sensor_calibration_data == _mock_sensor_cal_data
+    assert sensor.differential_calibration_data == _mock_differential_cal_data
+    assert isinstance(sensor.start_time, datetime.datetime)
diff --git a/scos_actions/hardware/tests/test_sigan.py b/scos_actions/hardware/tests/test_sigan.py
index c82ffeec..53081aa0 100644
--- a/scos_actions/hardware/tests/test_sigan.py
+++ b/scos_actions/hardware/tests/test_sigan.py
@@ -1,8 +1,12 @@
+import pytest
 from scos_actions.hardware.mocks.mock_sigan import MockSignalAnalyzer
 
 
-def test_sigan_default_cal():
+def test_mock_sigan():
     sigan = MockSignalAnalyzer()
-    sigan.recompute_sensor_calibration_data([])
-    sensor_cal = sigan.sensor_calibration_data
-    assert sensor_cal["gain"] == 0
+    # Test default values are available as properties
+    assert sigan.frequency == sigan._frequency
+    assert sigan.sample_rate == sigan._sample_rate
+    assert sigan.gain == sigan._gain
+    assert sigan.attenuation == sigan._attenuation
+    assert sigan.preamp_enable == sigan._preamp_enable

From 89e5ae7df54ad5291fe3195a85fe553e43c85159 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Thu, 25 Jan 2024 10:38:43 -0500
Subject: [PATCH 023/102] simplify and update mock sigan

- Remove outdated and unnecessary instance variables
- Remove retry logic from acquire_samples to account for changes in the Sensor object
- Remove redundant properties that are the same as the base class
---
 scos_actions/hardware/mocks/mock_sigan.py | 90 +++++++++--------------
 1 file changed, 35 insertions(+), 55 deletions(-)

diff --git a/scos_actions/hardware/mocks/mock_sigan.py b/scos_actions/hardware/mocks/mock_sigan.py
index 00a31804..4b678a6e 100644
--- a/scos_actions/hardware/mocks/mock_sigan.py
+++ b/scos_actions/hardware/mocks/mock_sigan.py
@@ -31,7 +31,7 @@ def __init__(
         randomize_values: bool = False,
     ):
         super().__init__(switches)
-        self.auto_dc_offset = False
+        self._model = "Mock Signal Analyzer"
         self._frequency = 700e6
         self._sample_rate = 10e6
         self.clock_rate = 40e6
@@ -39,8 +39,6 @@ def __init__(
         self._attenuation = 0
         self._preamp_enable = False
         self._reference_level = -30
-        self._overload = False
-        self._capture_time = None
         self._is_available = True
         self._plugin_version = SCOS_ACTIONS_VERSION
         self._firmware_version = "1.2.3"
@@ -61,14 +59,6 @@ def is_available(self):
     def plugin_version(self):
         return self._plugin_version
 
-    @property
-    def firmware_version(self):
-        return self._firmware_version
-
-    @property
-    def api_version(self):
-        return self._api_version
-
     @property
     def sample_rate(self):
         return self._sample_rate
@@ -124,56 +114,46 @@ def connect(self):
         pass
 
     def acquire_time_domain_samples(
-        self, num_samples: int, num_samples_skip: int = 0, retries: int = 5
+        self, num_samples: int, num_samples_skip: int = 0
     ) -> dict:
         logger.warning("Using mock signal analyzer!")
-        self.sigan_overload = False
-        self._capture_time = None
-        self._num_samples_skip = num_samples_skip
+        overload = False
+        capture_time = None
 
         # Try to acquire the samples
-        max_retries = retries
         data = []
-        while True:
-            if self.times_failed_recv < self.times_to_fail_recv:
-                self.times_failed_recv += 1
-                data = np.ones(0, dtype=np.complex64)
-            else:
-                self._capture_time = get_datetime_str_now()
-                if self.randomize_values:
-                    i = np.random.normal(0.5, 0.5, num_samples)
-                    q = np.random.normal(0.5, 0.5, num_samples)
-                    rand_iq = np.empty(num_samples, dtype=np.complex64)
-                    rand_iq.real = i
-                    rand_iq.imag = q
-                    data = rand_iq
-                else:
-                    data = np.ones(num_samples, dtype=np.complex64)
-
-            data_len = len(data)
-            if not len(data) == num_samples:
-                if retries > 0:
-                    msg = "Signal analyzer error: requested {} samples, but got {}."
-                    logger.warning(msg.format(num_samples + num_samples_skip, data_len))
-                    logger.warning(f"Retrying {retries} more times.")
-                    retries = retries - 1
-                else:
-                    err = "Failed to acquire correct number of samples "
-                    err += f"{max_retries} times in a row."
-                    raise RuntimeError(err)
+        if self.times_failed_recv < self.times_to_fail_recv:
+            self.times_failed_recv += 1
+            data = np.ones(0, dtype=np.complex64)
+        else:
+            capture_time = get_datetime_str_now()
+            if self.randomize_values:
+                i = np.random.normal(0.5, 0.5, num_samples)
+                q = np.random.normal(0.5, 0.5, num_samples)
+                rand_iq = np.empty(num_samples, dtype=np.complex64)
+                rand_iq.real = i
+                rand_iq.imag = q
+                data = rand_iq
             else:
-                logger.debug(f"Successfully acquired {num_samples} samples.")
-                return {
-                    "data": data,
-                    "overload": self._overload,
-                    "frequency": self._frequency,
-                    "gain": self._gain,
-                    "attenuation": self._attenuation,
-                    "preamp_enable": self._preamp_enable,
-                    "reference_level": self._reference_level,
-                    "sample_rate": self._sample_rate,
-                    "capture_time": self._capture_time,
-                }
+                data = np.ones(num_samples, dtype=np.complex64)
+
+        if (data_len := len(data)) != num_samples:
+            err = "Failed to acquire correct number of samples: "
+            err += f"got {data_len} instead of {num_samples}"
+            raise RuntimeError(err)
+        else:
+            logger.debug(f"Successfully acquired {num_samples} samples.")
+            return {
+                "data": data,
+                "overload": overload,
+                "frequency": self._frequency,
+                "gain": self._gain,
+                "attenuation": self._attenuation,
+                "preamp_enable": self._preamp_enable,
+                "reference_level": self._reference_level,
+                "sample_rate": self._sample_rate,
+                "capture_time": capture_time,
+            }
 
     def set_times_to_fail_recv(self, n):
         self.times_to_fail_recv = n

From 59b87cacdb6cd94529fe473590122d1d483278c6 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Thu, 25 Jan 2024 10:39:41 -0500
Subject: [PATCH 024/102] Make firmware and API version properties usable

These properties no longer need to be overriden. Subclass constructors can just set the correct instance variables instead.
---
 scos_actions/hardware/sigan_iface.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scos_actions/hardware/sigan_iface.py b/scos_actions/hardware/sigan_iface.py
index f7fb0bed..cb0e85cb 100644
--- a/scos_actions/hardware/sigan_iface.py
+++ b/scos_actions/hardware/sigan_iface.py
@@ -26,6 +26,8 @@ def __init__(
         switches: Optional[Dict[str, WebRelay]] = None,
     ):
         self._model = "Unknown"
+        self._api_version = "Unknown"
+        self._firmware_version = "Unknown"
         self.switches = switches
 
     @property
@@ -43,12 +45,12 @@ def plugin_version(self) -> str:
     @property
     def firmware_version(self) -> str:
         """Returns the version of the signal analyzer firmware."""
-        return "Unknown"
+        return self._firmware_version
 
     @property
     def api_version(self) -> str:
         """Returns the version of the underlying signal analyzer API."""
-        return "Unknown"
+        return self._api_version
 
     @abstractmethod
     def acquire_time_domain_samples(

From e75619561db495ab178ce14e9ce2d8aa32d0c03e Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Thu, 25 Jan 2024 12:06:12 -0500
Subject: [PATCH 025/102] Update test_sigan.py

---
 scos_actions/hardware/tests/test_sigan.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/scos_actions/hardware/tests/test_sigan.py b/scos_actions/hardware/tests/test_sigan.py
index 53081aa0..d4434d78 100644
--- a/scos_actions/hardware/tests/test_sigan.py
+++ b/scos_actions/hardware/tests/test_sigan.py
@@ -5,8 +5,14 @@
 def test_mock_sigan():
     sigan = MockSignalAnalyzer()
     # Test default values are available as properties
+    assert sigan.model == sigan._model
     assert sigan.frequency == sigan._frequency
     assert sigan.sample_rate == sigan._sample_rate
     assert sigan.gain == sigan._gain
     assert sigan.attenuation == sigan._attenuation
     assert sigan.preamp_enable == sigan._preamp_enable
+    assert sigan.reference_level == sigan._reference_level
+    assert sigan.is_available == sigan._is_available
+    assert sigan.plugin_version == sigan._plugin_version
+    assert sigan.firmware_version == sigan._firmware_version
+    assert sigan.api_version == sigan._api_version

From f1b1a633e0bf8eaa754e940efd5a15fd89d80b5c Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Thu, 25 Jan 2024 18:37:35 -0500
Subject: [PATCH 026/102] generalize matching of calibration params

- simplify recompute_calibration methods
- cal_params is now an argument of acquire_time_domain_samples, and is intended to be used by passing action.parameters dicts which contain the required key/values to match calibration data to the measurement
- add some specific calibration-related exceptions
- update actions and unit tests
---
 .../actions/acquire_sea_data_product.py       | 12 +--
 .../actions/acquire_single_freq_fft.py        |  2 +-
 .../actions/acquire_single_freq_tdomain_iq.py |  4 +-
 .../acquire_stepped_freq_tdomain_iq.py        |  4 +-
 .../actions/interfaces/measurement_action.py  |  7 +-
 .../tests/test_acquire_single_freq_fft.py     |  8 --
 .../tests/test_single_freq_tdomain_iq.py      |  8 --
 .../tests/test_stepped_freq_tdomain_iq.py     | 17 ----
 .../calibration/interfaces/calibration.py     | 73 +++++-------------
 .../calibration/sensor_calibration.py         | 22 +++++-
 .../calibration/tests/test_calibration.py     | 25 +-----
 .../tests/test_sensor_calibration.py          | 12 ++-
 scos_actions/calibration/utils.py             | 25 +++++-
 scos_actions/hardware/sensor.py               | 77 +++++++------------
 14 files changed, 118 insertions(+), 178 deletions(-)

diff --git a/scos_actions/actions/acquire_sea_data_product.py b/scos_actions/actions/acquire_sea_data_product.py
index 003db3b8..fcadb459 100644
--- a/scos_actions/actions/acquire_sea_data_product.py
+++ b/scos_actions/actions/acquire_sea_data_product.py
@@ -626,13 +626,11 @@ def capture_iq(self, params: dict) -> dict:
         nskip = utils.get_parameter(NUM_SKIP, params)
         num_samples = int(params[SAMPLE_RATE] * duration_ms * 1e-3)
         # Collect IQ data
-        measurement_result = self.sensor.acquire_time_domain_samples(num_samples, nskip)
+        measurement_result = self.sensor.acquire_time_domain_samples(
+            num_samples, nskip, cal_params=params
+        )
         # Store some metadata with the IQ
         measurement_result.update(params)
-        measurement_result["sensor_cal"] = self.sensor.sensor_calibration_data
-        measurement_result[
-            "differential_cal"
-        ] = self.sensor.differential_calibration_data
         toc = perf_counter()
         logger.debug(
             f"IQ Capture ({duration_ms} ms @ {(params[FREQUENCY]/1e6):.1f} MHz) completed in {toc-tic:.2f} s."
@@ -1126,7 +1124,9 @@ def create_capture_segment(
                 noise_figure=round(
                     measurement_result["applied_calibration"]["noise_figure"], 3
                 ),
-                temperature=round(measurement_result["sensor_cal"]["temperature"], 1),
+                temperature=round(
+                    self.sensor.sensor_calibration_data["temperature"], 1
+                ),
                 reference=measurement_result["reference"],
             ),
             sigan_settings=ntia_sensor.SiganSettings(
diff --git a/scos_actions/actions/acquire_single_freq_fft.py b/scos_actions/actions/acquire_single_freq_fft.py
index a54cb785..21bc8f5e 100644
--- a/scos_actions/actions/acquire_single_freq_fft.py
+++ b/scos_actions/actions/acquire_single_freq_fft.py
@@ -164,7 +164,7 @@ def __init__(self, parameters: dict):
     def execute(self, schedule_entry: dict, task_id: int) -> dict:
         # Acquire IQ data and generate M4S result
         measurement_result = self.acquire_data(
-            self.num_samples, self.nskip, self.cal_adjust
+            self.num_samples, self.nskip, self.cal_adjust, cal_params=self.parameters
         )
         # Actual sample rate may differ from configured value
         sample_rate_Hz = measurement_result["sample_rate"]
diff --git a/scos_actions/actions/acquire_single_freq_tdomain_iq.py b/scos_actions/actions/acquire_single_freq_tdomain_iq.py
index a2261fbf..6ff6b537 100644
--- a/scos_actions/actions/acquire_single_freq_tdomain_iq.py
+++ b/scos_actions/actions/acquire_single_freq_tdomain_iq.py
@@ -84,7 +84,9 @@ def execute(self, schedule_entry: dict, task_id: int) -> dict:
         # Use the sigan's actual reported instead of requested sample rate
         sample_rate = self.sensor.signal_analyzer.sample_rate
         num_samples = int(sample_rate * self.duration_ms * 1e-3)
-        measurement_result = self.acquire_data(num_samples, self.nskip, self.cal_adjust)
+        measurement_result = self.acquire_data(
+            num_samples, self.nskip, self.cal_adjust, cal_params=self.parameters
+        )
         end_time = utils.get_datetime_str_now()
         measurement_result.update(self.parameters)
         measurement_result["end_time"] = end_time
diff --git a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
index 72995f14..7278e5f1 100644
--- a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
+++ b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
@@ -100,7 +100,9 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
             cal_adjust = get_parameter(CAL_ADJUST, measurement_params)
             sample_rate = self.sensor.signal_analyzer.sample_rate
             num_samples = int(sample_rate * duration_ms * 1e-3)
-            measurement_result = super().acquire_data(num_samples, nskip, cal_adjust)
+            measurement_result = super().acquire_data(
+                num_samples, nskip, cal_adjust, cal_params=measurement_params
+            )
             measurement_result.update(measurement_params)
             end_time = utils.get_datetime_str_now()
             measurement_result["end_time"] = end_time
diff --git a/scos_actions/actions/interfaces/measurement_action.py b/scos_actions/actions/interfaces/measurement_action.py
index 8d16d763..fe5e197c 100644
--- a/scos_actions/actions/interfaces/measurement_action.py
+++ b/scos_actions/actions/interfaces/measurement_action.py
@@ -154,7 +154,11 @@ def send_signals(self, task_id, metadata, measurement_data):
         )
 
     def acquire_data(
-        self, num_samples: int, nskip: int = 0, cal_adjust: bool = True
+        self,
+        num_samples: int,
+        nskip: int = 0,
+        cal_adjust: bool = True,
+        cal_params: Optional[dict] = None,
     ) -> dict:
         logger.debug(
             f"Acquiring {num_samples} IQ samples, skipping the first {nskip} samples"
@@ -165,6 +169,7 @@ def acquire_data(
             num_samples,
             num_samples_skip=nskip,
             cal_adjust=cal_adjust,
+            cal_params=cal_params,
         )
 
         return measurement_result
diff --git a/scos_actions/actions/tests/test_acquire_single_freq_fft.py b/scos_actions/actions/tests/test_acquire_single_freq_fft.py
index 8a4bdf3e..e4d9c9fb 100644
--- a/scos_actions/actions/tests/test_acquire_single_freq_fft.py
+++ b/scos_actions/actions/tests/test_acquire_single_freq_fft.py
@@ -82,11 +82,3 @@ def callback(sender, **kwargs):
             ]
         ]
     )
-
-
-def test_num_samples_skip():
-    action = actions["test_single_frequency_m4s_action"]
-    assert action.description
-    sensor = MockSensor()
-    action(sensor, SINGLE_FREQUENCY_FFT_ACQUISITION, 1)
-    assert action.sensor.signal_analyzer._num_samples_skip == action.parameters["nskip"]
diff --git a/scos_actions/actions/tests/test_single_freq_tdomain_iq.py b/scos_actions/actions/tests/test_single_freq_tdomain_iq.py
index c5d4eacd..50c36b89 100644
--- a/scos_actions/actions/tests/test_single_freq_tdomain_iq.py
+++ b/scos_actions/actions/tests/test_single_freq_tdomain_iq.py
@@ -65,11 +65,3 @@ def test_required_components():
     with pytest.raises(RuntimeError):
         action(sensor, SINGLE_TIMEDOMAIN_IQ_ACQUISITION, 1)
     mock_sigan._is_available = True
-
-
-def test_num_samples_skip():
-    action = actions["test_single_frequency_iq_action"]
-    assert action.description
-    sensor = MockSensor()
-    action(sensor, SINGLE_TIMEDOMAIN_IQ_ACQUISITION, 1)
-    assert action.sensor.signal_analyzer._num_samples_skip == action.parameters["nskip"]
diff --git a/scos_actions/actions/tests/test_stepped_freq_tdomain_iq.py b/scos_actions/actions/tests/test_stepped_freq_tdomain_iq.py
index d389e024..2183d0bf 100644
--- a/scos_actions/actions/tests/test_stepped_freq_tdomain_iq.py
+++ b/scos_actions/actions/tests/test_stepped_freq_tdomain_iq.py
@@ -41,20 +41,3 @@ def callback(sender, **kwargs):
         assert _task_ids[i] == 1
         assert _recording_ids[i] == i + 1
     assert _count == 10
-
-
-def test_num_samples_skip():
-    action = actions["test_multi_frequency_iq_action"]
-    assert action.description
-    sensor = MockSensor()
-    action(sensor, SINGLE_TIMEDOMAIN_IQ_MULTI_RECORDING_ACQUISITION, 1)
-    if isinstance(action.parameters["nskip"], list):
-        assert (
-            action.sensor.signal_analyzer._num_samples_skip
-            == action.parameters["nskip"][-1]
-        )
-    else:
-        assert (
-            action.sensor.signal_analyzer._num_samples_skip
-            == action.parameters["nskip"]
-        )
diff --git a/scos_actions/calibration/interfaces/calibration.py b/scos_actions/calibration/interfaces/calibration.py
index 406e325e..42277253 100644
--- a/scos_actions/calibration/interfaces/calibration.py
+++ b/scos_actions/calibration/interfaces/calibration.py
@@ -3,9 +3,12 @@
 import logging
 from abc import abstractmethod
 from pathlib import Path
-from typing import Any, List, get_origin
+from typing import List, get_origin
 
-from scos_actions.calibration.utils import filter_by_parameter
+from scos_actions.calibration.utils import (
+    CalibrationParametersMissingException,
+    filter_by_parameter,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -38,65 +41,29 @@ def _validate_fields(self) -> None:
                     f"{c_name} field {f_name} must be {f_type}, not {actual_type}"
                 )
 
-    def get_calibration_dict(self, cal_params: List[Any]) -> dict:
+    def get_calibration_dict(self, params: dict) -> dict:
         """
-        Get calibration data closest to the specified parameter values.
-
-        :param cal_params: List of calibration parameter values. For example,
-            if ``calibration_parameters`` are ``["sample_rate", "gain"]``,
-            then the input to this method could be ``["15360000.0", "40"]``.
-        :return: The calibration data corresponding to the input parameter values.
-        """
-        cal_data = self.calibration_data
-        for i, setting_value in enumerate(cal_params):
-            setting = self.calibration_parameters[i]
-            logger.debug(f"Looking up calibration for {setting} at {setting_value}")
-            cal_data = filter_by_parameter(cal_data, setting_value)
-        logger.debug(f"Got calibration data: {cal_data}")
-
-        return cal_data
-
-    def _retrieve_data_to_update(self, params: dict) -> dict:
-        """
-        Locate the calibration data entry to update, based on a set
-        of calibration parameters.
+        Get calibration data entry at the specified parameter values.
 
         :param params: Parameters used for calibration. This must include
             entries for all of the ``Calibration.calibration_parameters``
             Example: ``{"sample_rate": 14000000.0, "attenuation": 10.0}``
-        :return: A dict containing the existing calibration entry at
-            the specified parameter set, which may be empty if none exists.
-        :raises ValueError: If not all calibration parameters exist as keys
-            in ``params``.
+        :return: The calibration data corresponding to the input parameter values.
         """
-        # Use params keys as calibration_parameters if none exist
-        if len(self.calibration_parameters) == 0:
-            logger.warning(
-                f"Setting required calibration parameters to {list(params.keys())}"
-            )
-            self.calibration_parameters = list(params.keys())
-        elif not set(params.keys()) >= set(self.calibration_parameters):
-            # Otherwise ensure all required parameters were used
-            raise ValueError(
-                "Not enough parameters specified to update calibration.\n"
-                + f"Required parameters are {self.calibration_parameters}"
+        # Check that input includes all required calibration parameters
+        if not set(params.keys()) >= set(self.calibration_parameters):
+            raise CalibrationParametersMissingException(
+                params, self.calibration_parameters
             )
+        cal_data = self.calibration_data
+        for p_name in self.calibration_parameters:
+            p_value = params[p_name]
+            logger.debug(f"Looking up calibration data at {p_name}={p_value}")
+            cal_data = filter_by_parameter(cal_data, p_value)
 
-        # Retrieve the existing calibration data entry based on
-        # the provided parameters and their values
-        data_entry = self.calibration_data
-        for parameter in self.calibration_parameters:
-            value = str(params[parameter]).lower()
-            logger.debug(f"Updating calibration at {parameter} = {value}")
-            try:
-                data_entry = data_entry[value]
-            except KeyError:
-                logger.debug(
-                    f"Creating required calibration data field for {parameter} = {value}"
-                )
-                data_entry[value] = {}
-                data_entry = data_entry[value]
-        return data_entry
+        logger.debug(f"Got calibration data: {cal_data}")
+
+        return cal_data
 
     @abstractmethod
     def update(self):
diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index 12997f22..de547fc9 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -3,6 +3,7 @@
 from typing import Dict, List, Union
 
 from scos_actions.calibration.interfaces.calibration import Calibration
+from scos_actions.calibration.utils import CalibrationEntryMissingException
 
 logger = logging.getLogger(__name__)
 
@@ -45,8 +46,25 @@ def update(
         :param temp_degC: Temperature at calibration time, in degrees Celsius.
         :param file_path: File path for saving the updated calibration data.
         """
-        # Get existing calibration data entry which will be updated
-        data_entry = self._retrieve_data_to_update(params)
+        try:
+            # Get existing calibration data entry which will be updated
+            data_entry = self.get_calibration_dict(params)
+        except CalibrationEntryMissingException:
+            # Existing entry does not exist for these parameters. Make one.
+            data_entry = self.calibration_data
+            for p_name in self.calibration_parameters:
+                p_val = params[p_name]
+                try:
+                    data_entry = data_entry[p_val]
+                except KeyError:
+                    logger.debug(
+                        f"Creating calibration data field for {p_name}={p_val}"
+                    )
+                    data_entry[p_val] = {}
+                    data_entry = data_entry[p_val]
+        except Exception as e:
+            logger.exception("Failed to update calibration data.")
+            raise e
 
         # Update last calibration datetime
         self.last_calibration_datetime = calibration_datetime_str
diff --git a/scos_actions/calibration/tests/test_calibration.py b/scos_actions/calibration/tests/test_calibration.py
index a335d86f..9fb181e4 100644
--- a/scos_actions/calibration/tests/test_calibration.py
+++ b/scos_actions/calibration/tests/test_calibration.py
@@ -16,7 +16,7 @@ def setup_calibration_file(self, tmp_path: Path):
         """Create a dummy calibration file in the pytest temp directory."""
         # Create some dummy calibration data
         self.cal_params = ["frequency", "gain"]
-        self.frequencies = [3555e9, 3565e9, 3575e9]
+        self.frequencies = [3555e6, 3565e6, 3575e6]
         self.gains = [10.0, 20.0, 30.0]
         cal_data = {}
         for frequency in self.frequencies:
@@ -75,31 +75,10 @@ def test_field_validator(self):
     def test_get_calibration_dict(self):
         """Check the get_calibration_dict method with all dummy data."""
         for f in self.frequencies:
-            assert json.loads(
-                json.dumps(self.cal_data[f])
-            ) == self.sample_cal.get_calibration_dict([f])
             for g in self.gains:
                 assert json.loads(
                     json.dumps(self.cal_data[f][g])
-                ) == self.sample_cal.get_calibration_dict([f, g])
-
-    def test_retrieve_data_to_update(self):
-        """Check that the calibration data entry is correctly returned."""
-        for f in self.frequencies:
-            for g in self.gains:
-                params = {"frequency": f, "gain": g}
-                # Use the "is" keyword since this must not be a copy/identical dict
-                assert self.sample_cal.calibration_data[str(f)][
-                    str(g)
-                ] is self.sample_cal._retrieve_data_to_update(params)
-        # Method should work with len=0 calibration parameters
-        test_cal = Calibration([], {}, False, Path(""))
-        _ = test_cal._retrieve_data_to_update({"frequency": 3555e9, "gain": 10.0})
-        # And should fail if calibration parameters are not all supplied
-        with pytest.raises(ValueError):
-            _ = self.sample_cal._retrieve_data_to_update(
-                {"frequency": self.frequencies[0]}
-            )
+                ) == self.sample_cal.get_calibration_dict({"frequency": f, "gain": g})
 
     def test_to_and_from_json(self, tmp_path: Path):
         """Test the ``from_json`` factory method."""
diff --git a/scos_actions/calibration/tests/test_sensor_calibration.py b/scos_actions/calibration/tests/test_sensor_calibration.py
index 6348c409..e9176718 100644
--- a/scos_actions/calibration/tests/test_sensor_calibration.py
+++ b/scos_actions/calibration/tests/test_sensor_calibration.py
@@ -61,7 +61,9 @@ def run_pytest_point(self, sr, f, g, reason, sr_m=False, f_m=False, g_m=False):
         calc_gain_sigan = easy_gain(sr_m, f_m, g_m)
 
         # Get the scale factor from the algorithm
-        interp_cal_data = self.sample_cal.get_calibration_dict([sr, f, g])
+        interp_cal_data = self.sample_cal.get_calibration_dict(
+            {"sample_rate": sr, "frequency": f, "gain": g}
+        )
         interp_gain_siggan = interp_cal_data["gain"]
 
         # Save the point so we don't duplicate
@@ -91,7 +93,9 @@ def run_pytest_point(self, sr, f, g, reason, sr_m=False, f_m=False, g_m=False):
             )
         )
         if not isclose(calc_gain_sigan, interp_gain_siggan, abs_tol=tolerance):
-            interp_cal_data = self.sample_cal.get_calibration_dict([sr, f, g])
+            interp_cal_data = self.sample_cal.get_calibration_dict(
+                {"sample_rate": sr, "frequency": f, "gain": g}
+            )
 
         assert isclose(calc_gain_sigan, interp_gain_siggan, abs_tol=tolerance), msg
         return True
@@ -247,7 +251,7 @@ def test_get_calibration_dict_exact_match_lookup(self):
             clock_rate_lookup_by_sample_rate=[],
             sensor_uid="TESTING",
         )
-        cal_data = cal.get_calibration_dict([100.0, 200.0])
+        cal_data = cal.get_calibration_dict({"sample_rate": 100.0, "frequency": 200.0})
         assert cal_data["NF"] == "NF at 100, 200"
 
     def test_get_calibration_dict_within_range(self):
@@ -267,7 +271,7 @@ def test_get_calibration_dict_within_range(self):
             sensor_uid="TESTING",
         )
         with pytest.raises(CalibrationException) as e_info:
-            _ = cal.get_calibration_dict([100.0, 250.0])
+            _ = cal.get_calibration_dict({"sample_rate": 100.0, "frequency": 250.0})
         assert e_info.value.args[0] == (
             f"Could not locate calibration data at 250.0"
             + f"\nAttempted lookup using key '250.0'"
diff --git a/scos_actions/calibration/utils.py b/scos_actions/calibration/utils.py
index b3aebdeb..6ddba00b 100644
--- a/scos_actions/calibration/utils.py
+++ b/scos_actions/calibration/utils.py
@@ -8,6 +8,25 @@ def __init__(self, msg):
         super().__init__(msg)
 
 
+class CalibrationEntryMissingException(CalibrationException):
+    """Raised when filter_by_parameter cannot locate calibration data."""
+
+    def __init__(self, msg):
+        super().__init__(msg)
+
+
+class CalibrationParametersMissingException(CalibrationException):
+    """Raised when a dictionary does not contain all calibration parameters as keys."""
+
+    def __init__(self, provided_dict: dict, required_keys: list):
+        msg = (
+            "Missing required parameters to lookup calibration data.\n"
+            + f"Required parameters are {required_keys}\n"
+            + f"Provided parameters are {list(provided_dict.keys())}"
+        )
+        super().__init__(msg)
+
+
 def filter_by_parameter(calibrations: dict, value: Union[float, int, bool]) -> dict:
     """
     Select a certain element by the value of a top-level key in a dictionary.
@@ -42,16 +61,16 @@ def filter_by_parameter(calibrations: dict, value: Union[float, int, bool]) -> d
             raise KeyError
         else:
             return filtered_data
-    except AttributeError as e:
+    except AttributeError:
         # calibrations does not have ".get()"
         # Generally means that calibrations is None or not a dict
         msg = f"Provided calibration data is not a dict: {calibrations}"
         raise CalibrationException(msg)
-    except KeyError as e:
+    except KeyError:
         msg = (
             f"Could not locate calibration data at {value}"
             + f"\nAttempted lookup using key '{str(value).lower()}'"
             + f"{f'and {float(value)}' if isinstance(value, int) else ''}"
             + f"\nUsing calibration data: {calibrations}"
         )
-        raise CalibrationException(msg)
+        raise CalibrationEntryMissingException(msg)
diff --git a/scos_actions/hardware/sensor.py b/scos_actions/hardware/sensor.py
index 6143b0a5..94aca3c3 100644
--- a/scos_actions/hardware/sensor.py
+++ b/scos_actions/hardware/sensor.py
@@ -2,7 +2,7 @@
 import hashlib
 import json
 import logging
-from typing import Any, Dict, Optional
+from typing import Any, Dict, List, Optional
 
 from its_preselector.preselector import Preselector
 from its_preselector.web_relay import WebRelay
@@ -10,10 +10,7 @@
 from scos_actions.calibration.interfaces.calibration import Calibration
 from scos_actions.calibration.sensor_calibration import SensorCalibration
 from scos_actions.hardware.gps_iface import GPSInterface
-from scos_actions.hardware.sigan_iface import (
-    SIGAN_SETTINGS_KEYS,
-    SignalAnalyzerInterface,
-)
+from scos_actions.hardware.sigan_iface import SignalAnalyzerInterface
 from scos_actions.utils import convert_string_to_millisecond_iso_format
 
 logger = logging.getLogger(__name__)
@@ -143,63 +140,37 @@ def last_calibration_time(self) -> str:
     @property
     def sensor_calibration_data(self) -> Dict[str, Any]:
         """Sensor calibration data for the current sensor settings."""
-        self._recompute_sensor_calibration_data()
         return self._sensor_calibration_data
 
     @property
     def differential_calibration_data(self) -> Dict[str, float]:
         """Differential calibration data for the current sensor settings."""
-        self._recompute_differential_calibration_data()
         return self._differential_calibration_data
 
-    def _get_calibration_args_from_sigan(self, calibration: Calibration) -> list:
-        """Get current values of signal analyzer settings which are calibration parameters."""
-        try:
-            # Get calibration parameters which are valid settings for signal analyzers
-            cal_params = [
-                p
-                for p in calibration.calibration_parameters
-                if p in SIGAN_SETTINGS_KEYS
-            ]
-            if len(cal_params) == 0:
-                err_text = "any"  # Formats error message below
-                raise ValueError
-            else:
-                err_text = "this"  # Formats error message below
-            cal_args = [vars(self.signal_analyzer)[f"_{p}"] for p in cal_params]
-        except Exception as e:
-            msg = (
-                f"One or more calibration parameters is not a valid setting for {err_text} "
-                + f"signal analyzer.\nRequired parameters: {calibration.calibration_parameters}"
-            )
-            logger.exception(msg)
-            raise e
-        logger.debug(f"Matched calibration params: {cal_args}")
-        return cal_args  # Order matches calibration.calibration_parameters
-
-    def _recompute_differential_calibration_data(self) -> None:
-        """Set the differential calibration data based on the current tuning."""
-        self._differential_calibration_data = {}
+    def recompute_calibration_data(self, params: dict) -> None:
+        """
+        Set the differential_calibration_data and sensor_calibration_data
+        based on the specified ``params``.
+        """
+        recomputed = False
         if self.differential_calibration is not None:
-            cal_args = self._get_calibration_args_from_sigan(
-                self.differential_calibration
-            )
             self._differential_calibration_data.update(
-                self.differential_calibration.get_calibration_dict(cal_args)
+                self.differential_calibration.get_calibration_dict(params)
             )
+            recomputed = True
         else:
-            logger.warning("Differential calibration does not exist.")
+            logger.debug("No differential calibration available to recompute")
 
-    def _recompute_sensor_calibration_data(self) -> None:
-        """Set the sensor calibration data based on the current tuning."""
-        self._sensor_calibration_data = {}
         if self.sensor_calibration is not None:
-            cal_args = self._get_calibration_args_from_sigan(self.sensor_calibration)
             self._sensor_calibration_data.update(
-                self.sensor_calibration.get_calibration_dict(cal_args)
+                self.sensor_calibration.get_calibration_dict(params)
             )
+            recomputed = True
         else:
-            logger.warning("Sensor calibration does not exist.")
+            logger.debug("No sensor calibration available to recompute")
+
+        if not recomputed:
+            logger.warning("Failed to recompute calibration data")
 
     def acquire_time_domain_samples(
         self,
@@ -207,6 +178,7 @@ def acquire_time_domain_samples(
         num_samples_skip: int = 0,
         retries: int = 5,
         cal_adjust: bool = True,
+        cal_params: Optional[dict] = None,
     ) -> dict:
         """
         Acquire time-domain IQ samples from the signal analyzer.
@@ -224,6 +196,9 @@ def acquire_time_domain_samples(
         :param num_samples_skip: Number of samples to skip
         :param retries: Maximum number of retries on failure
         :param cal_adjust: If True, use available calibration data to scale the samples.
+        :param cal_params: A dictionary with keys for all of the calibration parameters.
+            May contain additional keys. Example: ``{"sample_rate": 14000000.0, "gain": 10.0}``
+            Must be specified if ``cal_adjust`` is ``True``. Otherwise, ignored.
         :return: dictionary containing data, sample_rate, frequency, capture_time, etc
         :raises Exception: If the sample acquisition fails, or the sensor has
             no signal analyzer.
@@ -231,7 +206,6 @@ def acquire_time_domain_samples(
         logger.debug("Sensor.acquire_time_domain_samples starting")
         logger.debug(f"Number of retries = {retries}")
         max_retries = retries
-        # TODO: Include RF path as a sensor cal argument?
         # Acquire samples from signal analyzer
         if self.signal_analyzer is not None:
             while True:
@@ -258,15 +232,18 @@ def acquire_time_domain_samples(
 
         # Apply gain adjustment based on calibration
         if cal_adjust:
+            if cal_params is None:
+                raise ValueError(
+                    "Data scaling cannot occur without specified calibration parameters."
+                )
             if self.sensor_calibration is not None:
                 logger.debug("Scaling samples using calibration data")
+                self.recompute_calibration_data(cal_params)
                 calibrated_gain__db = self.sensor_calibration_data["gain"]
                 calibrated_nf__db = self.sensor_calibration_data["noise_figure"]
                 logger.debug(f"Using sensor gain: {calibrated_gain__db} dB")
                 if self.differential_calibration is not None:
                     # Also apply differential calibration correction
-                    # TODO recompute functions match to current signal analyzer
-                    # settings. should they use the measurement_result instead?
                     differential_loss = self.differential_calibration_data["loss"]
                     logger.debug(f"Using differential loss: {differential_loss} dB")
                     calibrated_gain__db -= differential_loss
@@ -297,6 +274,6 @@ def acquire_time_domain_samples(
         else:
             # Set the data reference in the measurement_result
             measurement_result["reference"] = "signal analyzer input"
-            measurement_result["calibration"] = None
+            measurement_result["applied_calibration"] = None
 
         return measurement_result

From 087a3c1690841fb3587caab882ddaf406cde4952 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Thu, 25 Jan 2024 18:48:23 -0500
Subject: [PATCH 027/102] update DifferentialCalibration docstring

---
 .../calibration/differential_calibration.py   | 25 ++++++++-----------
 1 file changed, 10 insertions(+), 15 deletions(-)

diff --git a/scos_actions/calibration/differential_calibration.py b/scos_actions/calibration/differential_calibration.py
index e25ad71d..ac518737 100644
--- a/scos_actions/calibration/differential_calibration.py
+++ b/scos_actions/calibration/differential_calibration.py
@@ -2,19 +2,16 @@
 Dataclass implementation for "differential calibration" handling.
 
 A differential calibration provides loss values which represent excess loss
-between the `SensorCalibration` reference point and another reference point.
-A typical usage would be for calibrating out measured cable losses which exist
-between the antenna and the Y-factor calibration terminal. At present, this is
-measured manually using a calibration probe consisting of a calibrated noise
-source and a programmable attenuator.
-
-The ``reference_point`` top-level key defines the point to which measurements
-are referenced after using the correction factors included in the file.
-
-The ``calibration_data`` entries are expected to include these correction factors,
-with the key name ``"loss"`` and values in decibels (dB). A positive value of
-``"loss"`` indicates a LOSS going FROM ``reference_point`` TO the calibration
-terminal used by the ``SensorCalibration``.
+between the ``SensorCalibration.calibration_reference`` reference point and
+another reference point. A typical usage would be for calibrating out measured
+cable losses which exist between the antenna and the Y-factor calibration terminal.
+At present, this is measured manually using a calibration probe consisting of a
+calibrated noise source and a programmable attenuator.
+
+The ``DifferentialCalibration.calibration_data`` entries should be dictionaries
+containing the key ``"loss"`` and a corresponding value in decibels (dB). A positive
+value of ``"loss"`` indicates a LOSS going FROM ``DifferentialCalibration.calibration_reference``
+TO ``SensorCalibration.calibration_reference``.
 """
 
 from dataclasses import dataclass
@@ -24,8 +21,6 @@
 
 @dataclass
 class DifferentialCalibration(Calibration):
-    reference_point: str
-
     def update(self):
         """
         SCOS Sensor should not update differential calibration files.

From 5c99a97aa3426a7f50b977848bfb809c8caa0a12 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Thu, 25 Jan 2024 18:57:38 -0500
Subject: [PATCH 028/102] make calibration_reference required for all
 calibrations

---
 scos_actions/calibration/interfaces/calibration.py |  4 ++++
 scos_actions/calibration/sensor_calibration.py     |  3 +++
 scos_actions/calibration/tests/test_calibration.py | 14 ++++++++++----
 .../tests/test_differential_calibration.py         |  4 ++--
 .../calibration/tests/test_sensor_calibration.py   |  4 ++++
 scos_actions/hardware/sensor.py                    |  4 ++--
 6 files changed, 25 insertions(+), 8 deletions(-)

diff --git a/scos_actions/calibration/interfaces/calibration.py b/scos_actions/calibration/interfaces/calibration.py
index 42277253..6f1354ef 100644
--- a/scos_actions/calibration/interfaces/calibration.py
+++ b/scos_actions/calibration/interfaces/calibration.py
@@ -1,3 +1,6 @@
+"""
+TODO
+"""
 import dataclasses
 import json
 import logging
@@ -17,6 +20,7 @@
 class Calibration:
     calibration_parameters: List[str]
     calibration_data: dict
+    calibration_reference: str
     is_default: bool
     file_path: Path
 
diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index de547fc9..a4f23a54 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -1,3 +1,6 @@
+"""
+TODO
+"""
 import logging
 from dataclasses import dataclass
 from typing import Dict, List, Union
diff --git a/scos_actions/calibration/tests/test_calibration.py b/scos_actions/calibration/tests/test_calibration.py
index 9fb181e4..1e19b127 100644
--- a/scos_actions/calibration/tests/test_calibration.py
+++ b/scos_actions/calibration/tests/test_calibration.py
@@ -34,6 +34,7 @@ def setup_calibration_file(self, tmp_path: Path):
         self.sample_cal = Calibration(
             calibration_parameters=self.cal_params,
             calibration_data=self.cal_data,
+            calibration_reference="testing",
             is_default=False,
             file_path=self.dummy_file_path,
         )
@@ -41,6 +42,7 @@ def setup_calibration_file(self, tmp_path: Path):
         self.sample_default_cal = Calibration(
             calibration_parameters=self.cal_params,
             calibration_data=self.cal_data,
+            calibration_reference="testing",
             is_default=True,
             file_path=self.dummy_default_file_path,
         )
@@ -56,6 +58,7 @@ def test_calibration_dataclass_fields(self):
         # Note: does not check field order
         assert fields == {
             "calibration_parameters": List[str],
+            "calibration_reference": str,
             "is_default": bool,
             "calibration_data": dict,
             "file_path": Path,
@@ -64,13 +67,15 @@ def test_calibration_dataclass_fields(self):
     def test_field_validator(self):
         """Check that the input field type validator works."""
         with pytest.raises(TypeError):
-            _ = Calibration([], {}, False, False)
+            _ = Calibration([], {}, "", False, False)
         with pytest.raises(TypeError):
-            _ = Calibration([], {}, 100, Path(""))
+            _ = Calibration([], {}, "", 100, Path(""))
         with pytest.raises(TypeError):
-            _ = Calibration([], [10, 20], False, Path(""))
+            _ = Calibration([], {}, 5, False, Path(""))
         with pytest.raises(TypeError):
-            _ = Calibration({"test": 1}, {}, False, Path(""))
+            _ = Calibration([], [10, 20], "", False, Path(""))
+        with pytest.raises(TypeError):
+            _ = Calibration({"test": 1}, {}, "", False, Path(""))
 
     def test_get_calibration_dict(self):
         """Check the get_calibration_dict method with all dummy data."""
@@ -105,6 +110,7 @@ def test_to_and_from_json(self, tmp_path: Path):
         sensor_cal = SensorCalibration(
             self.sample_cal.calibration_parameters,
             self.sample_cal.calibration_data,
+            "testing",
             False,
             tmp_path / "testing.json",
             "dt_str",
diff --git a/scos_actions/calibration/tests/test_differential_calibration.py b/scos_actions/calibration/tests/test_differential_calibration.py
index 8f1b5594..fe53dea9 100644
--- a/scos_actions/calibration/tests/test_differential_calibration.py
+++ b/scos_actions/calibration/tests/test_differential_calibration.py
@@ -11,7 +11,7 @@ class TestDifferentialCalibration:
     def setup_differential_calibration_file(self, tmp_path: Path):
         dict_to_json = {
             "calibration_parameters": ["frequency"],
-            "reference_point": "antenna input",
+            "calibration_reference": "antenna input",
             "calibration_data": {3555e6: 11.5},
         }
         self.valid_file_path = tmp_path / "sample_diff_cal.json"
@@ -24,7 +24,7 @@ def setup_differential_calibration_file(self, tmp_path: Path):
         with open(self.valid_file_path, "w") as f:
             f.write(json.dumps(dict_to_json))
 
-        dict_to_json.pop("reference_point", None)
+        dict_to_json.pop("calibration_reference", None)
 
         with open(self.invalid_file_path, "w") as f:
             f.write(json.dumps(dict_to_json))
diff --git a/scos_actions/calibration/tests/test_sensor_calibration.py b/scos_actions/calibration/tests/test_sensor_calibration.py
index e9176718..f4433321 100644
--- a/scos_actions/calibration/tests/test_sensor_calibration.py
+++ b/scos_actions/calibration/tests/test_sensor_calibration.py
@@ -138,6 +138,7 @@ def setup_calibration_file(self, tmp_path: Path):
         # Add the simple stuff to new cal format
         cal_data["last_calibration_datetime"] = get_datetime_str_now()
         cal_data["sensor_uid"] = "SAMPLE_CALIBRATION"
+        cal_data["calibration_reference"] = "TESTING"
 
         # Add SR/CF lookup table
         cal_data["clock_rate_lookup_by_sample_rate"] = []
@@ -245,6 +246,7 @@ def test_get_calibration_dict_exact_match_lookup(self):
         cal = SensorCalibration(
             calibration_parameters=calibration_params,
             calibration_data=calibration_data,
+            calibration_reference="testing",
             is_default=False,
             file_path=Path(""),
             last_calibration_datetime=calibration_datetime,
@@ -264,6 +266,7 @@ def test_get_calibration_dict_within_range(self):
         cal = SensorCalibration(
             calibration_parameters=calibration_params,
             calibration_data=calibration_data,
+            calibration_reference="testing",
             is_default=False,
             file_path=Path("test_calibration.json"),
             last_calibration_datetime=calibration_datetime,
@@ -306,6 +309,7 @@ def test_update(self):
         cal = SensorCalibration(
             calibration_parameters=calibration_params,
             calibration_data=calibration_data,
+            calibration_reference="testing",
             is_default=False,
             file_path=test_cal_path,
             last_calibration_datetime=calibration_datetime,
diff --git a/scos_actions/hardware/sensor.py b/scos_actions/hardware/sensor.py
index 94aca3c3..b95adfcf 100644
--- a/scos_actions/hardware/sensor.py
+++ b/scos_actions/hardware/sensor.py
@@ -242,6 +242,7 @@ def acquire_time_domain_samples(
                 calibrated_gain__db = self.sensor_calibration_data["gain"]
                 calibrated_nf__db = self.sensor_calibration_data["noise_figure"]
                 logger.debug(f"Using sensor gain: {calibrated_gain__db} dB")
+                measurement_result = self.sensor_calibration.calibration_reference
                 if self.differential_calibration is not None:
                     # Also apply differential calibration correction
                     differential_loss = self.differential_calibration_data["loss"]
@@ -250,11 +251,10 @@ def acquire_time_domain_samples(
                     calibrated_nf__db += differential_loss
                     measurement_result[
                         "reference"
-                    ] = self.differential_calibration.reference_point
+                    ] = self.differential_calibration.calibration_reference
                 else:
                     # No differential calibration exists
                     logger.debug("No differential calibration was applied")
-                    measurement_result["reference"] = "calibration terminal"
 
                 linear_gain = 10.0 ** (calibrated_gain__db / 20.0)
                 logger.debug(f"Applying total gain of {calibrated_gain__db}")

From 4990d2eeefd078417d5064f6dc84bc9dd28aac3a Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Wed, 7 Feb 2024 16:30:45 -0500
Subject: [PATCH 029/102] fail early if sensor has no calibration object

---
 scos_actions/actions/calibrate_y_factor.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py
index bf1099e7..b777e7ac 100644
--- a/scos_actions/actions/calibrate_y_factor.py
+++ b/scos_actions/actions/calibrate_y_factor.py
@@ -75,6 +75,8 @@
 import numpy as np
 from scipy.constants import Boltzmann
 from scipy.signal import sosfilt
+
+from scos_actions import utils
 from scos_actions.actions.interfaces.action import Action
 from scos_actions.hardware.sensor import Sensor
 from scos_actions.hardware.sigan_iface import SIGAN_SETTINGS_KEYS
@@ -92,8 +94,6 @@
 from scos_actions.signals import trigger_api_restart
 from scos_actions.utils import ParameterException, get_parameter
 
-from scos_actions import utils
-
 logger = logging.getLogger(__name__)
 
 # Define parameter keys
@@ -196,6 +196,8 @@ def __init__(self, parameters: dict):
     def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
         """This is the entrypoint function called by the scheduler."""
         self.sensor = sensor
+        if self.sensor.sensor_calibration is None:
+            raise Exception("Sensor object must have a SensorCalibration object")
         self.test_required_components()
         detail = ""
 

From 17595faddd19d229da21cae686ce7cb56f47035a Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Wed, 7 Feb 2024 17:02:15 -0500
Subject: [PATCH 030/102] fix missing indexing key

---
 scos_actions/hardware/sensor.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scos_actions/hardware/sensor.py b/scos_actions/hardware/sensor.py
index 62884d1a..b5178bf7 100644
--- a/scos_actions/hardware/sensor.py
+++ b/scos_actions/hardware/sensor.py
@@ -316,7 +316,9 @@ def acquire_time_domain_samples(
                 calibrated_gain__db = self.sensor_calibration_data["gain"]
                 calibrated_nf__db = self.sensor_calibration_data["noise_figure"]
                 logger.debug(f"Using sensor gain: {calibrated_gain__db} dB")
-                measurement_result = self.sensor_calibration.calibration_reference
+                measurement_result[
+                    "reference"
+                ] = self.sensor_calibration.calibration_reference
                 if self.differential_calibration is not None:
                     # Also apply differential calibration correction
                     differential_loss = self.differential_calibration_data["loss"]

From 2ddaecc07c368f1ee70cbdaeb3c389ddea5f1d97 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 26 Feb 2024 16:21:24 -0500
Subject: [PATCH 031/102] Update .pre-commit-config.yaml

---
 .pre-commit-config.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 9618961d..12ef9bfc 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -18,7 +18,7 @@ repos:
       - id: end-of-file-fixer
       - id: trailing-whitespace
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.15.0
+    rev: v3.15.1
     hooks:
       - id: pyupgrade
         args: ["--py38-plus"]
@@ -30,12 +30,12 @@ repos:
         types: [file, python]
         args: ["--profile", "black", "--filter-files", "--gitignore"]
   - repo: https://github.com/psf/black
-    rev: 23.12.1
+    rev: 24.2.0
     hooks:
       - id: black
         types: [file, python]
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.38.0
+    rev: v0.39.0
     hooks:
       - id: markdownlint
         types: [file, markdown]

From 88a6b32c2388398726340d3dbf2fa90c45df1477 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 26 Feb 2024 16:23:14 -0500
Subject: [PATCH 032/102] fix log message when checking integer keys

---
 scos_actions/calibration/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scos_actions/calibration/utils.py b/scos_actions/calibration/utils.py
index 6ddba00b..9370c068 100644
--- a/scos_actions/calibration/utils.py
+++ b/scos_actions/calibration/utils.py
@@ -70,7 +70,7 @@ def filter_by_parameter(calibrations: dict, value: Union[float, int, bool]) -> d
         msg = (
             f"Could not locate calibration data at {value}"
             + f"\nAttempted lookup using key '{str(value).lower()}'"
-            + f"{f'and {float(value)}' if isinstance(value, int) else ''}"
+            + f"{f'and {float(value)}' if isinstance(value, float) and value.is_integer() else ''}"
             + f"\nUsing calibration data: {calibrations}"
         )
         raise CalibrationEntryMissingException(msg)

From 382e9075563c999a3d27fcacac1f5387ade574ce Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 26 Feb 2024 16:31:33 -0500
Subject: [PATCH 033/102] add debug mesasges for testing

---
 scos_actions/hardware/sensor.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/scos_actions/hardware/sensor.py b/scos_actions/hardware/sensor.py
index b5178bf7..3e7552c0 100644
--- a/scos_actions/hardware/sensor.py
+++ b/scos_actions/hardware/sensor.py
@@ -277,8 +277,15 @@ def acquire_time_domain_samples(
         :raises Exception: If the sample acquisition fails, or the sensor has
             no signal analyzer.
         """
+        logger.debug("***********************************\n")
         logger.debug("Sensor.acquire_time_domain_samples starting")
         logger.debug(f"Number of retries = {retries}")
+        logger.debug(
+            f"USING DIFF. CAL: {self.differential_calibration.calibration_data}"
+        )
+        logger.debug(f"USING SENSOR CAL: {self.sensor_calibration.calibration_data}")
+        logger.debug("*************************************\n")
+
         max_retries = retries
         # Acquire samples from signal analyzer
         if self.signal_analyzer is not None:
@@ -316,18 +323,18 @@ def acquire_time_domain_samples(
                 calibrated_gain__db = self.sensor_calibration_data["gain"]
                 calibrated_nf__db = self.sensor_calibration_data["noise_figure"]
                 logger.debug(f"Using sensor gain: {calibrated_gain__db} dB")
-                measurement_result[
-                    "reference"
-                ] = self.sensor_calibration.calibration_reference
+                measurement_result["reference"] = (
+                    self.sensor_calibration.calibration_reference
+                )
                 if self.differential_calibration is not None:
                     # Also apply differential calibration correction
                     differential_loss = self.differential_calibration_data["loss"]
                     logger.debug(f"Using differential loss: {differential_loss} dB")
                     calibrated_gain__db -= differential_loss
                     calibrated_nf__db += differential_loss
-                    measurement_result[
-                        "reference"
-                    ] = self.differential_calibration.calibration_reference
+                    measurement_result["reference"] = (
+                        self.differential_calibration.calibration_reference
+                    )
                 else:
                     # No differential calibration exists
                     logger.debug("No differential calibration was applied")

From 3337f0c87d72ef8bc729e62eacad84d4d17b32b2 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Fri, 1 Mar 2024 16:46:00 -0500
Subject: [PATCH 034/102] Add disk_usage to Computer metadata

---
 scos_actions/actions/acquire_sea_data_product.py  | 11 ++++++++++-
 scos_actions/metadata/structs/ntia_diagnostics.py |  2 ++
 scos_actions/utils.py                             |  9 +++++++++
 3 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/scos_actions/actions/acquire_sea_data_product.py b/scos_actions/actions/acquire_sea_data_product.py
index 3409699a..337503c7 100644
--- a/scos_actions/actions/acquire_sea_data_product.py
+++ b/scos_actions/actions/acquire_sea_data_product.py
@@ -74,7 +74,11 @@
     create_statistical_detector,
 )
 from scos_actions.signals import measurement_action_completed, trigger_api_restart
-from scos_actions.utils import convert_datetime_to_millisecond_iso_format, get_days_up
+from scos_actions.utils import (
+    convert_datetime_to_millisecond_iso_format,
+    get_days_up,
+    get_disk_usage,
+)
 
 env = Env()
 logger = logging.getLogger(__name__)
@@ -774,6 +778,11 @@ def capture_diagnostics(
             cpu_diag["ssd_smart_data"] = ntia_diagnostics.SsdSmartData(**smart_data)
         except:
             logger.warning("Failed to get SSD SMART data")
+        try:  # Disk usage
+            disk_usage = get_disk_usage()
+            cpu_diag["disk_usage"] = disk_usage
+        except:
+            logger.warning("Failed to get disk usage")
 
         # Get software versions
         software_diag = {
diff --git a/scos_actions/metadata/structs/ntia_diagnostics.py b/scos_actions/metadata/structs/ntia_diagnostics.py
index def0cbd5..f5f0f01c 100644
--- a/scos_actions/metadata/structs/ntia_diagnostics.py
+++ b/scos_actions/metadata/structs/ntia_diagnostics.py
@@ -139,6 +139,7 @@ class Computer(msgspec.Struct, **SIGMF_OBJECT_KWARGS):
     :param scos_uptime: Number of days since the SCOS API container started.
     :param ssd_smart_data: Information provided by the drive Self-Monitoring,
         Analysis, and Reporting Technology.
+    :param disk_usage: Total computer disk usage, as a percentage.
     """
 
     cpu_min_clock: Optional[float] = None
@@ -154,6 +155,7 @@ class Computer(msgspec.Struct, **SIGMF_OBJECT_KWARGS):
     software_start: Optional[str] = None
     software_uptime: Optional[float] = None
     ssd_smart_data: Optional[SsdSmartData] = None
+    disk_usage: Optional[float] = None
 
 
 class ScosPlugin(msgspec.Struct, **SIGMF_OBJECT_KWARGS):
diff --git a/scos_actions/utils.py b/scos_actions/utils.py
index e70a4324..182474ce 100644
--- a/scos_actions/utils.py
+++ b/scos_actions/utils.py
@@ -1,5 +1,6 @@
 import json
 import logging
+import shutil
 from datetime import datetime
 from pathlib import Path
 
@@ -124,3 +125,11 @@ def get_days_up(start_time):
     days = elapsed.days
     fractional_day = elapsed.seconds / (60 * 60 * 24)
     return round(days + fractional_day, 4)
+
+
+def get_disk_usage() -> float:
+    """Return the total disk usage as a percentage."""
+    usage = shutil.disk_usage("/")
+    percent_used = round(100 * usage.used / usage.total)
+    logger.debug(f"{percent_used} disk used")
+    return round(percent_used, 2)

From ac0d549f1c3ec316604156e78c043082823248e2 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Fri, 1 Mar 2024 16:56:02 -0500
Subject: [PATCH 035/102] bump version to 8.0.1

---
 scos_actions/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scos_actions/__init__.py b/scos_actions/__init__.py
index 73d4c8be..9e17e009 100644
--- a/scos_actions/__init__.py
+++ b/scos_actions/__init__.py
@@ -1 +1 @@
-__version__ = "8.0.0"
+__version__ = "8.0.1"

From 32b5dee6ac35a9db31a1d20e02dd90aeb78a901b Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Fri, 1 Mar 2024 18:28:48 -0500
Subject: [PATCH 036/102] fix docstrings, add missing ntp diagnostics fields

---
 scos_actions/metadata/structs/ntia_diagnostics.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/scos_actions/metadata/structs/ntia_diagnostics.py b/scos_actions/metadata/structs/ntia_diagnostics.py
index f5f0f01c..a0ef7c8d 100644
--- a/scos_actions/metadata/structs/ntia_diagnostics.py
+++ b/scos_actions/metadata/structs/ntia_diagnostics.py
@@ -129,16 +129,19 @@ class Computer(msgspec.Struct, **SIGMF_OBJECT_KWARGS):
     :param cpu_mean_clock: Mean sampled clock speed, in MHz.
     :param cpu_uptime: Number of days since the computer started.
     :param action_cpu_usage: CPU utilization during action execution, as a percentage.
+    :param action_runtime: Total action execution time, in seconds.
     :param system_load_5m: Number of processes in a runnable state over the
         previous 5 minutes as a percentage of the number of CPUs.
     :param memory_usage: Average percent of memory used during action execution.
     :param cpu_overheating: Whether the CPU is overheating.
     :param cpu_temp: CPU temperature, in degrees Celsius.
-    :param scos_start: The time at which the SCOS API container started. Must be
+    :param software_start: The time at which the sensor software started. Must be
         an ISO 8601 formatted string.
-    :param scos_uptime: Number of days since the SCOS API container started.
+    :param software_uptime: Number of days since the sensor software started.
     :param ssd_smart_data: Information provided by the drive Self-Monitoring,
         Analysis, and Reporting Technology.
+    :param ntp_active: True if NTP service is active on the computer.
+    :param ntp_sync: True if the system clock is synchronized with NTP.
     :param disk_usage: Total computer disk usage, as a percentage.
     """
 
@@ -155,6 +158,8 @@ class Computer(msgspec.Struct, **SIGMF_OBJECT_KWARGS):
     software_start: Optional[str] = None
     software_uptime: Optional[float] = None
     ssd_smart_data: Optional[SsdSmartData] = None
+    ntp_active: Optional[bool] = None
+    ntp_sync: Optional[bool] = None
     disk_usage: Optional[float] = None
 
 

From 2d7c0af16ef3dafb6e8767a23a8b10263d5632a5 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Fri, 1 Mar 2024 18:32:06 -0500
Subject: [PATCH 037/102] update NTIA SigMF extension versions

---
 scos_actions/metadata/sigmf_builder.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scos_actions/metadata/sigmf_builder.py b/scos_actions/metadata/sigmf_builder.py
index 2a838d82..2ed489e7 100644
--- a/scos_actions/metadata/sigmf_builder.py
+++ b/scos_actions/metadata/sigmf_builder.py
@@ -27,7 +27,7 @@
         },
         {
             "name": "ntia-diagnostics",
-            "version": "2.0.0",
+            "version": "2.2.0",
             "optional": True,
         },
         {
@@ -266,7 +266,7 @@ def set_collection(self, collection: str) -> None:
         """
         self.sigmf_md.set_global_field("core:collection", collection)
 
-    ### ntia-algorithm v2.0.0 ###
+    ### ntia-algorithm v2.0.1 ###
 
     def set_data_products(self, data_products: List[Graph]) -> None:
         """
@@ -311,7 +311,7 @@ def set_classification(self, classification: str) -> None:
         """
         self.sigmf_md.set_global_field("ntia-core:classification", classification)
 
-    ### ntia-diagnostics v1.0.0 ###
+    ### ntia-diagnostics v2.2.0 ###
 
     def set_diagnostics(self, diagnostics: Diagnostics) -> None:
         """

From 6ff053dfacba5e1c9c33fffaffbb4c9c2364f75d Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Fri, 1 Mar 2024 18:52:36 -0500
Subject: [PATCH 038/102] add NTP status to SEA metadata

---
 .../actions/acquire_sea_data_product.py       |  5 ++++
 scos_actions/hardware/utils.py                | 23 ++++++++++++++++---
 2 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/scos_actions/actions/acquire_sea_data_product.py b/scos_actions/actions/acquire_sea_data_product.py
index 337503c7..f3d368a1 100644
--- a/scos_actions/actions/acquire_sea_data_product.py
+++ b/scos_actions/actions/acquire_sea_data_product.py
@@ -45,6 +45,7 @@
     get_current_cpu_temperature,
     get_disk_smart_data,
     get_max_cpu_temperature,
+    get_ntp_status,
 )
 from scos_actions.metadata.sigmf_builder import SigMFBuilder
 from scos_actions.metadata.structs import (
@@ -778,6 +779,10 @@ def capture_diagnostics(
             cpu_diag["ssd_smart_data"] = ntia_diagnostics.SsdSmartData(**smart_data)
         except:
             logger.warning("Failed to get SSD SMART data")
+        try:
+            cpu_diag["ntp_active"], cpu_diag["ntp_sync"] = get_ntp_status()
+        except:
+            logger.warning("Failed to get NTP status")
         try:  # Disk usage
             disk_usage = get_disk_usage()
             cpu_diag["disk_usage"] = disk_usage
diff --git a/scos_actions/hardware/utils.py b/scos_actions/hardware/utils.py
index 49642da7..05de2956 100644
--- a/scos_actions/hardware/utils.py
+++ b/scos_actions/hardware/utils.py
@@ -1,6 +1,6 @@
 import logging
 import subprocess
-from typing import Dict
+from typing import Dict, Tuple, Union
 
 import psutil
 from its_preselector.web_relay import WebRelay
@@ -66,7 +66,23 @@ def get_current_cpu_temperature(fahrenheit: bool = False) -> float:
         raise e
 
 
-def get_disk_smart_data(disk: str) -> dict:
+def get_ntp_status() -> Union[Tuple[bool, bool], str]:
+    """
+    Get system NTP status by parsing the output of ``timedatectl``.
+
+    :returns: A tuple of booleans: (ntp_active, ntp_synchronized).
+    """
+    try:
+        status = subprocess.check_output(["timedatectl"]).decode("utf-8")
+    except Exception:
+        logger.exception(f"Unable to get NTP status from timedatectl")
+        return "Unavailable"
+    ntp_active = "NTP service: active" in status
+    ntp_synchronized = "System clock synchronized: yes" in status
+    return ntp_active, ntp_synchronized
+
+
+def get_disk_smart_data(disk: str) -> Union[dict, str]:
     """
     Get selected SMART data for the chosen disk.
 
@@ -81,7 +97,8 @@ def get_disk_smart_data(disk: str) -> dict:
     https://nvmexpress.org/wp-content/uploads/NVM-Express-1_4-2019.06.10-Ratified.pdf
 
     :param disk: The desired disk, e.g., ``/dev/nvme0n1``.
-    :return: A dictionary containing the retrieved data from the SMART report.
+    :return: A dictionary containing the retrieved data from the SMART report, or
+        the string "Unavailable" if ``smartctl`` fails to run.
     """
     try:
         report = subprocess.check_output(["smartctl", "-a", disk]).decode("utf-8")

From 89313d84092a97bbbea4634546ea048672fa05b0 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Tue, 5 Mar 2024 12:58:08 -0500
Subject: [PATCH 039/102] remove is_default calibration parameter

---
 .../calibration/interfaces/calibration.py         | 15 ++++++---------
 .../calibration/tests/test_calibration.py         | 14 ++------------
 .../tests/test_differential_calibration.py        |  4 +++-
 .../calibration/tests/test_sensor_calibration.py  |  5 ++---
 4 files changed, 13 insertions(+), 25 deletions(-)

diff --git a/scos_actions/calibration/interfaces/calibration.py b/scos_actions/calibration/interfaces/calibration.py
index 6f1354ef..b243bea4 100644
--- a/scos_actions/calibration/interfaces/calibration.py
+++ b/scos_actions/calibration/interfaces/calibration.py
@@ -1,6 +1,7 @@
 """
 TODO
 """
+
 import dataclasses
 import json
 import logging
@@ -21,7 +22,6 @@ class Calibration:
     calibration_parameters: List[str]
     calibration_data: dict
     calibration_reference: str
-    is_default: bool
     file_path: Path
 
     def __post_init__(self):
@@ -75,7 +75,7 @@ def update(self):
         raise NotImplementedError
 
     @classmethod
-    def from_json(cls, fname: Path, is_default: bool):
+    def from_json(cls, fname: Path):
         """
         Load a calibration from a JSON file.
 
@@ -84,8 +84,6 @@ def from_json(cls, fname: Path, is_default: bool):
         the class being constructed.
 
         :param fname: The ``Path`` to the JSON calibration file.
-        :param is_default: If True, the loaded calibration file
-            is treated as the default calibration file.
         :raises Exception: If the provided file does not include
             the required keys.
         :return: The ``Calibration`` object generated from the file.
@@ -96,7 +94,7 @@ def from_json(cls, fname: Path, is_default: bool):
 
         # Check that only the required fields are in the dict
         required_keys = {f.name for f in dataclasses.fields(cls)}
-        required_keys -= {"is_default", "file_path"}  # are not required in JSON
+        required_keys -= {"file_path"}  # not required in JSON
         if cal_file_keys == required_keys:
             pass
         elif cal_file_keys >= required_keys:
@@ -115,7 +113,7 @@ def from_json(cls, fname: Path, is_default: bool):
             )
 
         # Create and return the Calibration object
-        return cls(is_default=is_default, file_path=fname, **calibration)
+        return cls(file_path=fname, **calibration)
 
     def to_json(self) -> None:
         """
@@ -123,12 +121,11 @@ def to_json(self) -> None:
 
         The JSON file will be located at ``self.file_path`` and will
         contain a copy of ``self.__dict__``, except for the ``file_path``
-        and ``is_default`` key/value pairs. This includes all dataclass
-        fields, with their parameter names as JSON key names.
+        key/value pair. This includes all dataclass fields, with their
+        parameter names as JSON key names.
         """
         dict_to_json = self.__dict__.copy()
         # Remove keys which should not save to JSON
         dict_to_json.pop("file_path", None)
-        dict_to_json.pop("is_default", None)
         with open(self.file_path, "w") as outfile:
             outfile.write(json.dumps(dict_to_json))
diff --git a/scos_actions/calibration/tests/test_calibration.py b/scos_actions/calibration/tests/test_calibration.py
index 1e19b127..297808ef 100644
--- a/scos_actions/calibration/tests/test_calibration.py
+++ b/scos_actions/calibration/tests/test_calibration.py
@@ -1,10 +1,12 @@
 """Test the Calibration base dataclass."""
+
 import dataclasses
 import json
 from pathlib import Path
 from typing import List
 
 import pytest
+
 from scos_actions.calibration.interfaces.calibration import Calibration
 from scos_actions.calibration.sensor_calibration import SensorCalibration
 from scos_actions.calibration.tests.utils import recursive_check_keys
@@ -35,7 +37,6 @@ def setup_calibration_file(self, tmp_path: Path):
             calibration_parameters=self.cal_params,
             calibration_data=self.cal_data,
             calibration_reference="testing",
-            is_default=False,
             file_path=self.dummy_file_path,
         )
 
@@ -43,7 +44,6 @@ def setup_calibration_file(self, tmp_path: Path):
             calibration_parameters=self.cal_params,
             calibration_data=self.cal_data,
             calibration_reference="testing",
-            is_default=True,
             file_path=self.dummy_default_file_path,
         )
 
@@ -59,7 +59,6 @@ def test_calibration_dataclass_fields(self):
         assert fields == {
             "calibration_parameters": List[str],
             "calibration_reference": str,
-            "is_default": bool,
             "calibration_data": dict,
             "file_path": Path,
         }, "Calibration class fields have changed"
@@ -96,15 +95,6 @@ def test_to_and_from_json(self, tmp_path: Path):
             self.dummy_default_file_path, True
         )
 
-        # These should fail: the is_default parameter is different
-        # even though the other contents are identical.
-        with pytest.raises(AssertionError):
-            assert self.sample_cal == Calibration.from_json(self.dummy_file_path, True)
-        with pytest.raises(AssertionError):
-            assert self.sample_default_cal == Calibration.from_json(
-                self.dummy_default_file_path, False
-            )
-
         # from_json should ignore extra keys in the loaded file, but not fail
         # Test this by trying to load a SensorCalibration as a Calibration
         sensor_cal = SensorCalibration(
diff --git a/scos_actions/calibration/tests/test_differential_calibration.py b/scos_actions/calibration/tests/test_differential_calibration.py
index fe53dea9..88025a82 100644
--- a/scos_actions/calibration/tests/test_differential_calibration.py
+++ b/scos_actions/calibration/tests/test_differential_calibration.py
@@ -1,8 +1,10 @@
 """Test the DifferentialCalibration dataclass."""
+
 import json
 from pathlib import Path
 
 import pytest
+
 from scos_actions.calibration.differential_calibration import DifferentialCalibration
 
 
@@ -18,7 +20,7 @@ def setup_differential_calibration_file(self, tmp_path: Path):
         self.invalid_file_path = tmp_path / "sample_diff_cal_invalid.json"
 
         self.sample_diff_cal = DifferentialCalibration(
-            is_default=False, file_path=self.valid_file_path, **dict_to_json
+            file_path=self.valid_file_path, **dict_to_json
         )
 
         with open(self.valid_file_path, "w") as f:
diff --git a/scos_actions/calibration/tests/test_sensor_calibration.py b/scos_actions/calibration/tests/test_sensor_calibration.py
index f4433321..86c0fdd7 100644
--- a/scos_actions/calibration/tests/test_sensor_calibration.py
+++ b/scos_actions/calibration/tests/test_sensor_calibration.py
@@ -1,4 +1,5 @@
 """Test the SensorCalibration dataclass."""
+
 import dataclasses
 import datetime
 import json
@@ -9,6 +10,7 @@
 from typing import Dict, List
 
 import pytest
+
 from scos_actions.calibration.interfaces.calibration import Calibration
 from scos_actions.calibration.sensor_calibration import SensorCalibration
 from scos_actions.calibration.tests.utils import recursive_check_keys
@@ -247,7 +249,6 @@ def test_get_calibration_dict_exact_match_lookup(self):
             calibration_parameters=calibration_params,
             calibration_data=calibration_data,
             calibration_reference="testing",
-            is_default=False,
             file_path=Path(""),
             last_calibration_datetime=calibration_datetime,
             clock_rate_lookup_by_sample_rate=[],
@@ -267,7 +268,6 @@ def test_get_calibration_dict_within_range(self):
             calibration_parameters=calibration_params,
             calibration_data=calibration_data,
             calibration_reference="testing",
-            is_default=False,
             file_path=Path("test_calibration.json"),
             last_calibration_datetime=calibration_datetime,
             clock_rate_lookup_by_sample_rate=[],
@@ -310,7 +310,6 @@ def test_update(self):
             calibration_parameters=calibration_params,
             calibration_data=calibration_data,
             calibration_reference="testing",
-            is_default=False,
             file_path=test_cal_path,
             last_calibration_datetime=calibration_datetime,
             clock_rate_lookup_by_sample_rate=[],

From 785149388c07c75aaabc594026ff49d54a0d1960 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Tue, 5 Mar 2024 17:10:10 -0500
Subject: [PATCH 040/102] Do not overwrite sensor calibration file

---
 scos_actions/actions/calibrate_y_factor.py | 64 +++++++++++++++++++---
 1 file changed, 56 insertions(+), 8 deletions(-)

diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py
index b777e7ac..f3a75f4d 100644
--- a/scos_actions/actions/calibrate_y_factor.py
+++ b/scos_actions/actions/calibrate_y_factor.py
@@ -17,7 +17,13 @@
 # - SCOS Markdown Editor: https://ntia.github.io/scos-md-editor/
 #
 r"""Perform a Y-Factor Calibration.
-Supports calibration of gain and noise figure for one or more channels.
+Supports calculation of gain and noise figure for one or more channels using the
+Y-Factor method. Results are written to the file specified by the environment
+variable ``ONBOARD_CALIBRATION_FILE``. If the sensor already has a sensor calibration
+object, it is used as the starting point, and copied to a new onboard calibration object
+which is updated by this action. The sensor object's sensor calibration will be set to
+the updated onboard calibration object after this action is run.
+
 For each center frequency, sets the preselector to the noise diode path, turns
 noise diode on, performs a mean power measurement, turns the noise diode off and
 performs another mean power measurement. The mean power on and mean power off
@@ -73,11 +79,13 @@
 import time
 
 import numpy as np
+from environs import Env
 from scipy.constants import Boltzmann
 from scipy.signal import sosfilt
 
 from scos_actions import utils
 from scos_actions.actions.interfaces.action import Action
+from scos_actions.calibration.sensor_calibration import SensorCalibration
 from scos_actions.hardware.sensor import Sensor
 from scos_actions.hardware.sigan_iface import SIGAN_SETTINGS_KEYS
 from scos_actions.signal_processing.calibration import (
@@ -92,9 +100,10 @@
 from scos_actions.signal_processing.power_analysis import calculate_power_watts
 from scos_actions.signal_processing.unit_conversion import convert_watts_to_dBm
 from scos_actions.signals import trigger_api_restart
-from scos_actions.utils import ParameterException, get_parameter
+from scos_actions.utils import ParameterException, get_datetime_str_now, get_parameter
 
 logger = logging.getLogger(__name__)
+env = Env()
 
 # Define parameter keys
 RF_PATH = Action.PRESELECTOR_PATH_KEY
@@ -112,6 +121,7 @@
 IIR_RESP_FREQS = "iir_num_response_frequencies"
 CAL_SOURCE_IDX = "cal_source_idx"
 TEMP_SENSOR_IDX = "temp_sensor_idx"
+REFERENCE_POINT = "reference_point"
 
 
 class YFactorCalibration(Action):
@@ -196,8 +206,44 @@ def __init__(self, parameters: dict):
     def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
         """This is the entrypoint function called by the scheduler."""
         self.sensor = sensor
+
+        # Prepare the sensor calibration object.
+        assert all(
+            self.iteration_params[0][REFERENCE_POINT] == p[REFERENCE_POINT]
+            for p in self.iteration_params
+        ), f"All iterations must use the same '{REFERENCE_POINT}' setting"
+        onboard_cal_reference = self.iteration_params[0][REFERENCE_POINT]
+
         if self.sensor.sensor_calibration is None:
-            raise Exception("Sensor object must have a SensorCalibration object")
+            # Create a new sensor calibration object and attach it to the sensor.
+            # The calibration parameters will be set to the sigan parameters used
+            # in the action YAML parameters.
+            logger.debug(f"Creating a new onboard cal object for the sensor.")
+            cal_params = [k for k in self.iteration_params if k in SIGAN_SETTINGS_KEYS]
+            cal_data = dict()
+            last_cal_datetime = get_datetime_str_now()
+            clock_rate_lookup_by_sample_rate = []
+            sensor_uid = "Sensor calibration file not provided"
+            self.sensor.sensor_calibration = SensorCalibration(
+                calibration_parameters=cal_params,
+                calibration_data=cal_data,
+                calibration_reference=onboard_cal_reference,
+                file_path=env("ONBOARD_CALIBRATION_FILE"),
+                last_calibration_datetime=last_cal_datetime,
+                clock_rate_lookup_by_sample_rate=clock_rate_lookup_by_sample_rate,
+                sensor_uid=sensor_uid,
+            )
+        elif self.sensor.sensor_calibration.file_path == env(
+            "ONBOARD_CALIBRATION_FILE"
+        ):
+            # Already using an onboard cal file.
+            logger.debug("Onboard calibration file already in use. Continuing.")
+        else:
+            # Sensor calibration file exists. Change it to an onboard cal file
+            logger.debug("Making new onboard cal file from existing sensor cal")
+            self.sensor.sensor_calibration.calibration_reference = onboard_cal_reference
+            self.sensor.sensor_calibration.file_path = env("ONBOARD_CALIBRATION_FILE")
+
         self.test_required_components()
         detail = ""
 
@@ -207,6 +253,8 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
                 detail += self.calibrate(p)
             else:
                 detail += os.linesep + self.calibrate(p)
+        # Save results to onboard calibration file
+        self.sensor.sensor_calibration.to_json()
         return detail
 
     def calibrate(self, params: dict):
@@ -257,11 +305,6 @@ def calibrate(self, params: dict):
             noise_on_data = sosfilt(self.iir_sos, noise_on_measurement_result["data"])
             noise_off_data = sosfilt(self.iir_sos, noise_off_measurement_result["data"])
         else:
-            if self.sensor.sensor_calibration.is_default:
-                raise Exception(
-                    "Calibrations without IIR filter cannot be performed with default calibration."
-                )
-
             logger.debug("Skipping IIR filtering")
             # Get ENBW from sensor calibration
             assert set(self.sensor.sensor_calibration.calibration_parameters) <= set(
@@ -272,6 +315,11 @@ def calibrate(self, params: dict):
                 for k in self.sensor.sensor_calibration.calibration_parameters
             ]
             self.sensor.recompute_sensor_calibration_data(cal_args)
+            if "enbw" not in self.sensor.sensor_calibration_data:
+                raise Exception(
+                    "Unable to perform Y-Factor calibration without IIR filtering when no"
+                    " ENBW is provided in the sensor calibration file."
+                )
             enbw_hz = self.sensor.sensor_calibration_data["enbw"]
             noise_on_data = noise_on_measurement_result["data"]
             noise_off_data = noise_off_measurement_result["data"]

From 51820e8508795fcf89752b179b7479c6ba25e259 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Tue, 5 Mar 2024 17:25:07 -0500
Subject: [PATCH 041/102] fix environment variable reference

---
 scos_actions/actions/calibrate_y_factor.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py
index f3a75f4d..a1593b12 100644
--- a/scos_actions/actions/calibrate_y_factor.py
+++ b/scos_actions/actions/calibrate_y_factor.py
@@ -19,7 +19,7 @@
 r"""Perform a Y-Factor Calibration.
 Supports calculation of gain and noise figure for one or more channels using the
 Y-Factor method. Results are written to the file specified by the environment
-variable ``ONBOARD_CALIBRATION_FILE``. If the sensor already has a sensor calibration
+variable ``ONBOARD_SENSOR_CALIBRATION_FILE``. If the sensor already has a sensor calibration
 object, it is used as the starting point, and copied to a new onboard calibration object
 which is updated by this action. The sensor object's sensor calibration will be set to
 the updated onboard calibration object after this action is run.
@@ -228,13 +228,13 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
                 calibration_parameters=cal_params,
                 calibration_data=cal_data,
                 calibration_reference=onboard_cal_reference,
-                file_path=env("ONBOARD_CALIBRATION_FILE"),
+                file_path=env("ONBOARD_SENSOR_CALIBRATION_FILE"),
                 last_calibration_datetime=last_cal_datetime,
                 clock_rate_lookup_by_sample_rate=clock_rate_lookup_by_sample_rate,
                 sensor_uid=sensor_uid,
             )
         elif self.sensor.sensor_calibration.file_path == env(
-            "ONBOARD_CALIBRATION_FILE"
+            "ONBOARD_SENSOR_CALIBRATION_FILE"
         ):
             # Already using an onboard cal file.
             logger.debug("Onboard calibration file already in use. Continuing.")

From afbf2bf4837b8e25e1da2ec84c8f157829480758 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Tue, 5 Mar 2024 17:31:07 -0500
Subject: [PATCH 042/102] add missing import

---
 scos_actions/hardware/mocks/mock_sigan.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scos_actions/hardware/mocks/mock_sigan.py b/scos_actions/hardware/mocks/mock_sigan.py
index ae675556..b4394e06 100644
--- a/scos_actions/hardware/mocks/mock_sigan.py
+++ b/scos_actions/hardware/mocks/mock_sigan.py
@@ -1,10 +1,12 @@
 """Mock a signal analyzer for testing."""
+
 import logging
 from collections import namedtuple
 from typing import Optional
 
 import numpy as np
 
+from scos_actions import __version__ as SCOS_ACTIONS_VERSION
 from scos_actions.hardware.sigan_iface import SignalAnalyzerInterface
 from scos_actions.utils import get_datetime_str_now
 

From 8bddcf2b9d0465d7c502b3650e091dc3487d29a3 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Tue, 5 Mar 2024 17:35:53 -0500
Subject: [PATCH 043/102] fix from_json calibration file tests

---
 .../calibration/tests/test_differential_calibration.py        | 4 ++--
 scos_actions/calibration/tests/test_sensor_calibration.py     | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/scos_actions/calibration/tests/test_differential_calibration.py b/scos_actions/calibration/tests/test_differential_calibration.py
index 88025a82..5c9c80ca 100644
--- a/scos_actions/calibration/tests/test_differential_calibration.py
+++ b/scos_actions/calibration/tests/test_differential_calibration.py
@@ -33,10 +33,10 @@ def setup_differential_calibration_file(self, tmp_path: Path):
 
     def test_from_json(self):
         """Check from_json functionality with valid and invalid dummy data."""
-        diff_cal = DifferentialCalibration.from_json(self.valid_file_path, False)
+        diff_cal = DifferentialCalibration.from_json(self.valid_file_path)
         assert diff_cal == self.sample_diff_cal
         with pytest.raises(Exception):
-            _ = DifferentialCalibration.from_json(self.invalid_file_path, False)
+            _ = DifferentialCalibration.from_json(self.invalid_file_path)
 
     def test_update_not_implemented(self):
         """Check that the update method is not implemented."""
diff --git a/scos_actions/calibration/tests/test_sensor_calibration.py b/scos_actions/calibration/tests/test_sensor_calibration.py
index 86c0fdd7..d8914a68 100644
--- a/scos_actions/calibration/tests/test_sensor_calibration.py
+++ b/scos_actions/calibration/tests/test_sensor_calibration.py
@@ -184,7 +184,7 @@ def setup_calibration_file(self, tmp_path: Path):
             json.dump(cal_data, file, indent=4)
 
         # Load the data back in
-        self.sample_cal = SensorCalibration.from_json(self.calibration_file, False)
+        self.sample_cal = SensorCalibration.from_json(self.calibration_file)
 
         # Create a list of previous points to ensure that we don't repeat
         self.pytest_points = []
@@ -318,7 +318,7 @@ def test_update(self):
         action_params = {"sample_rate": 100.0, "frequency": 200.0}
         update_time = get_datetime_str_now()
         cal.update(action_params, update_time, 30.0, 5.0, 21)
-        cal_from_file = SensorCalibration.from_json(test_cal_path, False)
+        cal_from_file = SensorCalibration.from_json(test_cal_path)
         test_cal_path.unlink()
         file_utc_time = parse_datetime_iso_format_str(cal.last_calibration_datetime)
         cal_time_utc = parse_datetime_iso_format_str(update_time)

From c36efd84e86bb8850af5b051da401bc0f610bc19 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Tue, 5 Mar 2024 17:38:59 -0500
Subject: [PATCH 044/102] remove unused imports

---
 scos_actions/actions/acquire_single_freq_fft.py       |  8 ++++----
 .../actions/acquire_single_freq_tdomain_iq.py         | 11 +++++------
 2 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/scos_actions/actions/acquire_single_freq_fft.py b/scos_actions/actions/acquire_single_freq_fft.py
index 21bc8f5e..3ae3fe7a 100644
--- a/scos_actions/actions/acquire_single_freq_fft.py
+++ b/scos_actions/actions/acquire_single_freq_fft.py
@@ -89,8 +89,8 @@
 import logging
 
 from numpy import float32, ndarray
+
 from scos_actions.actions.interfaces.measurement_action import MeasurementAction
-from scos_actions.hardware.mocks.mock_gps import MockGPS
 from scos_actions.metadata.structs import ntia_algorithm
 from scos_actions.signal_processing.fft import (
     get_fft,
@@ -173,9 +173,9 @@ def execute(self, schedule_entry: dict, task_id: int) -> dict:
         # Save measurement results
         measurement_result["data"] = m4s_result
         measurement_result.update(self.parameters)
-        measurement_result[
-            "calibration_datetime"
-        ] = self.sensor.sensor_calibration_data["datetime"]
+        measurement_result["calibration_datetime"] = (
+            self.sensor.sensor_calibration_data["datetime"]
+        )
         measurement_result["task_id"] = task_id
         measurement_result["classification"] = self.classification
 
diff --git a/scos_actions/actions/acquire_single_freq_tdomain_iq.py b/scos_actions/actions/acquire_single_freq_tdomain_iq.py
index 6ff6b537..14796a05 100644
--- a/scos_actions/actions/acquire_single_freq_tdomain_iq.py
+++ b/scos_actions/actions/acquire_single_freq_tdomain_iq.py
@@ -34,11 +34,10 @@
 import logging
 
 from numpy import complex64
-from scos_actions.actions.interfaces.measurement_action import MeasurementAction
-from scos_actions.hardware.mocks.mock_gps import MockGPS
-from scos_actions.utils import get_parameter
 
 from scos_actions import utils
+from scos_actions.actions.interfaces.measurement_action import MeasurementAction
+from scos_actions.utils import get_parameter
 
 logger = logging.getLogger(__name__)
 
@@ -91,9 +90,9 @@ def execute(self, schedule_entry: dict, task_id: int) -> dict:
         measurement_result.update(self.parameters)
         measurement_result["end_time"] = end_time
         measurement_result["task_id"] = task_id
-        measurement_result[
-            "calibration_datetime"
-        ] = self.sensor.sensor_calibration_data["datetime"]
+        measurement_result["calibration_datetime"] = (
+            self.sensor.sensor_calibration_data["datetime"]
+        )
         measurement_result["classification"] = self.classification
         sigan_settings = self.get_sigan_settings(measurement_result)
         logger.debug(f"sigan settings:{sigan_settings}")

From 57a3dca3cc03a3c20f12218cc3806a6396a41ea5 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Tue, 5 Mar 2024 17:43:37 -0500
Subject: [PATCH 045/102] fix cal data lookup unit tests

---
 scos_actions/calibration/tests/test_sensor_calibration.py | 2 +-
 scos_actions/calibration/tests/test_utils.py              | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/scos_actions/calibration/tests/test_sensor_calibration.py b/scos_actions/calibration/tests/test_sensor_calibration.py
index d8914a68..743faf1b 100644
--- a/scos_actions/calibration/tests/test_sensor_calibration.py
+++ b/scos_actions/calibration/tests/test_sensor_calibration.py
@@ -277,7 +277,7 @@ def test_get_calibration_dict_within_range(self):
             _ = cal.get_calibration_dict({"sample_rate": 100.0, "frequency": 250.0})
         assert e_info.value.args[0] == (
             f"Could not locate calibration data at 250.0"
-            + f"\nAttempted lookup using key '250.0'"
+            + f"\nAttempted lookup using key '250.0' and 250.0"
             + f"\nUsing calibration data: {cal.calibration_data['100.0']}"
         )
 
diff --git a/scos_actions/calibration/tests/test_utils.py b/scos_actions/calibration/tests/test_utils.py
index 2a751831..15ab29cf 100644
--- a/scos_actions/calibration/tests/test_utils.py
+++ b/scos_actions/calibration/tests/test_utils.py
@@ -1,4 +1,5 @@
 import pytest
+
 from scos_actions.calibration.utils import CalibrationException, filter_by_parameter
 
 
@@ -10,7 +11,7 @@ def test_filter_by_parameter_out_of_range(self):
         assert (
             e_info.value.args[0]
             == f"Could not locate calibration data at 400.0"
-            + f"\nAttempted lookup using key '400.0'"
+            + f"\nAttempted lookup using key '400.0' and 400.0"
             + f"\nUsing calibration data: {calibrations}"
         )
 
@@ -23,7 +24,7 @@ def test_filter_by_parameter_in_range_requires_match(self):
             _ = filter_by_parameter(calibrations, 150.0)
         assert e_info.value.args[0] == (
             f"Could not locate calibration data at 150.0"
-            + f"\nAttempted lookup using key '150.0'"
+            + f"\nAttempted lookup using key '150.0' and 150.0"
             + f"\nUsing calibration data: {calibrations}"
         )
 

From 3d52f8e41317048bcc5011bb9d9c0867f9fd1fd4 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Tue, 5 Mar 2024 17:44:37 -0500
Subject: [PATCH 046/102] fix formatting in cal lookup error

---
 scos_actions/calibration/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scos_actions/calibration/utils.py b/scos_actions/calibration/utils.py
index 9370c068..fa977653 100644
--- a/scos_actions/calibration/utils.py
+++ b/scos_actions/calibration/utils.py
@@ -70,7 +70,7 @@ def filter_by_parameter(calibrations: dict, value: Union[float, int, bool]) -> d
         msg = (
             f"Could not locate calibration data at {value}"
             + f"\nAttempted lookup using key '{str(value).lower()}'"
-            + f"{f'and {float(value)}' if isinstance(value, float) and value.is_integer() else ''}"
+            + f"{f' and {float(value)}' if isinstance(value, float) and value.is_integer() else ''}"
             + f"\nUsing calibration data: {calibrations}"
         )
         raise CalibrationEntryMissingException(msg)

From 28ec1b0a8e4e7ac74caf85d70930f7ed3a55a1f3 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Wed, 6 Mar 2024 11:47:49 -0500
Subject: [PATCH 047/102] fix calibration from_json unit tests

---
 scos_actions/calibration/tests/test_calibration.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/scos_actions/calibration/tests/test_calibration.py b/scos_actions/calibration/tests/test_calibration.py
index 297808ef..20c160b8 100644
--- a/scos_actions/calibration/tests/test_calibration.py
+++ b/scos_actions/calibration/tests/test_calibration.py
@@ -90,9 +90,9 @@ def test_to_and_from_json(self, tmp_path: Path):
         self.sample_cal.to_json()
         self.sample_default_cal.to_json()
         # Then load and compare
-        assert self.sample_cal == Calibration.from_json(self.dummy_file_path, False)
+        assert self.sample_cal == Calibration.from_json(self.dummy_file_path)
         assert self.sample_default_cal == Calibration.from_json(
-            self.dummy_default_file_path, True
+            self.dummy_default_file_path
         )
 
         # from_json should ignore extra keys in the loaded file, but not fail
@@ -101,14 +101,13 @@ def test_to_and_from_json(self, tmp_path: Path):
             self.sample_cal.calibration_parameters,
             self.sample_cal.calibration_data,
             "testing",
-            False,
             tmp_path / "testing.json",
             "dt_str",
             [],
             "uid",
         )
         sensor_cal.to_json()
-        loaded_cal = Calibration.from_json(tmp_path / "testing.json", False)
+        loaded_cal = Calibration.from_json(tmp_path / "testing.json")
         loaded_cal.file_path = self.sample_cal.file_path  # Force these to be the same
         assert loaded_cal == self.sample_cal
 
@@ -118,7 +117,7 @@ def test_to_and_from_json(self, tmp_path: Path):
         with open(tmp_path / "almost_a_cal.json", "w") as outfile:
             outfile.write(json.dumps(almost_a_cal))
         with pytest.raises(Exception):
-            almost = Calibration.from_json(tmp_path / "almost_a_cal.json", False)
+            almost = Calibration.from_json(tmp_path / "almost_a_cal.json")
 
     def test_update_not_implemented(self):
         """Ensure the update abstract method is not implemented in the base class"""

From 20845a51395ac59c19fb79be07ec2114420dae16 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Wed, 6 Mar 2024 11:48:17 -0500
Subject: [PATCH 048/102] fix kwargs in call to parent class init

---
 scos_actions/hardware/mocks/mock_sensor.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/scos_actions/hardware/mocks/mock_sensor.py b/scos_actions/hardware/mocks/mock_sensor.py
index 4e432f9f..529919de 100644
--- a/scos_actions/hardware/mocks/mock_sensor.py
+++ b/scos_actions/hardware/mocks/mock_sensor.py
@@ -40,14 +40,14 @@ def __init__(
                 "Calibration object provided to mock sensor will not be used to query calibration data."
             )
         super().__init__(
-            signal_analyzer,
-            gps,
-            preselector,
-            switches,
-            location,
-            capabilities,
-            sensor_cal,
-            differential_cal,
+            signal_analyzer=signal_analyzer,
+            gps=gps,
+            preselector=preselector,
+            switches=switches,
+            location=location,
+            capabilities=capabilities,
+            sensor_cal=sensor_cal,
+            differential_cal=differential_cal,
         )
 
     @property

From 3fe4adc7f1a03dfbc3cfffa6997b4b0ac6c55b88 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Wed, 6 Mar 2024 11:49:59 -0500
Subject: [PATCH 049/102] fix incorrect attribute name in unit test

---
 scos_actions/hardware/tests/test_sensor.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scos_actions/hardware/tests/test_sensor.py b/scos_actions/hardware/tests/test_sensor.py
index e8f74552..1c3eac95 100644
--- a/scos_actions/hardware/tests/test_sensor.py
+++ b/scos_actions/hardware/tests/test_sensor.py
@@ -28,8 +28,8 @@ def test_mock_sensor_defaults(mock_sensor):
     assert mock_sensor.switches == {}
     assert mock_sensor.location == _mock_location
     assert mock_sensor.capabilities == _mock_capabilities
-    assert mock_sensor.sensor_cal is None
-    assert mock_sensor.differential_cal is None
+    assert mock_sensor.sensor_calibration is None
+    assert mock_sensor.differential_calibration is None
     assert mock_sensor.has_configurable_preselector is False
     assert mock_sensor.has_configurable_preselector is False
     assert mock_sensor.sensor_calibration_data == _mock_sensor_cal_data

From f901514a68cec56ebe4d33ccda9be8aba34145b7 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Wed, 6 Mar 2024 11:54:03 -0500
Subject: [PATCH 050/102] remove tests for no longer used sigan attribute

---
 .../tests/test_acquire_single_freq_fft.py     |  9 ---------
 .../tests/test_single_freq_tdomain_iq.py      |  9 ---------
 .../tests/test_stepped_freq_tdomain_iq.py     | 20 -------------------
 3 files changed, 38 deletions(-)

diff --git a/scos_actions/actions/tests/test_acquire_single_freq_fft.py b/scos_actions/actions/tests/test_acquire_single_freq_fft.py
index 1e65abec..e4d9c9fb 100644
--- a/scos_actions/actions/tests/test_acquire_single_freq_fft.py
+++ b/scos_actions/actions/tests/test_acquire_single_freq_fft.py
@@ -82,12 +82,3 @@ def callback(sender, **kwargs):
             ]
         ]
     )
-
-
-def test_num_samples_skip():
-    action = actions["test_single_frequency_m4s_action"]
-    assert action.description
-    action(
-        sensor=MockSensor(), schedule_entry=SINGLE_FREQUENCY_FFT_ACQUISITION, task_id=1
-    )
-    assert action.sensor.signal_analyzer._num_samples_skip == action.parameters["nskip"]
diff --git a/scos_actions/actions/tests/test_single_freq_tdomain_iq.py b/scos_actions/actions/tests/test_single_freq_tdomain_iq.py
index 2095d856..774ce91e 100644
--- a/scos_actions/actions/tests/test_single_freq_tdomain_iq.py
+++ b/scos_actions/actions/tests/test_single_freq_tdomain_iq.py
@@ -66,12 +66,3 @@ def test_required_components():
     with pytest.raises(RuntimeError):
         action(sensor, SINGLE_TIMEDOMAIN_IQ_ACQUISITION, 1)
     mock_sigan._is_available = True
-
-
-def test_num_samples_skip():
-    action = actions["test_single_frequency_iq_action"]
-    assert action.description
-    action(
-        sensor=MockSensor(), schedule_entry=SINGLE_TIMEDOMAIN_IQ_ACQUISITION, task_id=1
-    )
-    assert action.sensor.signal_analyzer._num_samples_skip == action.parameters["nskip"]
diff --git a/scos_actions/actions/tests/test_stepped_freq_tdomain_iq.py b/scos_actions/actions/tests/test_stepped_freq_tdomain_iq.py
index 20873922..2183d0bf 100644
--- a/scos_actions/actions/tests/test_stepped_freq_tdomain_iq.py
+++ b/scos_actions/actions/tests/test_stepped_freq_tdomain_iq.py
@@ -41,23 +41,3 @@ def callback(sender, **kwargs):
         assert _task_ids[i] == 1
         assert _recording_ids[i] == i + 1
     assert _count == 10
-
-
-def test_num_samples_skip():
-    action = actions["test_multi_frequency_iq_action"]
-    assert action.description
-    action(
-        sensor=MockSensor(),
-        schedule_entry=SINGLE_TIMEDOMAIN_IQ_MULTI_RECORDING_ACQUISITION,
-        task_id=1,
-    )
-    if isinstance(action.parameters["nskip"], list):
-        assert (
-            action.sensor.signal_analyzer._num_samples_skip
-            == action.parameters["nskip"][-1]
-        )
-    else:
-        assert (
-            action.sensor.signal_analyzer._num_samples_skip
-            == action.parameters["nskip"]
-        )

From ab2831d589432d54035009ede705bede0fef2a3a Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Wed, 6 Mar 2024 11:54:36 -0500
Subject: [PATCH 051/102] avoid error due to debug messages when run with no
 calibration

---
 scos_actions/hardware/sensor.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/scos_actions/hardware/sensor.py b/scos_actions/hardware/sensor.py
index 3e7552c0..08dbcd9c 100644
--- a/scos_actions/hardware/sensor.py
+++ b/scos_actions/hardware/sensor.py
@@ -280,10 +280,14 @@ def acquire_time_domain_samples(
         logger.debug("***********************************\n")
         logger.debug("Sensor.acquire_time_domain_samples starting")
         logger.debug(f"Number of retries = {retries}")
-        logger.debug(
-            f"USING DIFF. CAL: {self.differential_calibration.calibration_data}"
-        )
-        logger.debug(f"USING SENSOR CAL: {self.sensor_calibration.calibration_data}")
+        if self.differential_calibration is not None:
+            logger.debug(
+                f"USING DIFF. CAL: {self.differential_calibration.calibration_data}"
+            )
+        if self.sensor_calibration is not None:
+            logger.debug(
+                f"USING SENSOR CAL: {self.sensor_calibration.calibration_data}"
+            )
         logger.debug("*************************************\n")
 
         max_retries = retries

From 71dcd8501952712d8917ed366c0c4a377f696725 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Wed, 6 Mar 2024 11:55:52 -0500
Subject: [PATCH 052/102] fix generator return type hint

---
 scos_actions/actions/acquire_sea_data_product.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scos_actions/actions/acquire_sea_data_product.py b/scos_actions/actions/acquire_sea_data_product.py
index fcadb459..b50ab53b 100644
--- a/scos_actions/actions/acquire_sea_data_product.py
+++ b/scos_actions/actions/acquire_sea_data_product.py
@@ -34,6 +34,9 @@
 from environs import Env
 from its_preselector import __version__ as PRESELECTOR_API_VERSION
 from scipy.signal import sos2tf, sosfilt
+
+from scos_actions import __version__ as SCOS_ACTIONS_VERSION
+from scos_actions import utils
 from scos_actions.actions.interfaces.action import Action
 from scos_actions.hardware.sensor import Sensor
 from scos_actions.hardware.utils import (
@@ -73,9 +76,6 @@
 from scos_actions.signals import measurement_action_completed, trigger_api_restart
 from scos_actions.utils import convert_datetime_to_millisecond_iso_format, get_days_up
 
-from scos_actions import __version__ as SCOS_ACTIONS_VERSION
-from scos_actions import utils
-
 env = Env()
 logger = logging.getLogger(__name__)
 
@@ -425,7 +425,7 @@ def __init__(self, params: dict, iir_sos: np.ndarray):
         ]
         del params
 
-    def run(self, iqdata: np.ndarray) -> list:
+    def run(self, iqdata: np.ndarray):
         """
         Filter the input IQ data and concurrently compute FFT, PVT, PFP, and APD results.
 

From 4f33c3e1725fc43a7088fff20ed11aec0fabaebd Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Wed, 6 Mar 2024 12:41:24 -0500
Subject: [PATCH 053/102] don't require datetime in sensor cal data

---
 scos_actions/actions/acquire_single_freq_fft.py        | 6 +++---
 scos_actions/actions/acquire_single_freq_tdomain_iq.py | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/scos_actions/actions/acquire_single_freq_fft.py b/scos_actions/actions/acquire_single_freq_fft.py
index 3ae3fe7a..25f0b3f3 100644
--- a/scos_actions/actions/acquire_single_freq_fft.py
+++ b/scos_actions/actions/acquire_single_freq_fft.py
@@ -173,9 +173,9 @@ def execute(self, schedule_entry: dict, task_id: int) -> dict:
         # Save measurement results
         measurement_result["data"] = m4s_result
         measurement_result.update(self.parameters)
-        measurement_result["calibration_datetime"] = (
-            self.sensor.sensor_calibration_data["datetime"]
-        )
+        # measurement_result["calibration_datetime"] = (
+        #     self.sensor.sensor_calibration_data["datetime"]
+        # )
         measurement_result["task_id"] = task_id
         measurement_result["classification"] = self.classification
 
diff --git a/scos_actions/actions/acquire_single_freq_tdomain_iq.py b/scos_actions/actions/acquire_single_freq_tdomain_iq.py
index 14796a05..8ddaa11a 100644
--- a/scos_actions/actions/acquire_single_freq_tdomain_iq.py
+++ b/scos_actions/actions/acquire_single_freq_tdomain_iq.py
@@ -90,9 +90,9 @@ def execute(self, schedule_entry: dict, task_id: int) -> dict:
         measurement_result.update(self.parameters)
         measurement_result["end_time"] = end_time
         measurement_result["task_id"] = task_id
-        measurement_result["calibration_datetime"] = (
-            self.sensor.sensor_calibration_data["datetime"]
-        )
+        # measurement_result["calibration_datetime"] = (
+        #     self.sensor.sensor_calibration_data["datetime"]
+        # )
         measurement_result["classification"] = self.classification
         sigan_settings = self.get_sigan_settings(measurement_result)
         logger.debug(f"sigan settings:{sigan_settings}")

From 1c6f700f2cd659cf0bccd3339dfd78a66a8ba736 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Tue, 12 Mar 2024 15:13:05 -0400
Subject: [PATCH 054/102] Fill metadata reference from loaded calibration

---
 .../actions/acquire_sea_data_product.py       | 26 ++++++++++++-------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/scos_actions/actions/acquire_sea_data_product.py b/scos_actions/actions/acquire_sea_data_product.py
index b50ab53b..9e4e827f 100644
--- a/scos_actions/actions/acquire_sea_data_product.py
+++ b/scos_actions/actions/acquire_sea_data_product.py
@@ -112,7 +112,6 @@
 FFT_WINDOW = get_fft_window(FFT_WINDOW_TYPE, FFT_SIZE)
 FFT_WINDOW_ECF = get_fft_window_correction(FFT_WINDOW, "energy")
 IMPEDANCE_OHMS = 50.0
-DATA_REFERENCE_POINT = "noise source output"  # TODO delete
 NUM_ACTORS = 3  # Number of ray actors to initialize
 
 # Create power detectors
@@ -521,7 +520,6 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
             self.iteration_params,
         )
         self.create_global_sensor_metadata(self.sensor)
-        self.create_global_data_product_metadata()
 
         # Initialize remote supervisor actors for IQ processing
         tic = perf_counter()
@@ -534,7 +532,7 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
         logger.debug(f"Spawned {NUM_ACTORS} supervisor actors in {toc-tic:.2f} s")
 
         # Collect all IQ data and spawn data product computation processes
-        dp_procs, cpu_speed = [], []
+        dp_procs, cpu_speed, reference_points = [], [], []
         capture_tic = perf_counter()
         for i, parameters in enumerate(self.iteration_params):
             measurement_result = self.capture_iq(parameters)
@@ -548,16 +546,23 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
             toc = perf_counter()
             logger.debug(f"IQ data delivered for processing in {toc-tic:.2f} s")
             # Create capture segment with channel-specific metadata before sigan is reconfigured
-            tic = perf_counter()
             self.create_capture_segment(i, measurement_result)
-            toc = perf_counter()
-            logger.debug(f"Created capture metadata in {toc-tic:.2f} s")
+            # Query CPU speed for later averaging in diagnostics metadata
             cpu_speed.append(get_current_cpu_clock_speed())
+            # Append list of data reference points; later we require these to be identical
+            reference_points.append(measurement_result["reference"])
         capture_toc = perf_counter()
         logger.debug(
             f"Collected all IQ data and started all processing in {capture_toc-capture_tic:.2f} s"
         )
 
+        # Create data product metadata: requires all data reference points
+        # to be identical.
+        assert (
+            len(set(reference_points)) == 1
+        ), "Channel data were scaled to different reference points. Cannot build metadata."
+        self.create_global_data_product_metadata(reference_points[0])
+
         # Collect processed data product results
         all_data, max_max_ch_pwrs, med_mean_ch_pwrs, mean_ch_pwrs, median_ch_pwrs = (
             [],
@@ -971,7 +976,7 @@ def test_required_components(self):
             trigger_api_restart.send(sender=self.__class__)
         return None
 
-    def create_global_data_product_metadata(self) -> None:
+    def create_global_data_product_metadata(self, data_products_reference: str) -> None:
         p = self.parameters
         num_iq_samples = int(p[SAMPLE_RATE] * p[DURATION_MS] * 1e-3)
         iir_obj = ntia_algorithm.DigitalFilter(
@@ -1016,7 +1021,7 @@ def create_global_data_product_metadata(self) -> None:
             x_step=[p[SAMPLE_RATE] / FFT_SIZE],
             y_units="dBm/Hz",
             processing=[dft_obj.id],
-            reference=DATA_REFERENCE_POINT,  # TODO update
+            reference=data_products_reference,
             description=(
                 "Results of statistical detectors (max, mean, median, 25th_percentile, 75th_percentile, "
                 + "90th_percentile, 95th_percentile, 99th_percentile, 99.9th_percentile, 99.99th_percentile) "
@@ -1036,7 +1041,7 @@ def create_global_data_product_metadata(self) -> None:
             x_stop=[pvt_x_axis__s[-1]],
             x_step=[pvt_x_axis__s[1] - pvt_x_axis__s[0]],
             y_units="dBm",
-            reference=DATA_REFERENCE_POINT,  # TODO update
+            reference=data_products_reference,
             description=(
                 "Max- and mean-detected channel power vs. time, with "
                 + f"an integration time of {p[TD_BIN_SIZE_MS]} ms. "
@@ -1063,7 +1068,7 @@ def create_global_data_product_metadata(self) -> None:
             x_stop=[pfp_x_axis__s[-1]],
             x_step=[pfp_x_axis__s[1] - pfp_x_axis__s[0]],
             y_units="dBm",
-            reference=DATA_REFERENCE_POINT,  # TODO update
+            reference=data_products_reference,
             description=(
                 "Channelized periodic frame power statistics reported over"
                 + f" a {p[PFP_FRAME_PERIOD_MS]} ms frame period, with frame resolution"
@@ -1086,6 +1091,7 @@ def create_global_data_product_metadata(self) -> None:
             y_start=[apd_y_axis__dBm[0]],
             y_stop=[apd_y_axis__dBm[-1]],
             y_step=[apd_y_axis__dBm[1] - apd_y_axis__dBm[0]],
+            reference=data_products_reference,
             description=(
                 f"Estimate of the APD, using a {p[APD_BIN_SIZE_DB]} dB "
                 + "bin size for amplitude values. The data payload includes"

From e849a0df4c5dddc57a2b25648f17c5d29f0aaf7b Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Thu, 14 Mar 2024 09:35:47 -0600
Subject: [PATCH 055/102] Correct ONBOARD_CALIBRATION_FILE var.

---
 scos_actions/actions/calibrate_y_factor.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py
index a1593b12..f3a75f4d 100644
--- a/scos_actions/actions/calibrate_y_factor.py
+++ b/scos_actions/actions/calibrate_y_factor.py
@@ -19,7 +19,7 @@
 r"""Perform a Y-Factor Calibration.
 Supports calculation of gain and noise figure for one or more channels using the
 Y-Factor method. Results are written to the file specified by the environment
-variable ``ONBOARD_SENSOR_CALIBRATION_FILE``. If the sensor already has a sensor calibration
+variable ``ONBOARD_CALIBRATION_FILE``. If the sensor already has a sensor calibration
 object, it is used as the starting point, and copied to a new onboard calibration object
 which is updated by this action. The sensor object's sensor calibration will be set to
 the updated onboard calibration object after this action is run.
@@ -228,13 +228,13 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
                 calibration_parameters=cal_params,
                 calibration_data=cal_data,
                 calibration_reference=onboard_cal_reference,
-                file_path=env("ONBOARD_SENSOR_CALIBRATION_FILE"),
+                file_path=env("ONBOARD_CALIBRATION_FILE"),
                 last_calibration_datetime=last_cal_datetime,
                 clock_rate_lookup_by_sample_rate=clock_rate_lookup_by_sample_rate,
                 sensor_uid=sensor_uid,
             )
         elif self.sensor.sensor_calibration.file_path == env(
-            "ONBOARD_SENSOR_CALIBRATION_FILE"
+            "ONBOARD_CALIBRATION_FILE"
         ):
             # Already using an onboard cal file.
             logger.debug("Onboard calibration file already in use. Continuing.")

From b17b749847c43f4d807732bea879d20c16889a05 Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Thu, 14 Mar 2024 09:43:07 -0600
Subject: [PATCH 056/102] Use Path for file_path.

---
 scos_actions/actions/calibrate_y_factor.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py
index f3a75f4d..af00dcbe 100644
--- a/scos_actions/actions/calibrate_y_factor.py
+++ b/scos_actions/actions/calibrate_y_factor.py
@@ -77,6 +77,7 @@
 import logging
 import os
 import time
+from pathlib import Path
 
 import numpy as np
 from environs import Env
@@ -228,7 +229,7 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
                 calibration_parameters=cal_params,
                 calibration_data=cal_data,
                 calibration_reference=onboard_cal_reference,
-                file_path=env("ONBOARD_CALIBRATION_FILE"),
+                file_path=Path(env("ONBOARD_CALIBRATION_FILE")),
                 last_calibration_datetime=last_cal_datetime,
                 clock_rate_lookup_by_sample_rate=clock_rate_lookup_by_sample_rate,
                 sensor_uid=sensor_uid,

From 4e22e836edce827b6a3c4c4445e6c241dd17169d Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Thu, 14 Mar 2024 10:02:10 -0600
Subject: [PATCH 057/102] debug

---
 scos_actions/actions/acquire_sea_data_product.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scos_actions/actions/acquire_sea_data_product.py b/scos_actions/actions/acquire_sea_data_product.py
index 9e4e827f..0ba93d54 100644
--- a/scos_actions/actions/acquire_sea_data_product.py
+++ b/scos_actions/actions/acquire_sea_data_product.py
@@ -451,6 +451,7 @@ class NasctnSeaDataProduct(Action):
     def __init__(self, parameters: dict):
         super().__init__(parameters)
         # Assume preselector is present
+        self.total_channel_data_length = None
         rf_path_name = utils.get_parameter(RF_PATH, self.parameters)
         self.rf_path = {self.PRESELECTOR_PATH_KEY: rf_path_name}
 
@@ -1110,6 +1111,7 @@ def create_global_data_product_metadata(self, data_products_reference: str) -> N
             + pfp_length * len(PFP_M3_DETECTOR) * 2
             + apd_graph.length
         )
+        logger.debug(f"Total channel length:{self.total_channel_data_length}")
 
     def create_capture_segment(
         self,

From d9d9bb4646698855ad4d0e7249589c423195a187 Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Thu, 14 Mar 2024 11:23:34 -0600
Subject: [PATCH 058/102] create global data product metadata after first IQ
 capture so reference is available and total_channel_data_length is known
 during capture creation.

---
 scos_actions/actions/acquire_sea_data_product.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scos_actions/actions/acquire_sea_data_product.py b/scos_actions/actions/acquire_sea_data_product.py
index 0ba93d54..2e438dd4 100644
--- a/scos_actions/actions/acquire_sea_data_product.py
+++ b/scos_actions/actions/acquire_sea_data_product.py
@@ -521,7 +521,6 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
             self.iteration_params,
         )
         self.create_global_sensor_metadata(self.sensor)
-
         # Initialize remote supervisor actors for IQ processing
         tic = perf_counter()
         # This uses iteration_params[0] because
@@ -535,8 +534,11 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
         # Collect all IQ data and spawn data product computation processes
         dp_procs, cpu_speed, reference_points = [], [], []
         capture_tic = perf_counter()
+
         for i, parameters in enumerate(self.iteration_params):
             measurement_result = self.capture_iq(parameters)
+            if i == 0:
+                self.create_global_data_product_metadata(measurement_result["reference"])
             # Start data product processing but do not block next IQ capture
             tic = perf_counter()
 
@@ -562,7 +564,7 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
         assert (
             len(set(reference_points)) == 1
         ), "Channel data were scaled to different reference points. Cannot build metadata."
-        self.create_global_data_product_metadata(reference_points[0])
+
 
         # Collect processed data product results
         all_data, max_max_ch_pwrs, med_mean_ch_pwrs, mean_ch_pwrs, median_ch_pwrs = (

From 8012c2b92a4e9a016074c77d03ba168175208811 Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Thu, 14 Mar 2024 13:00:05 -0600
Subject: [PATCH 059/102] Add expired method to SensorCalibration.

---
 .../calibration/sensor_calibration.py         | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index a4f23a54..b12a4287 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -3,10 +3,13 @@
 """
 import logging
 from dataclasses import dataclass
+from datetime import datetime
+from environs import Env
 from typing import Dict, List, Union
 
 from scos_actions.calibration.interfaces.calibration import Calibration
 from scos_actions.calibration.utils import CalibrationEntryMissingException
+from scos_actions.utils import parse_datetime_iso_format_str
 
 logger = logging.getLogger(__name__)
 
@@ -84,3 +87,21 @@ def update(
 
         # Write updated calibration data to file
         self.to_json()
+
+    def expired(self) -> bool:
+        env = Env()
+        time_limit = env("CALIBRATION_EXPIRATION_LIMIT")
+        if time_limit is None:
+            return False
+        elif self.calibration_data is None:
+            return True;
+        elif len(self.calibration_data) == 0:
+            return True;
+        else:
+            now = datetime.now()
+            for cal_data in self.calibration_data:
+                cal_datetime =  parse_datetime_iso_format_str(cal_data["datetime"])
+                elapsed = now - cal_datetime
+                if elapsed.total_seconds() > time_limit:
+                    return True
+            return False
\ No newline at end of file

From 9a1004eaa5ed588451e0e2879f7c4098d0c64b23 Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Thu, 14 Mar 2024 13:18:11 -0600
Subject: [PATCH 060/102] set CALIBRATION_EXPIRATION_LIMIT to None if not set.

---
 scos_actions/calibration/sensor_calibration.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index b12a4287..950f0b35 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -90,7 +90,7 @@ def update(
 
     def expired(self) -> bool:
         env = Env()
-        time_limit = env("CALIBRATION_EXPIRATION_LIMIT")
+        time_limit = env("CALIBRATION_EXPIRATION_LIMIT", default=None)
         if time_limit is None:
             return False
         elif self.calibration_data is None:

From bdb142bf59c51d042637158ca411a0ad250f2e6f Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Thu, 14 Mar 2024 14:49:24 -0600
Subject: [PATCH 061/102] recusive check if calibration has expired and
 associated tests.

---
 .../calibration/sensor_calibration.py         | 28 ++++++++++++++-----
 .../tests/test_sensor_calibration.py          | 25 +++++++++++++++++
 2 files changed, 46 insertions(+), 7 deletions(-)

diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index 950f0b35..23dccf90 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -98,10 +98,24 @@ def expired(self) -> bool:
         elif len(self.calibration_data) == 0:
             return True;
         else:
-            now = datetime.now()
-            for cal_data in self.calibration_data:
-                cal_datetime =  parse_datetime_iso_format_str(cal_data["datetime"])
-                elapsed = now - cal_datetime
-                if elapsed.total_seconds() > time_limit:
-                    return True
-            return False
\ No newline at end of file
+            now_string = datetime.utcnow().isoformat(timespec="milliseconds") + "Z"
+            now = parse_datetime_iso_format_str(now_string)
+            cal_data = self.calibration_data
+            return has_expired_cal_data(cal_data, now,time_limit)
+
+def has_expired_cal_data( cal_data: dict, now: datetime, time_limit: int) -> bool:
+    expired = False
+    if "datetime" in cal_data:
+        expired = expired or date_expired(cal_data, now, time_limit)
+
+    for key, value in cal_data.items():
+        if isinstance(value, dict):
+            expired = expired or has_expired_cal_data(value,now,time_limit)
+    return expired
+
+def date_expired( cal_data: dict, now: datetime, time_limit: int):
+    cal_datetime = parse_datetime_iso_format_str(cal_data["datetime"])
+    elapsed = now - cal_datetime
+    if elapsed.total_seconds() > time_limit:
+        return True
+    return False
diff --git a/scos_actions/calibration/tests/test_sensor_calibration.py b/scos_actions/calibration/tests/test_sensor_calibration.py
index 743faf1b..1f351e41 100644
--- a/scos_actions/calibration/tests/test_sensor_calibration.py
+++ b/scos_actions/calibration/tests/test_sensor_calibration.py
@@ -13,6 +13,7 @@
 
 from scos_actions.calibration.interfaces.calibration import Calibration
 from scos_actions.calibration.sensor_calibration import SensorCalibration
+from scos_actions.calibration.sensor_calibration import has_expired_cal_data
 from scos_actions.calibration.tests.utils import recursive_check_keys
 from scos_actions.calibration.utils import CalibrationException
 from scos_actions.tests.resources.utils import easy_gain
@@ -331,3 +332,27 @@ def test_update(self):
         assert cal.calibration_data["100.0"]["200.0"]["noise_figure"] == 5.0
         assert cal_from_file.calibration_data["100.0"]["200.0"]["gain"] == 30.0
         assert cal_from_file.calibration_data["100.0"]["200.0"]["noise_figure"] == 5.0
+
+    def test_has_expired_cal_data_not_expired(self):
+        cal_date = "2024-03-14T15:48:38.039Z"
+        now_date = "2024-03-14T15:49:38.039Z"
+        cal_data = {"3550": {"20.0": {"false": {"datetime": cal_date},},}}
+        expired = has_expired_cal_data(cal_data,  parse_datetime_iso_format_str( now_date), 100)
+        assert expired == False
+
+    def test_has_expired_cal_data_expired(self):
+        cal_date = "2024-03-14T15:48:38.039Z"
+        now_date = "2024-03-14T15:49:38.039Z"
+        cal_data = {"3550": {"20.0": {"false": {"datetime": cal_date},},}}
+        expired = has_expired_cal_data(cal_data, parse_datetime_iso_format_str( now_date), 30)
+        assert expired == True
+
+    def test_has_expired_cal_data_multipledates_expired(self):
+        cal_date_1 = "2024-03-14T15:48:38.039Z"
+        cal_date_2 = "2024-03-14T15:40:38.039Z"
+        now_date = "2024-03-14T15:49:38.039Z"
+        cal_data = {"3550": {"20.0": {"false": {"datetime": cal_date_1},}, "true":{ "datetime": cal_date_2},}}
+        expired = has_expired_cal_data(cal_data, parse_datetime_iso_format_str( now_date), 100)
+        assert expired == True
+        cal_data = {"3550": {"20.0": {"false": {"datetime": cal_date_2},}, "true":{ "datetime": cal_date_1},}}
+        expired = has_expired_cal_data(cal_data, parse_datetime_iso_format_str( now_date), 100)
\ No newline at end of file

From 16366c78e4a41dacdd0fc243181f54bf7e3a10f5 Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Thu, 14 Mar 2024 14:51:14 -0600
Subject: [PATCH 062/102] pre-commit

---
 sample_debug.py                               |  1 +
 .../actions/acquire_sea_data_product.py       |  5 +-
 .../acquire_stepped_freq_tdomain_iq.py        |  4 +-
 .../actions/interfaces/measurement_action.py  |  1 +
 .../calibration/sensor_calibration.py         | 18 +++---
 .../tests/test_sensor_calibration.py          | 56 +++++++++++++++----
 scos_actions/hardware/sigan_iface.py          |  1 +
 scos_actions/hardware/tests/test_sigan.py     |  1 +
 scos_actions/metadata/structs/capture.py      |  1 +
 scos_actions/signal_processing/calibration.py |  1 +
 .../tests/test_calibration.py                 |  1 +
 .../signal_processing/tests/test_fft.py       |  1 +
 .../signal_processing/tests/test_filtering.py |  1 +
 .../tests/test_power_analysis.py              |  1 +
 .../tests/test_unit_conversion.py             |  1 +
 15 files changed, 73 insertions(+), 21 deletions(-)

diff --git a/sample_debug.py b/sample_debug.py
index cc04ae29..3cf415fe 100644
--- a/sample_debug.py
+++ b/sample_debug.py
@@ -2,6 +2,7 @@
 This is a sample file showing how an action be created and called for debugging purposes
 using a mock signal analyzer.
 """
+
 import json
 
 from scos_actions.actions.acquire_single_freq_fft import SingleFrequencyFftAcquisition
diff --git a/scos_actions/actions/acquire_sea_data_product.py b/scos_actions/actions/acquire_sea_data_product.py
index 2e438dd4..4046fc33 100644
--- a/scos_actions/actions/acquire_sea_data_product.py
+++ b/scos_actions/actions/acquire_sea_data_product.py
@@ -538,7 +538,9 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
         for i, parameters in enumerate(self.iteration_params):
             measurement_result = self.capture_iq(parameters)
             if i == 0:
-                self.create_global_data_product_metadata(measurement_result["reference"])
+                self.create_global_data_product_metadata(
+                    measurement_result["reference"]
+                )
             # Start data product processing but do not block next IQ capture
             tic = perf_counter()
 
@@ -565,7 +567,6 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
             len(set(reference_points)) == 1
         ), "Channel data were scaled to different reference points. Cannot build metadata."
 
-
         # Collect processed data product results
         all_data, max_max_ch_pwrs, med_mean_ch_pwrs, mean_ch_pwrs, median_ch_pwrs = (
             [],
diff --git a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
index 7278e5f1..6de6c5a9 100644
--- a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
+++ b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
@@ -38,6 +38,8 @@
 import logging
 
 import numpy as np
+
+from scos_actions import utils
 from scos_actions.actions.acquire_single_freq_tdomain_iq import (
     CAL_ADJUST,
     DURATION_MS,
@@ -51,8 +53,6 @@
 from scos_actions.signals import measurement_action_completed
 from scos_actions.utils import get_parameter
 
-from scos_actions import utils
-
 logger = logging.getLogger(__name__)
 
 
diff --git a/scos_actions/actions/interfaces/measurement_action.py b/scos_actions/actions/interfaces/measurement_action.py
index fe5e197c..c8fb1847 100644
--- a/scos_actions/actions/interfaces/measurement_action.py
+++ b/scos_actions/actions/interfaces/measurement_action.py
@@ -3,6 +3,7 @@
 from typing import Optional
 
 import numpy as np
+
 from scos_actions.actions.interfaces.action import Action
 from scos_actions.hardware.sensor import Sensor
 from scos_actions.metadata.structs import ntia_sensor
diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index 23dccf90..f5e03b63 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -1,12 +1,14 @@
 """
 TODO
 """
+
 import logging
 from dataclasses import dataclass
 from datetime import datetime
-from environs import Env
 from typing import Dict, List, Union
 
+from environs import Env
+
 from scos_actions.calibration.interfaces.calibration import Calibration
 from scos_actions.calibration.utils import CalibrationEntryMissingException
 from scos_actions.utils import parse_datetime_iso_format_str
@@ -94,26 +96,28 @@ def expired(self) -> bool:
         if time_limit is None:
             return False
         elif self.calibration_data is None:
-            return True;
+            return True
         elif len(self.calibration_data) == 0:
-            return True;
+            return True
         else:
             now_string = datetime.utcnow().isoformat(timespec="milliseconds") + "Z"
             now = parse_datetime_iso_format_str(now_string)
             cal_data = self.calibration_data
-            return has_expired_cal_data(cal_data, now,time_limit)
+            return has_expired_cal_data(cal_data, now, time_limit)
+
 
-def has_expired_cal_data( cal_data: dict, now: datetime, time_limit: int) -> bool:
+def has_expired_cal_data(cal_data: dict, now: datetime, time_limit: int) -> bool:
     expired = False
     if "datetime" in cal_data:
         expired = expired or date_expired(cal_data, now, time_limit)
 
     for key, value in cal_data.items():
         if isinstance(value, dict):
-            expired = expired or has_expired_cal_data(value,now,time_limit)
+            expired = expired or has_expired_cal_data(value, now, time_limit)
     return expired
 
-def date_expired( cal_data: dict, now: datetime, time_limit: int):
+
+def date_expired(cal_data: dict, now: datetime, time_limit: int):
     cal_datetime = parse_datetime_iso_format_str(cal_data["datetime"])
     elapsed = now - cal_datetime
     if elapsed.total_seconds() > time_limit:
diff --git a/scos_actions/calibration/tests/test_sensor_calibration.py b/scos_actions/calibration/tests/test_sensor_calibration.py
index 1f351e41..01d70854 100644
--- a/scos_actions/calibration/tests/test_sensor_calibration.py
+++ b/scos_actions/calibration/tests/test_sensor_calibration.py
@@ -12,8 +12,10 @@
 import pytest
 
 from scos_actions.calibration.interfaces.calibration import Calibration
-from scos_actions.calibration.sensor_calibration import SensorCalibration
-from scos_actions.calibration.sensor_calibration import has_expired_cal_data
+from scos_actions.calibration.sensor_calibration import (
+    SensorCalibration,
+    has_expired_cal_data,
+)
 from scos_actions.calibration.tests.utils import recursive_check_keys
 from scos_actions.calibration.utils import CalibrationException
 from scos_actions.tests.resources.utils import easy_gain
@@ -336,23 +338,57 @@ def test_update(self):
     def test_has_expired_cal_data_not_expired(self):
         cal_date = "2024-03-14T15:48:38.039Z"
         now_date = "2024-03-14T15:49:38.039Z"
-        cal_data = {"3550": {"20.0": {"false": {"datetime": cal_date},},}}
-        expired = has_expired_cal_data(cal_data,  parse_datetime_iso_format_str( now_date), 100)
+        cal_data = {
+            "3550": {
+                "20.0": {
+                    "false": {"datetime": cal_date},
+                },
+            }
+        }
+        expired = has_expired_cal_data(
+            cal_data, parse_datetime_iso_format_str(now_date), 100
+        )
         assert expired == False
 
     def test_has_expired_cal_data_expired(self):
         cal_date = "2024-03-14T15:48:38.039Z"
         now_date = "2024-03-14T15:49:38.039Z"
-        cal_data = {"3550": {"20.0": {"false": {"datetime": cal_date},},}}
-        expired = has_expired_cal_data(cal_data, parse_datetime_iso_format_str( now_date), 30)
+        cal_data = {
+            "3550": {
+                "20.0": {
+                    "false": {"datetime": cal_date},
+                },
+            }
+        }
+        expired = has_expired_cal_data(
+            cal_data, parse_datetime_iso_format_str(now_date), 30
+        )
         assert expired == True
 
     def test_has_expired_cal_data_multipledates_expired(self):
         cal_date_1 = "2024-03-14T15:48:38.039Z"
         cal_date_2 = "2024-03-14T15:40:38.039Z"
         now_date = "2024-03-14T15:49:38.039Z"
-        cal_data = {"3550": {"20.0": {"false": {"datetime": cal_date_1},}, "true":{ "datetime": cal_date_2},}}
-        expired = has_expired_cal_data(cal_data, parse_datetime_iso_format_str( now_date), 100)
+        cal_data = {
+            "3550": {
+                "20.0": {
+                    "false": {"datetime": cal_date_1},
+                },
+                "true": {"datetime": cal_date_2},
+            }
+        }
+        expired = has_expired_cal_data(
+            cal_data, parse_datetime_iso_format_str(now_date), 100
+        )
         assert expired == True
-        cal_data = {"3550": {"20.0": {"false": {"datetime": cal_date_2},}, "true":{ "datetime": cal_date_1},}}
-        expired = has_expired_cal_data(cal_data, parse_datetime_iso_format_str( now_date), 100)
\ No newline at end of file
+        cal_data = {
+            "3550": {
+                "20.0": {
+                    "false": {"datetime": cal_date_2},
+                },
+                "true": {"datetime": cal_date_1},
+            }
+        }
+        expired = has_expired_cal_data(
+            cal_data, parse_datetime_iso_format_str(now_date), 100
+        )
diff --git a/scos_actions/hardware/sigan_iface.py b/scos_actions/hardware/sigan_iface.py
index cb0e85cb..8b6d207c 100644
--- a/scos_actions/hardware/sigan_iface.py
+++ b/scos_actions/hardware/sigan_iface.py
@@ -4,6 +4,7 @@
 from typing import Dict, Optional
 
 from its_preselector.web_relay import WebRelay
+
 from scos_actions.hardware.utils import power_cycle_sigan
 
 logger = logging.getLogger(__name__)
diff --git a/scos_actions/hardware/tests/test_sigan.py b/scos_actions/hardware/tests/test_sigan.py
index d4434d78..4fce6562 100644
--- a/scos_actions/hardware/tests/test_sigan.py
+++ b/scos_actions/hardware/tests/test_sigan.py
@@ -1,4 +1,5 @@
 import pytest
+
 from scos_actions.hardware.mocks.mock_sigan import MockSignalAnalyzer
 
 
diff --git a/scos_actions/metadata/structs/capture.py b/scos_actions/metadata/structs/capture.py
index 59fe6128..abcff4fb 100644
--- a/scos_actions/metadata/structs/capture.py
+++ b/scos_actions/metadata/structs/capture.py
@@ -1,6 +1,7 @@
 from typing import Optional
 
 import msgspec
+
 from scos_actions.metadata.structs.ntia_sensor import Calibration, SiganSettings
 from scos_actions.metadata.utils import SIGMF_OBJECT_KWARGS
 
diff --git a/scos_actions/signal_processing/calibration.py b/scos_actions/signal_processing/calibration.py
index 3f98ade8..e1a4c847 100644
--- a/scos_actions/signal_processing/calibration.py
+++ b/scos_actions/signal_processing/calibration.py
@@ -5,6 +5,7 @@
 from its_preselector.preselector import Preselector
 from numpy.typing import NDArray
 from scipy.constants import Boltzmann
+
 from scos_actions.calibration.utils import CalibrationException
 from scos_actions.signal_processing.unit_conversion import (
     convert_celsius_to_fahrenheit,
diff --git a/scos_actions/signal_processing/tests/test_calibration.py b/scos_actions/signal_processing/tests/test_calibration.py
index c47a2a37..6b29b472 100644
--- a/scos_actions/signal_processing/tests/test_calibration.py
+++ b/scos_actions/signal_processing/tests/test_calibration.py
@@ -1,6 +1,7 @@
 """
 Unit test for scos_actions.signal_processing.calibration
 """
+
 import numpy as np
 from scipy.constants import Boltzmann
 
diff --git a/scos_actions/signal_processing/tests/test_fft.py b/scos_actions/signal_processing/tests/test_fft.py
index a848340f..1ea7458c 100644
--- a/scos_actions/signal_processing/tests/test_fft.py
+++ b/scos_actions/signal_processing/tests/test_fft.py
@@ -1,6 +1,7 @@
 """
 Unit test for scos_actions.signal_processing.fft
 """
+
 import numpy as np
 import pytest
 from scipy.signal import get_window
diff --git a/scos_actions/signal_processing/tests/test_filtering.py b/scos_actions/signal_processing/tests/test_filtering.py
index d33c37c6..0df7b845 100644
--- a/scos_actions/signal_processing/tests/test_filtering.py
+++ b/scos_actions/signal_processing/tests/test_filtering.py
@@ -5,6 +5,7 @@
 tests mostly exist to ensure that tests will fail if substantial changes
 are made to the wrappers.
 """
+
 import numpy as np
 import pytest
 from scipy.signal import ellip, ellipord, firwin, kaiserord, sos2zpk, sosfreqz
diff --git a/scos_actions/signal_processing/tests/test_power_analysis.py b/scos_actions/signal_processing/tests/test_power_analysis.py
index 42e2db75..02aaf454 100644
--- a/scos_actions/signal_processing/tests/test_power_analysis.py
+++ b/scos_actions/signal_processing/tests/test_power_analysis.py
@@ -1,6 +1,7 @@
 """
 Unit test for scos_actions.signal_processing.power_analysis
 """
+
 from enum import EnumMeta
 
 import numpy as np
diff --git a/scos_actions/signal_processing/tests/test_unit_conversion.py b/scos_actions/signal_processing/tests/test_unit_conversion.py
index 4da61f86..fe404472 100644
--- a/scos_actions/signal_processing/tests/test_unit_conversion.py
+++ b/scos_actions/signal_processing/tests/test_unit_conversion.py
@@ -1,6 +1,7 @@
 """
 Unit test for scos_actions.signal_processing.unit_conversion
 """
+
 import numpy as np
 import pytest
 

From 15d63d87b92af7a8018386e7ac525e7869cef55c Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Thu, 14 Mar 2024 14:56:41 -0600
Subject: [PATCH 063/102] get CALIBRATION_EXPIRATION_LIMIT as int.

---
 scos_actions/calibration/sensor_calibration.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index f5e03b63..7ccdb03f 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -92,7 +92,7 @@ def update(
 
     def expired(self) -> bool:
         env = Env()
-        time_limit = env("CALIBRATION_EXPIRATION_LIMIT", default=None)
+        time_limit = env.int("CALIBRATION_EXPIRATION_LIMIT", default=None)
         if time_limit is None:
             return False
         elif self.calibration_data is None:

From 57452a6b7d00527701827e88e373c37f5909bc8b Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Thu, 14 Mar 2024 16:05:15 -0600
Subject: [PATCH 064/102] move import of ray into __call__ just in case.

---
 scos_actions/actions/acquire_sea_data_product.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scos_actions/actions/acquire_sea_data_product.py b/scos_actions/actions/acquire_sea_data_product.py
index 560375cf..03e3f764 100644
--- a/scos_actions/actions/acquire_sea_data_product.py
+++ b/scos_actions/actions/acquire_sea_data_product.py
@@ -30,7 +30,6 @@
 
 import numpy as np
 import psutil
-import ray
 from environs import Env
 from its_preselector import __version__ as PRESELECTOR_API_VERSION
 from scipy.signal import sos2tf, sosfilt
@@ -506,6 +505,8 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
         action_start_tic = perf_counter()
         # Ray should have already been initialized within scos-sensor,
         # but check and initialize just in case.
+        import ray
+
         if not ray.is_initialized():
             logger.info("Initializing ray.")
             logger.info("Set RAY_INIT=true to avoid initializing within " + __name__)

From 521518ab91a7bf8dfa9d8bf0b71be0f49171a92c Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Thu, 14 Mar 2024 16:09:21 -0600
Subject: [PATCH 065/102] Restore import or ray.

---
 scos_actions/actions/acquire_sea_data_product.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scos_actions/actions/acquire_sea_data_product.py b/scos_actions/actions/acquire_sea_data_product.py
index 03e3f764..bd2a8d82 100644
--- a/scos_actions/actions/acquire_sea_data_product.py
+++ b/scos_actions/actions/acquire_sea_data_product.py
@@ -23,6 +23,7 @@
 import logging
 import lzma
 import platform
+import ray
 import sys
 from enum import EnumMeta
 from time import perf_counter
@@ -505,7 +506,6 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
         action_start_tic = perf_counter()
         # Ray should have already been initialized within scos-sensor,
         # but check and initialize just in case.
-        import ray
 
         if not ray.is_initialized():
             logger.info("Initializing ray.")

From 23faff1da732ca7ad35da96fdea73d2d6ccf2e94 Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Thu, 14 Mar 2024 16:29:07 -0600
Subject: [PATCH 066/102] Add logging.

---
 scos_actions/calibration/sensor_calibration.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index 7ccdb03f..f2f66718 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -93,6 +93,7 @@ def update(
     def expired(self) -> bool:
         env = Env()
         time_limit = env.int("CALIBRATION_EXPIRATION_LIMIT", default=None)
+        logger.debug("Checking if calibration has expired.")
         if time_limit is None:
             return False
         elif self.calibration_data is None:
@@ -121,5 +122,8 @@ def date_expired(cal_data: dict, now: datetime, time_limit: int):
     cal_datetime = parse_datetime_iso_format_str(cal_data["datetime"])
     elapsed = now - cal_datetime
     if elapsed.total_seconds() > time_limit:
+        logger.debug(
+            f"Calibration {cal_data} has expired at {elapsed.total_seconds()} seconds old."
+        )
         return True
     return False

From 838b9507557775f12a813cfe5d857cdce3150bc9 Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Fri, 15 Mar 2024 09:04:34 -0600
Subject: [PATCH 067/102] Additional debug logging.

---
 scos_actions/calibration/interfaces/calibration.py | 1 +
 scos_actions/calibration/sensor_calibration.py     | 1 +
 2 files changed, 2 insertions(+)

diff --git a/scos_actions/calibration/interfaces/calibration.py b/scos_actions/calibration/interfaces/calibration.py
index b243bea4..011e56b5 100644
--- a/scos_actions/calibration/interfaces/calibration.py
+++ b/scos_actions/calibration/interfaces/calibration.py
@@ -129,3 +129,4 @@ def to_json(self) -> None:
         dict_to_json.pop("file_path", None)
         with open(self.file_path, "w") as outfile:
             outfile.write(json.dumps(dict_to_json))
+        logger.debug("Finished updating calibration file.")
diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index f2f66718..0330b944 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -54,6 +54,7 @@ def update(
         :param temp_degC: Temperature at calibration time, in degrees Celsius.
         :param file_path: File path for saving the updated calibration data.
         """
+        logger.debug("Updating calibration file.")
         try:
             # Get existing calibration data entry which will be updated
             data_entry = self.get_calibration_dict(params)

From 2e0d8c1cbe891a38c15f1ee26a3efac7ef78fe83 Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Fri, 15 Mar 2024 10:50:56 -0600
Subject: [PATCH 068/102] Get rid of hard coded SIGAN_SETTINGS_KEYS. Use sensor
 capabilities to set sensor id in cal file.

---
 scos_actions/actions/calibrate_y_factor.py     | 16 +++++++++++-----
 scos_actions/actions/interfaces/action.py      |  6 ++----
 scos_actions/calibration/sensor_calibration.py |  2 +-
 scos_actions/hardware/sigan_iface.py           | 11 -----------
 scos_actions/hardware/utils.py                 |  6 ++++++
 5 files changed, 20 insertions(+), 21 deletions(-)

diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py
index af00dcbe..d714dcbf 100644
--- a/scos_actions/actions/calibrate_y_factor.py
+++ b/scos_actions/actions/calibrate_y_factor.py
@@ -88,7 +88,7 @@
 from scos_actions.actions.interfaces.action import Action
 from scos_actions.calibration.sensor_calibration import SensorCalibration
 from scos_actions.hardware.sensor import Sensor
-from scos_actions.hardware.sigan_iface import SIGAN_SETTINGS_KEYS
+from scos_actions.hardware.utils import get_sigan_params
 from scos_actions.signal_processing.calibration import (
     get_linear_enr,
     get_temperature,
@@ -219,12 +219,17 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
             # Create a new sensor calibration object and attach it to the sensor.
             # The calibration parameters will be set to the sigan parameters used
             # in the action YAML parameters.
-            logger.debug(f"Creating a new onboard cal object for the sensor.")
-            cal_params = [k for k in self.iteration_params if k in SIGAN_SETTINGS_KEYS]
+            sensor_uid = self.sensor.capabilities["sensor"]["sensor_spec"]["id"]
+            logger.debug(
+                f"Creating a new onboard cal object for the sensor {sensor_uid}."
+            )
+            cal_params = get_sigan_params(
+                self.iteration_params[0], self.sensor.signal_analyzer
+            )
+            logger.debug(f"cal_params: {cal_params}")
             cal_data = dict()
             last_cal_datetime = get_datetime_str_now()
             clock_rate_lookup_by_sample_rate = []
-            sensor_uid = "Sensor calibration file not provided"
             self.sensor.sensor_calibration = SensorCalibration(
                 calibration_parameters=cal_params,
                 calibration_data=cal_data,
@@ -297,7 +302,8 @@ def calibrate(self, params: dict):
         assert (
             sample_rate == noise_off_measurement_result["sample_rate"]
         ), "Sample rate mismatch"
-        sigan_params = {k: v for k, v in params.items() if k in SIGAN_SETTINGS_KEYS}
+        sigan_params = get_sigan_params(params, self.sensor.signal_analyzer)
+        logger.debug(f"sigan_params: {sigan_params}")
         # Apply IIR filtering to both captures if configured
         if self.iir_apply:
             # Estimate of IIR filter ENBW does NOT account for passband ripple in sensor transfer function!
diff --git a/scos_actions/actions/interfaces/action.py b/scos_actions/actions/interfaces/action.py
index 496723dd..0070e777 100644
--- a/scos_actions/actions/interfaces/action.py
+++ b/scos_actions/actions/interfaces/action.py
@@ -4,7 +4,6 @@
 from typing import Optional
 
 from scos_actions.hardware.sensor import Sensor
-from scos_actions.hardware.sigan_iface import SIGAN_SETTINGS_KEYS
 from scos_actions.metadata.sigmf_builder import SigMFBuilder
 from scos_actions.metadata.structs import ntia_scos, ntia_sensor
 from scos_actions.utils import ParameterException, get_parameter
@@ -51,13 +50,12 @@ def sensor(self, value: Sensor):
         self._sensor = value
 
     def configure_sigan(self, params: dict):
-        sigan_params = {k: v for k, v in params.items() if k in SIGAN_SETTINGS_KEYS}
-        for key, value in sigan_params.items():
+        for key, value in params:
             if hasattr(self.sensor.signal_analyzer, key):
                 logger.debug(f"Applying setting to sigan: {key}: {value}")
                 setattr(self.sensor.signal_analyzer, key, value)
             else:
-                logger.warning(f"Sigan does not have attribute {key}")
+                logger.debug(f"Sigan does not have attribute {key}")
 
     def configure_preselector(self, params: dict):
         preselector = self.sensor.preselector
diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index 0330b944..3f64454c 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -54,7 +54,7 @@ def update(
         :param temp_degC: Temperature at calibration time, in degrees Celsius.
         :param file_path: File path for saving the updated calibration data.
         """
-        logger.debug("Updating calibration file.")
+        logger.debug(f"Updating calibration file for params {params}")
         try:
             # Get existing calibration data entry which will be updated
             data_entry = self.get_calibration_dict(params)
diff --git a/scos_actions/hardware/sigan_iface.py b/scos_actions/hardware/sigan_iface.py
index 8b6d207c..bbf42de6 100644
--- a/scos_actions/hardware/sigan_iface.py
+++ b/scos_actions/hardware/sigan_iface.py
@@ -10,17 +10,6 @@
 logger = logging.getLogger(__name__)
 
 
-# All setting names for all supported sigans
-SIGAN_SETTINGS_KEYS = [
-    "sample_rate",
-    "frequency",
-    "gain",
-    "attenuation",
-    "reference_level",
-    "preamp_enable",
-]
-
-
 class SignalAnalyzerInterface(ABC):
     def __init__(
         self,
diff --git a/scos_actions/hardware/utils.py b/scos_actions/hardware/utils.py
index 49642da7..c6438911 100644
--- a/scos_actions/hardware/utils.py
+++ b/scos_actions/hardware/utils.py
@@ -8,6 +8,7 @@
 from scos_actions.hardware.hardware_configuration_exception import (
     HardwareConfigurationException,
 )
+from scos_actions.hardware.sigan_iface import SignalAnalyzerInterface
 from scos_actions.settings import SIGAN_POWER_CYCLE_STATES, SIGAN_POWER_SWITCH
 
 logger = logging.getLogger(__name__)
@@ -162,3 +163,8 @@ def power_cycle_sigan(switches: Dict[str, WebRelay]):
         raise HardwareConfigurationException(
             "Call to power cycle sigan, but no power switch or power cycle states specified "
         )
+
+
+def get_sigan_params(params: dict, sigan: SignalAnalyzerInterface) -> list:
+    sigan_params = [k for k in params.keys() if hasattr(sigan, k)]
+    return sigan_params

From 54e2b39c6c52669f75903436f25c4962d31b6dda Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Fri, 15 Mar 2024 11:02:32 -0600
Subject: [PATCH 069/102] move get_sigan_params into yfactor action.

---
 scos_actions/actions/calibrate_y_factor.py | 11 ++++++++---
 scos_actions/hardware/utils.py             |  6 ------
 2 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py
index d714dcbf..b9443109 100644
--- a/scos_actions/actions/calibrate_y_factor.py
+++ b/scos_actions/actions/calibrate_y_factor.py
@@ -88,7 +88,8 @@
 from scos_actions.actions.interfaces.action import Action
 from scos_actions.calibration.sensor_calibration import SensorCalibration
 from scos_actions.hardware.sensor import Sensor
-from scos_actions.hardware.utils import get_sigan_params
+from scos_actions.hardware.sigan_iface import SignalAnalyzerInterface
+
 from scos_actions.signal_processing.calibration import (
     get_linear_enr,
     get_temperature,
@@ -223,7 +224,7 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
             logger.debug(
                 f"Creating a new onboard cal object for the sensor {sensor_uid}."
             )
-            cal_params = get_sigan_params(
+            cal_params = self.get_sigan_params(
                 self.iteration_params[0], self.sensor.signal_analyzer
             )
             logger.debug(f"cal_params: {cal_params}")
@@ -302,7 +303,7 @@ def calibrate(self, params: dict):
         assert (
             sample_rate == noise_off_measurement_result["sample_rate"]
         ), "Sample rate mismatch"
-        sigan_params = get_sigan_params(params, self.sensor.signal_analyzer)
+        sigan_params = self.get_sigan_params(params, self.sensor.signal_analyzer)
         logger.debug(f"sigan_params: {sigan_params}")
         # Apply IIR filtering to both captures if configured
         if self.iir_apply:
@@ -436,3 +437,7 @@ def test_required_components(self):
             raise RuntimeError(msg)
         if not self.sensor.signal_analyzer.healthy():
             trigger_api_restart.send(sender=self.__class__)
+
+    def get_sigan_params(params: dict, sigan: SignalAnalyzerInterface) -> list:
+        sigan_params = [k for k in params.keys() if hasattr(sigan, k)]
+        return sigan_params
diff --git a/scos_actions/hardware/utils.py b/scos_actions/hardware/utils.py
index c6438911..49642da7 100644
--- a/scos_actions/hardware/utils.py
+++ b/scos_actions/hardware/utils.py
@@ -8,7 +8,6 @@
 from scos_actions.hardware.hardware_configuration_exception import (
     HardwareConfigurationException,
 )
-from scos_actions.hardware.sigan_iface import SignalAnalyzerInterface
 from scos_actions.settings import SIGAN_POWER_CYCLE_STATES, SIGAN_POWER_SWITCH
 
 logger = logging.getLogger(__name__)
@@ -163,8 +162,3 @@ def power_cycle_sigan(switches: Dict[str, WebRelay]):
         raise HardwareConfigurationException(
             "Call to power cycle sigan, but no power switch or power cycle states specified "
         )
-
-
-def get_sigan_params(params: dict, sigan: SignalAnalyzerInterface) -> list:
-    sigan_params = [k for k in params.keys() if hasattr(sigan, k)]
-    return sigan_params

From a278bf8553b1f5d94ff6214222d7ccebd50871fa Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Fri, 15 Mar 2024 11:05:43 -0600
Subject: [PATCH 070/102] fix get_sigan_params.

---
 scos_actions/actions/calibrate_y_factor.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py
index b9443109..1091cbac 100644
--- a/scos_actions/actions/calibrate_y_factor.py
+++ b/scos_actions/actions/calibrate_y_factor.py
@@ -438,6 +438,6 @@ def test_required_components(self):
         if not self.sensor.signal_analyzer.healthy():
             trigger_api_restart.send(sender=self.__class__)
 
-    def get_sigan_params(params: dict, sigan: SignalAnalyzerInterface) -> list:
+    def get_sigan_params(self, params: dict, sigan: SignalAnalyzerInterface) -> list:
         sigan_params = [k for k in params.keys() if hasattr(sigan, k)]
         return sigan_params

From 0ed012b2c2b1160bfebab8323caac8c79318edb5 Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Fri, 15 Mar 2024 11:12:06 -0600
Subject: [PATCH 071/102] fix configure_sigan.

---
 scos_actions/actions/interfaces/action.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scos_actions/actions/interfaces/action.py b/scos_actions/actions/interfaces/action.py
index 0070e777..16a51fc9 100644
--- a/scos_actions/actions/interfaces/action.py
+++ b/scos_actions/actions/interfaces/action.py
@@ -50,7 +50,7 @@ def sensor(self, value: Sensor):
         self._sensor = value
 
     def configure_sigan(self, params: dict):
-        for key, value in params:
+        for key, value in params.items():
             if hasattr(self.sensor.signal_analyzer, key):
                 logger.debug(f"Applying setting to sigan: {key}: {value}")
                 setattr(self.sensor.signal_analyzer, key, value)

From d49345b9fef8491ff3f153f7a0ca0728c51ee3bc Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Fri, 15 Mar 2024 11:24:02 -0600
Subject: [PATCH 072/102] Change get_sigan_params to return a dictionary of
 keys and values for properties that exist in sigan.

---
 scos_actions/actions/calibrate_y_factor.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py
index 1091cbac..10b7c7c8 100644
--- a/scos_actions/actions/calibrate_y_factor.py
+++ b/scos_actions/actions/calibrate_y_factor.py
@@ -224,8 +224,10 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
             logger.debug(
                 f"Creating a new onboard cal object for the sensor {sensor_uid}."
             )
-            cal_params = self.get_sigan_params(
-                self.iteration_params[0], self.sensor.signal_analyzer
+            cal_params = list(
+                self.get_sigan_params(
+                    self.iteration_params[0], self.sensor.signal_analyzer
+                ).keys()
             )
             logger.debug(f"cal_params: {cal_params}")
             cal_data = dict()
@@ -438,6 +440,10 @@ def test_required_components(self):
         if not self.sensor.signal_analyzer.healthy():
             trigger_api_restart.send(sender=self.__class__)
 
-    def get_sigan_params(self, params: dict, sigan: SignalAnalyzerInterface) -> list:
-        sigan_params = [k for k in params.keys() if hasattr(sigan, k)]
+    def get_sigan_params(self, params: dict, sigan: SignalAnalyzerInterface) -> dict:
+        sigan_params = {}
+        for k, v in params.items():
+            if hasattr(sigan, k):
+                sigan_params[k] = v
+
         return sigan_params

From 5d5289f63c41e112b60d588ecf54a4caa0dd4cbf Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Sun, 17 Mar 2024 13:11:13 -0600
Subject: [PATCH 073/102] Comment out final call to save cal file since it
 saves it every time. Always use lower case strings when updating and
 retrieving cal data. Only print cal data if scaling samples.

---
 scos_actions/actions/calibrate_y_factor.py    |  3 +--
 .../calibration/interfaces/calibration.py     |  2 +-
 .../calibration/sensor_calibration.py         |  2 +-
 scos_actions/hardware/sensor.py               | 20 +++++++++----------
 4 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py
index 10b7c7c8..c16d054f 100644
--- a/scos_actions/actions/calibrate_y_factor.py
+++ b/scos_actions/actions/calibrate_y_factor.py
@@ -89,7 +89,6 @@
 from scos_actions.calibration.sensor_calibration import SensorCalibration
 from scos_actions.hardware.sensor import Sensor
 from scos_actions.hardware.sigan_iface import SignalAnalyzerInterface
-
 from scos_actions.signal_processing.calibration import (
     get_linear_enr,
     get_temperature,
@@ -263,7 +262,7 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
             else:
                 detail += os.linesep + self.calibrate(p)
         # Save results to onboard calibration file
-        self.sensor.sensor_calibration.to_json()
+        # self.sensor.sensor_calibration.to_json()
         return detail
 
     def calibrate(self, params: dict):
diff --git a/scos_actions/calibration/interfaces/calibration.py b/scos_actions/calibration/interfaces/calibration.py
index 011e56b5..d89e1c50 100644
--- a/scos_actions/calibration/interfaces/calibration.py
+++ b/scos_actions/calibration/interfaces/calibration.py
@@ -61,7 +61,7 @@ def get_calibration_dict(self, params: dict) -> dict:
             )
         cal_data = self.calibration_data
         for p_name in self.calibration_parameters:
-            p_value = params[p_name]
+            p_value = str(params[p_name]).lower()
             logger.debug(f"Looking up calibration data at {p_name}={p_value}")
             cal_data = filter_by_parameter(cal_data, p_value)
 
diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index 3f64454c..8499ac77 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -62,7 +62,7 @@ def update(
             # Existing entry does not exist for these parameters. Make one.
             data_entry = self.calibration_data
             for p_name in self.calibration_parameters:
-                p_val = params[p_name]
+                p_val = str(params[p_name]).lower()
                 try:
                     data_entry = data_entry[p_val]
                 except KeyError:
diff --git a/scos_actions/hardware/sensor.py b/scos_actions/hardware/sensor.py
index 08dbcd9c..2d767165 100644
--- a/scos_actions/hardware/sensor.py
+++ b/scos_actions/hardware/sensor.py
@@ -280,14 +280,6 @@ def acquire_time_domain_samples(
         logger.debug("***********************************\n")
         logger.debug("Sensor.acquire_time_domain_samples starting")
         logger.debug(f"Number of retries = {retries}")
-        if self.differential_calibration is not None:
-            logger.debug(
-                f"USING DIFF. CAL: {self.differential_calibration.calibration_data}"
-            )
-        if self.sensor_calibration is not None:
-            logger.debug(
-                f"USING SENSOR CAL: {self.sensor_calibration.calibration_data}"
-            )
         logger.debug("*************************************\n")
 
         max_retries = retries
@@ -301,7 +293,7 @@ def acquire_time_domain_samples(
                         )
                     )
                     break
-                except Exception as e:
+                except BaseException as e:
                     retries -= 1
                     logger.info("Error while acquiring samples from signal analyzer.")
                     if retries == 0:
@@ -322,8 +314,16 @@ def acquire_time_domain_samples(
                     "Data scaling cannot occur without specified calibration parameters."
                 )
             if self.sensor_calibration is not None:
-                logger.debug("Scaling samples using calibration data")
+                logger.debug("Scaling samples. Fetching calibration data.")
                 self.recompute_calibration_data(cal_params)
+                if self.differential_calibration is not None:
+                    logger.debug(
+                        f"USING DIFF. CAL: {self.differential_calibration.calibration_data}"
+                    )
+                if self.sensor_calibration is not None:
+                    logger.debug(
+                        f"USING SENSOR CAL: {self.sensor_calibration.calibration_data}"
+                    )
                 calibrated_gain__db = self.sensor_calibration_data["gain"]
                 calibrated_nf__db = self.sensor_calibration_data["noise_figure"]
                 logger.debug(f"Using sensor gain: {calibrated_gain__db} dB")

From 40d6d0a64c7a02688b0cee844760999c62aeacd8 Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Sun, 17 Mar 2024 13:57:16 -0600
Subject: [PATCH 074/102] additional debug logging.

---
 scos_actions/calibration/sensor_calibration.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index 8499ac77..daaceb55 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -122,6 +122,7 @@ def has_expired_cal_data(cal_data: dict, now: datetime, time_limit: int) -> bool
 def date_expired(cal_data: dict, now: datetime, time_limit: int):
     cal_datetime = parse_datetime_iso_format_str(cal_data["datetime"])
     elapsed = now - cal_datetime
+    logger.debug(f"{cal_datetime} is {elapsed} seconds old")
     if elapsed.total_seconds() > time_limit:
         logger.debug(
             f"Calibration {cal_data} has expired at {elapsed.total_seconds()} seconds old."

From 13e502a0f751e6f67d052d7395ccc022dd99611f Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Sun, 17 Mar 2024 14:09:08 -0600
Subject: [PATCH 075/102] Check last_calibration_datetime for expiration as
 well.

---
 scos_actions/calibration/sensor_calibration.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index daaceb55..b2ce5c7c 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -95,15 +95,17 @@ def expired(self) -> bool:
         env = Env()
         time_limit = env.int("CALIBRATION_EXPIRATION_LIMIT", default=None)
         logger.debug("Checking if calibration has expired.")
+        now_string = datetime.utcnow().isoformat(timespec="milliseconds") + "Z"
+        now = parse_datetime_iso_format_str(now_string)
         if time_limit is None:
             return False
         elif self.calibration_data is None:
             return True
         elif len(self.calibration_data) == 0:
             return True
+        elif date_expired(self.last_calibration_datetime, now, time_limit):
+            return True
         else:
-            now_string = datetime.utcnow().isoformat(timespec="milliseconds") + "Z"
-            now = parse_datetime_iso_format_str(now_string)
             cal_data = self.calibration_data
             return has_expired_cal_data(cal_data, now, time_limit)
 
@@ -111,7 +113,7 @@ def expired(self) -> bool:
 def has_expired_cal_data(cal_data: dict, now: datetime, time_limit: int) -> bool:
     expired = False
     if "datetime" in cal_data:
-        expired = expired or date_expired(cal_data, now, time_limit)
+        expired = expired or date_expired(cal_data["datetime"], now, time_limit)
 
     for key, value in cal_data.items():
         if isinstance(value, dict):
@@ -119,8 +121,8 @@ def has_expired_cal_data(cal_data: dict, now: datetime, time_limit: int) -> bool
     return expired
 
 
-def date_expired(cal_data: dict, now: datetime, time_limit: int):
-    cal_datetime = parse_datetime_iso_format_str(cal_data["datetime"])
+def date_expired(cal_date: str, now: datetime, time_limit: int):
+    cal_datetime = parse_datetime_iso_format_str(cal_date)
     elapsed = now - cal_datetime
     logger.debug(f"{cal_datetime} is {elapsed} seconds old")
     if elapsed.total_seconds() > time_limit:

From 28e99172f5e99cfd14c67631beb71c9ccf3b1230 Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Sun, 17 Mar 2024 14:12:29 -0600
Subject: [PATCH 076/102] fix debug statement.

---
 scos_actions/calibration/sensor_calibration.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index b2ce5c7c..3224331e 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -127,7 +127,7 @@ def date_expired(cal_date: str, now: datetime, time_limit: int):
     logger.debug(f"{cal_datetime} is {elapsed} seconds old")
     if elapsed.total_seconds() > time_limit:
         logger.debug(
-            f"Calibration {cal_data} has expired at {elapsed.total_seconds()} seconds old."
+            f"Calibration at {cal_date} has expired at {elapsed.total_seconds()} seconds old."
         )
         return True
     return False

From fed433657110efe080dfccfc46371aef6ec79198 Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Sun, 17 Mar 2024 16:50:33 -0600
Subject: [PATCH 077/102] Add comporession point in sea data product if it
 exists in the cal data. Update calibration metadata in iq, m4, and stepped IQ
 actions to account for new differential cal.

---
 .../actions/acquire_sea_data_product.py       |  6 +++-
 .../actions/acquire_single_freq_fft.py        | 10 ++----
 .../actions/acquire_single_freq_tdomain_iq.py |  5 +--
 .../acquire_stepped_freq_tdomain_iq.py        | 11 ++----
 .../actions/interfaces/measurement_action.py  | 35 ++++++++++++-------
 5 files changed, 33 insertions(+), 34 deletions(-)

diff --git a/scos_actions/actions/acquire_sea_data_product.py b/scos_actions/actions/acquire_sea_data_product.py
index bd2a8d82..3097efcf 100644
--- a/scos_actions/actions/acquire_sea_data_product.py
+++ b/scos_actions/actions/acquire_sea_data_product.py
@@ -23,7 +23,6 @@
 import logging
 import lzma
 import platform
-import ray
 import sys
 from enum import EnumMeta
 from time import perf_counter
@@ -31,6 +30,7 @@
 
 import numpy as np
 import psutil
+import ray
 from environs import Env
 from its_preselector import __version__ as PRESELECTOR_API_VERSION
 from scipy.signal import sos2tf, sosfilt
@@ -1151,6 +1151,10 @@ def create_capture_segment(
                 preamp_enable=self.sensor.signal_analyzer.preamp_enable,
             ),
         )
+        if "compression_point" in measurement_result:
+            capture_segment.sensor_calibration.compression_point = measurement_result[
+                "compression_point"
+            ]
         self.sigmf_builder.add_capture(capture_segment)
 
     def get_sigmf_builder(
diff --git a/scos_actions/actions/acquire_single_freq_fft.py b/scos_actions/actions/acquire_single_freq_fft.py
index 25f0b3f3..5859dc3e 100644
--- a/scos_actions/actions/acquire_single_freq_fft.py
+++ b/scos_actions/actions/acquire_single_freq_fft.py
@@ -173,22 +173,18 @@ def execute(self, schedule_entry: dict, task_id: int) -> dict:
         # Save measurement results
         measurement_result["data"] = m4s_result
         measurement_result.update(self.parameters)
-        # measurement_result["calibration_datetime"] = (
-        #     self.sensor.sensor_calibration_data["datetime"]
-        # )
         measurement_result["task_id"] = task_id
         measurement_result["classification"] = self.classification
 
         # Build capture metadata
         sigan_settings = self.get_sigan_settings(measurement_result)
         logger.debug(f"sigan settings:{sigan_settings}")
+        measurement_result["duration_ms"] = int(self.num_samples / sample_rate_Hz)
+        measurement_result["center_frequency_Hz"] = self.frequency_Hz
         measurement_result["capture_segment"] = self.create_capture_segment(
             sample_start=0,
-            start_time=measurement_result["capture_time"],
-            center_frequency_Hz=self.frequency_Hz,
-            duration_ms=int(self.num_samples / sample_rate_Hz),
-            overload=measurement_result["overload"],
             sigan_settings=sigan_settings,
+            measurement_result=measurement_result,
         )
 
         return measurement_result
diff --git a/scos_actions/actions/acquire_single_freq_tdomain_iq.py b/scos_actions/actions/acquire_single_freq_tdomain_iq.py
index 8ddaa11a..50d57e78 100644
--- a/scos_actions/actions/acquire_single_freq_tdomain_iq.py
+++ b/scos_actions/actions/acquire_single_freq_tdomain_iq.py
@@ -98,11 +98,8 @@ def execute(self, schedule_entry: dict, task_id: int) -> dict:
         logger.debug(f"sigan settings:{sigan_settings}")
         measurement_result["capture_segment"] = self.create_capture_segment(
             sample_start=0,
-            start_time=measurement_result["capture_time"],
-            center_frequency_Hz=self.frequency_Hz,
-            duration_ms=self.duration_ms,
-            overload=measurement_result["overload"],
             sigan_settings=sigan_settings,
+            measurement_result=measurement_result,
         )
         return measurement_result
 
diff --git a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
index 6de6c5a9..e8cc1d5f 100644
--- a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
+++ b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
@@ -121,15 +121,8 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
             )
             sensor_cal = self.sensor.sensor_calibration_data
             if sensor_cal is not None:
-                if "1db_compression_point" in sensor_cal:
-                    sensor_cal["compression_point"] = sensor_cal.pop(
-                        "1db_compression_point"
-                    )
-                if "reference" not in sensor_cal:
-                    # If the calibration data already includes this, don't overwrite
-                    sensor_cal["reference"] = measurement_result["reference"]
-                capture_segment.sensor_calibration = ntia_sensor.Calibration(
-                    **sensor_cal
+                capture_segment.sensor_calibration = self.get_calibration(
+                    measurement_result
                 )
             measurement_result["capture_segment"] = capture_segment
 
diff --git a/scos_actions/actions/interfaces/measurement_action.py b/scos_actions/actions/interfaces/measurement_action.py
index c8fb1847..117a9154 100644
--- a/scos_actions/actions/interfaces/measurement_action.py
+++ b/scos_actions/actions/interfaces/measurement_action.py
@@ -38,31 +38,40 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
     def create_capture_segment(
         self,
         sample_start: int,
-        start_time: str,
-        center_frequency_Hz: float,
-        duration_ms: int,
-        overload: bool,
         sigan_settings: Optional[ntia_sensor.SiganSettings],
+        measurement_result: dict,
     ) -> CaptureSegment:
         capture_segment = CaptureSegment(
             sample_start=sample_start,
-            frequency=center_frequency_Hz,
-            datetime=start_time,
-            duration=duration_ms,
-            overload=overload,
+            frequency=measurement_result["center_frequency_Hz"],
+            datetime=measurement_result["start_time"],
+            duration=measurement_result["duration_ms"],
+            overload=measurement_result["overload"],
             sigan_settings=sigan_settings,
         )
         sensor_cal = self.sensor.sensor_calibration_data
         # Rename compression point keys if they exist
         # then set calibration metadata if it exists
         if sensor_cal is not None:
-            if "1db_compression_point" in sensor_cal:
-                sensor_cal["compression_point"] = sensor_cal.pop(
-                    "1db_compression_point"
-                )
-            capture_segment.sensor_calibration = ntia_sensor.Calibration(**sensor_cal)
+            capture_segment.sensor_calibration = self.get_calibration(
+                measurement_result
+            )
         return capture_segment
 
+    def get_calibration(self, measurement_result: dict) -> ntia_sensor.Calibration:
+        cal_meta = ntia_sensor.Calibration(
+            datetime=self.sensor.sensor_calibration_data["datetime"],
+            gain=round(measurement_result["applied_calibration"]["gain"], 3),
+            noise_figure=round(
+                measurement_result["applied_calibration"]["noise_figure"], 3
+            ),
+            temperature=round(self.sensor.sensor_calibration_data["temperature"], 1),
+            reference=measurement_result["reference"],
+        )
+        if "comporession_point" in measurement_result:
+            cal_meta.compression_point = measurement_result["compression_point"]
+        return cal_meta
+
     def create_metadata(
         self,
         measurement_result: dict,

From 6217ca33a78c98f1e16809d50af08ad724e92a74 Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Mon, 18 Mar 2024 07:37:51 -0600
Subject: [PATCH 078/102] use sigan frequency in measurement action capture.

---
 scos_actions/actions/interfaces/measurement_action.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scos_actions/actions/interfaces/measurement_action.py b/scos_actions/actions/interfaces/measurement_action.py
index 117a9154..3b136dc1 100644
--- a/scos_actions/actions/interfaces/measurement_action.py
+++ b/scos_actions/actions/interfaces/measurement_action.py
@@ -43,7 +43,7 @@ def create_capture_segment(
     ) -> CaptureSegment:
         capture_segment = CaptureSegment(
             sample_start=sample_start,
-            frequency=measurement_result["center_frequency_Hz"],
+            frequency=self.sensor.signal_analyzer.frequency,
             datetime=measurement_result["start_time"],
             duration=measurement_result["duration_ms"],
             overload=measurement_result["overload"],

From 413fd771ecb089937f6ef187fdc9b23664de4aa5 Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Mon, 18 Mar 2024 07:45:45 -0600
Subject: [PATCH 079/102] fix capture_time.

---
 scos_actions/actions/interfaces/measurement_action.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scos_actions/actions/interfaces/measurement_action.py b/scos_actions/actions/interfaces/measurement_action.py
index 3b136dc1..2784129f 100644
--- a/scos_actions/actions/interfaces/measurement_action.py
+++ b/scos_actions/actions/interfaces/measurement_action.py
@@ -44,7 +44,7 @@ def create_capture_segment(
         capture_segment = CaptureSegment(
             sample_start=sample_start,
             frequency=self.sensor.signal_analyzer.frequency,
-            datetime=measurement_result["start_time"],
+            datetime=measurement_result["capture_time"],
             duration=measurement_result["duration_ms"],
             overload=measurement_result["overload"],
             sigan_settings=sigan_settings,

From ea5156b1f7c28c5884d90719a3780ba03cedac12 Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Mon, 18 Mar 2024 08:07:06 -0600
Subject: [PATCH 080/102] Typo fix.

---
 scos_actions/actions/interfaces/measurement_action.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scos_actions/actions/interfaces/measurement_action.py b/scos_actions/actions/interfaces/measurement_action.py
index 2784129f..dcc04de5 100644
--- a/scos_actions/actions/interfaces/measurement_action.py
+++ b/scos_actions/actions/interfaces/measurement_action.py
@@ -68,7 +68,7 @@ def get_calibration(self, measurement_result: dict) -> ntia_sensor.Calibration:
             temperature=round(self.sensor.sensor_calibration_data["temperature"], 1),
             reference=measurement_result["reference"],
         )
-        if "comporession_point" in measurement_result:
+        if "compression_point" in measurement_result:
             cal_meta.compression_point = measurement_result["compression_point"]
         return cal_meta
 

From db3fefebb4a6045e0d696c7c7e3d7c9407d420a3 Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Mon, 18 Mar 2024 08:38:09 -0600
Subject: [PATCH 081/102] put the compression point in the applied calibration.

---
 scos_actions/actions/interfaces/measurement_action.py | 6 ++++--
 scos_actions/hardware/sensor.py                       | 6 ++++++
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/scos_actions/actions/interfaces/measurement_action.py b/scos_actions/actions/interfaces/measurement_action.py
index dcc04de5..ee429e48 100644
--- a/scos_actions/actions/interfaces/measurement_action.py
+++ b/scos_actions/actions/interfaces/measurement_action.py
@@ -68,8 +68,10 @@ def get_calibration(self, measurement_result: dict) -> ntia_sensor.Calibration:
             temperature=round(self.sensor.sensor_calibration_data["temperature"], 1),
             reference=measurement_result["reference"],
         )
-        if "compression_point" in measurement_result:
-            cal_meta.compression_point = measurement_result["compression_point"]
+        if "compression_point" in measurement_result["applied_calibration"]:
+            cal_meta.compression_point = measurement_result["applied_calibration"][
+                "compression_point"
+            ]
         return cal_meta
 
     def create_metadata(
diff --git a/scos_actions/hardware/sensor.py b/scos_actions/hardware/sensor.py
index 2d767165..8a72ba08 100644
--- a/scos_actions/hardware/sensor.py
+++ b/scos_actions/hardware/sensor.py
@@ -339,6 +339,7 @@ def acquire_time_domain_samples(
                     measurement_result["reference"] = (
                         self.differential_calibration.calibration_reference
                     )
+
                 else:
                     # No differential calibration exists
                     logger.debug("No differential calibration was applied")
@@ -353,6 +354,11 @@ def acquire_time_domain_samples(
                     "gain": calibrated_gain__db,
                     "noise_figure": calibrated_nf__db,
                 }
+                if "compression_point" in self.sensor_calibration_data:
+                    measurement_result["applied_calibration"]["compression_point"] = (
+                        self.sensor_calibration_data["compression_point"]
+                    )
+
             else:
                 # No sensor calibration exists
                 msg = "Unable to scale samples without sensor calibration data"

From 83fcd1115d83a34a5cd8dcc6ea7f23d7f823469a Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Mon, 18 Mar 2024 08:49:22 -0600
Subject: [PATCH 082/102] Get compression_point from applied_calibration in sea
 action.

---
 scos_actions/actions/acquire_sea_data_product.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scos_actions/actions/acquire_sea_data_product.py b/scos_actions/actions/acquire_sea_data_product.py
index 3097efcf..4c5a3355 100644
--- a/scos_actions/actions/acquire_sea_data_product.py
+++ b/scos_actions/actions/acquire_sea_data_product.py
@@ -1151,10 +1151,10 @@ def create_capture_segment(
                 preamp_enable=self.sensor.signal_analyzer.preamp_enable,
             ),
         )
-        if "compression_point" in measurement_result:
+        if "compression_point" in measurement_result["applied_calibration"]:
             capture_segment.sensor_calibration.compression_point = measurement_result[
-                "compression_point"
-            ]
+                "applied_calibration"
+            ]["compression_point"]
         self.sigmf_builder.add_capture(capture_segment)
 
     def get_sigmf_builder(

From 83cf9de1440cc5a23865320082eedc91d5dbd46c Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Tue, 19 Mar 2024 12:13:58 -0600
Subject: [PATCH 083/102] Fix tests.

---
 .../acquire_stepped_freq_tdomain_iq.py        |  5 ++-
 .../actions/interfaces/measurement_action.py  | 39 ++++++++++++-------
 .../tests/test_sensor_calibration.py          |  2 +-
 3 files changed, 30 insertions(+), 16 deletions(-)

diff --git a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
index bfb06d55..199473d3 100644
--- a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
+++ b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
@@ -119,7 +119,10 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
                 sigan_settings=sigan_settings,
             )
             sensor_cal = self.sensor.sensor_calibration_data
-            if sensor_cal is not None:
+            if (
+                sensor_cal is not None
+                and measurement_result["applied_calibration"] is not None
+            ):
                 capture_segment.sensor_calibration = self.get_calibration(
                     measurement_result
                 )
diff --git a/scos_actions/actions/interfaces/measurement_action.py b/scos_actions/actions/interfaces/measurement_action.py
index ee429e48..e3081c7f 100644
--- a/scos_actions/actions/interfaces/measurement_action.py
+++ b/scos_actions/actions/interfaces/measurement_action.py
@@ -52,26 +52,37 @@ def create_capture_segment(
         sensor_cal = self.sensor.sensor_calibration_data
         # Rename compression point keys if they exist
         # then set calibration metadata if it exists
-        if sensor_cal is not None:
+        if (
+            sensor_cal is not None
+            and measurement_result["applied_calibration"] is not None
+        ):
             capture_segment.sensor_calibration = self.get_calibration(
                 measurement_result
             )
         return capture_segment
 
     def get_calibration(self, measurement_result: dict) -> ntia_sensor.Calibration:
-        cal_meta = ntia_sensor.Calibration(
-            datetime=self.sensor.sensor_calibration_data["datetime"],
-            gain=round(measurement_result["applied_calibration"]["gain"], 3),
-            noise_figure=round(
-                measurement_result["applied_calibration"]["noise_figure"], 3
-            ),
-            temperature=round(self.sensor.sensor_calibration_data["temperature"], 1),
-            reference=measurement_result["reference"],
-        )
-        if "compression_point" in measurement_result["applied_calibration"]:
-            cal_meta.compression_point = measurement_result["applied_calibration"][
-                "compression_point"
-            ]
+        cal_meta = None
+        if (
+            self.sensor.sensor_calibration_data
+            is measurement_result["applied_calibration"]
+            is not None
+        ):
+            cal_meta = ntia_sensor.Calibration(
+                datetime=self.sensor.sensor_calibration_data["datetime"],
+                gain=round(measurement_result["applied_calibration"]["gain"], 3),
+                noise_figure=round(
+                    measurement_result["applied_calibration"]["noise_figure"], 3
+                ),
+                temperature=round(
+                    self.sensor.sensor_calibration_data["temperature"], 1
+                ),
+                reference=measurement_result["reference"],
+            )
+            if "compression_point" in measurement_result["applied_calibration"]:
+                cal_meta.compression_point = measurement_result["applied_calibration"][
+                    "compression_point"
+                ]
         return cal_meta
 
     def create_metadata(
diff --git a/scos_actions/calibration/tests/test_sensor_calibration.py b/scos_actions/calibration/tests/test_sensor_calibration.py
index 01d70854..3e3006f5 100644
--- a/scos_actions/calibration/tests/test_sensor_calibration.py
+++ b/scos_actions/calibration/tests/test_sensor_calibration.py
@@ -280,7 +280,7 @@ def test_get_calibration_dict_within_range(self):
             _ = cal.get_calibration_dict({"sample_rate": 100.0, "frequency": 250.0})
         assert e_info.value.args[0] == (
             f"Could not locate calibration data at 250.0"
-            + f"\nAttempted lookup using key '250.0' and 250.0"
+            + f"\nAttempted lookup using key '250.0'"
             + f"\nUsing calibration data: {cal.calibration_data['100.0']}"
         )
 

From aaaeeb938600dcdd814edf462f22fe196009b309 Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Thu, 21 Mar 2024 14:20:42 -0600
Subject: [PATCH 084/102] Restore type hint in acquire_sea_data_product. Fix
 duration ms computation. Remove commented out code. Reuse
 create_capture_segment in stepped frequency IQ action. Use frequency from
 measurement_result in MeasurementAction. Fix description of the calibration
 data that will be used in the docstring in Sensor
 acquire_time_domain_samples.

---
 .../actions/acquire_sea_data_product.py         |  2 +-
 scos_actions/actions/acquire_single_freq_fft.py |  4 +++-
 .../actions/acquire_single_freq_tdomain_iq.py   |  3 ---
 .../actions/acquire_stepped_freq_tdomain_iq.py  | 17 ++---------------
 scos_actions/actions/calibrate_y_factor.py      |  2 --
 .../actions/interfaces/measurement_action.py    |  5 ++---
 scos_actions/hardware/sensor.py                 | 11 ++++++++---
 7 files changed, 16 insertions(+), 28 deletions(-)

diff --git a/scos_actions/actions/acquire_sea_data_product.py b/scos_actions/actions/acquire_sea_data_product.py
index caeb5688..d7b54c99 100644
--- a/scos_actions/actions/acquire_sea_data_product.py
+++ b/scos_actions/actions/acquire_sea_data_product.py
@@ -425,7 +425,7 @@ def __init__(self, params: dict, iir_sos: np.ndarray):
         ]
         del params
 
-    def run(self, iqdata: np.ndarray):
+    def run(self, iqdata: np.ndarray) -> list:
         """
         Filter the input IQ data and concurrently compute FFT, PVT, PFP, and APD results.
 
diff --git a/scos_actions/actions/acquire_single_freq_fft.py b/scos_actions/actions/acquire_single_freq_fft.py
index 5859dc3e..d99ba8dd 100644
--- a/scos_actions/actions/acquire_single_freq_fft.py
+++ b/scos_actions/actions/acquire_single_freq_fft.py
@@ -179,7 +179,9 @@ def execute(self, schedule_entry: dict, task_id: int) -> dict:
         # Build capture metadata
         sigan_settings = self.get_sigan_settings(measurement_result)
         logger.debug(f"sigan settings:{sigan_settings}")
-        measurement_result["duration_ms"] = int(self.num_samples / sample_rate_Hz)
+        measurement_result["duration_ms"] = round(
+            (self.num_samples / sample_rate_Hz) * 1000
+        )
         measurement_result["center_frequency_Hz"] = self.frequency_Hz
         measurement_result["capture_segment"] = self.create_capture_segment(
             sample_start=0,
diff --git a/scos_actions/actions/acquire_single_freq_tdomain_iq.py b/scos_actions/actions/acquire_single_freq_tdomain_iq.py
index 50d57e78..faa3ece9 100644
--- a/scos_actions/actions/acquire_single_freq_tdomain_iq.py
+++ b/scos_actions/actions/acquire_single_freq_tdomain_iq.py
@@ -90,9 +90,6 @@ def execute(self, schedule_entry: dict, task_id: int) -> dict:
         measurement_result.update(self.parameters)
         measurement_result["end_time"] = end_time
         measurement_result["task_id"] = task_id
-        # measurement_result["calibration_datetime"] = (
-        #     self.sensor.sensor_calibration_data["datetime"]
-        # )
         measurement_result["classification"] = self.classification
         sigan_settings = self.get_sigan_settings(measurement_result)
         logger.debug(f"sigan settings:{sigan_settings}")
diff --git a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
index 199473d3..6fd4c685 100644
--- a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
+++ b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
@@ -110,22 +110,9 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
             measurement_result["name"] = self.name
             measurement_result["classification"] = self.classification
             sigan_settings = self.get_sigan_settings(measurement_result)
-            capture_segment = CaptureSegment(
-                sample_start=0,
-                frequency=measurement_params[FREQUENCY],
-                datetime=measurement_result["capture_time"],
-                duration=duration_ms,
-                overload=measurement_result["overload"],
-                sigan_settings=sigan_settings,
+            capture_segment = self.create_capture_segment(
+                0, sigan_settings, measurement_result
             )
-            sensor_cal = self.sensor.sensor_calibration_data
-            if (
-                sensor_cal is not None
-                and measurement_result["applied_calibration"] is not None
-            ):
-                capture_segment.sensor_calibration = self.get_calibration(
-                    measurement_result
-                )
             measurement_result["capture_segment"] = capture_segment
 
             self.create_metadata(measurement_result, recording_id)
diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py
index c16d054f..69fc5d5b 100644
--- a/scos_actions/actions/calibrate_y_factor.py
+++ b/scos_actions/actions/calibrate_y_factor.py
@@ -261,8 +261,6 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
                 detail += self.calibrate(p)
             else:
                 detail += os.linesep + self.calibrate(p)
-        # Save results to onboard calibration file
-        # self.sensor.sensor_calibration.to_json()
         return detail
 
     def calibrate(self, params: dict):
diff --git a/scos_actions/actions/interfaces/measurement_action.py b/scos_actions/actions/interfaces/measurement_action.py
index e3081c7f..c5241f84 100644
--- a/scos_actions/actions/interfaces/measurement_action.py
+++ b/scos_actions/actions/interfaces/measurement_action.py
@@ -43,15 +43,14 @@ def create_capture_segment(
     ) -> CaptureSegment:
         capture_segment = CaptureSegment(
             sample_start=sample_start,
-            frequency=self.sensor.signal_analyzer.frequency,
+            frequency=measurement_result["frequency"],
             datetime=measurement_result["capture_time"],
             duration=measurement_result["duration_ms"],
             overload=measurement_result["overload"],
             sigan_settings=sigan_settings,
         )
         sensor_cal = self.sensor.sensor_calibration_data
-        # Rename compression point keys if they exist
-        # then set calibration metadata if it exists
+        # Set calibration metadata if it exists
         if (
             sensor_cal is not None
             and measurement_result["applied_calibration"] is not None
diff --git a/scos_actions/hardware/sensor.py b/scos_actions/hardware/sensor.py
index 8a72ba08..18adfeaa 100644
--- a/scos_actions/hardware/sensor.py
+++ b/scos_actions/hardware/sensor.py
@@ -262,9 +262,14 @@ def acquire_time_domain_samples(
 
         Gain adjustment can be applied to acquired samples using ``cal_adjust``.
         If ``True``, the samples acquired from the signal analyzer will be
-        scaled based on the calibrated ``gain`` value in the ``SensorCalibration``,
-        if one exists for this sensor, and "calibration terminal" will be the value
-        of the "reference" key in the returned dict.
+        scaled based on the calibrated ``gain`` and ``loss`` values in
+        the ``SensorCalibration`` and ``DifferentialCalibration.``
+        If no ``DifferentialCalibration`` exists, "calibration terminal"
+        will be the value of the "reference" key in the
+        returned dict. If a ``DifferentialCalibration`` exists, the gain and
+        noise figure will be adjusted with the loss specified in the
+        ``DifferentialCalibration`` and the "reference" will be set to the
+        calibration_reference of the ``DifferentialCalibration``.
 
         :param num_samples: Number of samples to acquire
         :param num_samples_skip: Number of samples to skip

From b0e4a043b4f47f4f5d9b189bce404b22743b0468 Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Fri, 22 Mar 2024 07:55:57 -0600
Subject: [PATCH 085/102] removed center_frequency_Hz from measurement_result
 in acquire_single_freq_fft.py.

---
 scos_actions/actions/acquire_single_freq_fft.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scos_actions/actions/acquire_single_freq_fft.py b/scos_actions/actions/acquire_single_freq_fft.py
index d99ba8dd..ac78d802 100644
--- a/scos_actions/actions/acquire_single_freq_fft.py
+++ b/scos_actions/actions/acquire_single_freq_fft.py
@@ -182,7 +182,6 @@ def execute(self, schedule_entry: dict, task_id: int) -> dict:
         measurement_result["duration_ms"] = round(
             (self.num_samples / sample_rate_Hz) * 1000
         )
-        measurement_result["center_frequency_Hz"] = self.frequency_Hz
         measurement_result["capture_segment"] = self.create_capture_segment(
             sample_start=0,
             sigan_settings=sigan_settings,

From b1764c7808c00f25f03018e75e4fbb47ddbf755d Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Fri, 22 Mar 2024 08:54:17 -0600
Subject: [PATCH 086/102] Fix get_calibration. Remove
 set_last_calibration_time.

---
 .../actions/interfaces/measurement_action.py  | 21 +++++--------------
 scos_actions/hardware/sensor.py               |  3 ++-
 scos_actions/metadata/sigmf_builder.py        |  5 -----
 3 files changed, 7 insertions(+), 22 deletions(-)

diff --git a/scos_actions/actions/interfaces/measurement_action.py b/scos_actions/actions/interfaces/measurement_action.py
index c5241f84..c52f1724 100644
--- a/scos_actions/actions/interfaces/measurement_action.py
+++ b/scos_actions/actions/interfaces/measurement_action.py
@@ -51,21 +51,16 @@ def create_capture_segment(
         )
         sensor_cal = self.sensor.sensor_calibration_data
         # Set calibration metadata if it exists
-        if (
-            sensor_cal is not None
-            and measurement_result["applied_calibration"] is not None
-        ):
-            capture_segment.sensor_calibration = self.get_calibration(
-                measurement_result
-            )
+        cal_meta = self.get_calibration(measurement_result)
+        if cal_meta is not None:
+            capture_segment.sensor_calibration = cal_meta
         return capture_segment
 
     def get_calibration(self, measurement_result: dict) -> ntia_sensor.Calibration:
         cal_meta = None
         if (
-            self.sensor.sensor_calibration_data
-            is measurement_result["applied_calibration"]
-            is not None
+            self.sensor.sensor_calibration_data is not None
+            and measurement_result["applied_calibration"] is not None
         ):
             cal_meta = ntia_sensor.Calibration(
                 datetime=self.sensor.sensor_calibration_data["datetime"],
@@ -120,12 +115,6 @@ def create_metadata(
             self.sigmf_builder.set_classification(measurement_result["classification"])
         except KeyError:
             logger.warning(warning_str.format("classification"))
-        try:
-            self.sigmf_builder.set_last_calibration_time(
-                measurement_result["calibration_datetime"]
-            )
-        except KeyError:
-            logger.warning(warning_str.format("calibration_datetime"))
         try:
             cap = measurement_result["capture_segment"]
             logger.debug(f"Adding capture:{cap}")
diff --git a/scos_actions/hardware/sensor.py b/scos_actions/hardware/sensor.py
index 18adfeaa..6179c421 100644
--- a/scos_actions/hardware/sensor.py
+++ b/scos_actions/hardware/sensor.py
@@ -363,7 +363,8 @@ def acquire_time_domain_samples(
                     measurement_result["applied_calibration"]["compression_point"] = (
                         self.sensor_calibration_data["compression_point"]
                     )
-
+                applied_cal = measurement_result["applied_calibration"]
+                logger.debug(f"Setting applied_calibration to: {applied_cal}")
             else:
                 # No sensor calibration exists
                 msg = "Unable to scale samples without sensor calibration data"
diff --git a/scos_actions/metadata/sigmf_builder.py b/scos_actions/metadata/sigmf_builder.py
index 2ed489e7..1e6106df 100644
--- a/scos_actions/metadata/sigmf_builder.py
+++ b/scos_actions/metadata/sigmf_builder.py
@@ -426,11 +426,6 @@ def add_annotation(self, start_index, length, annotation_md):
             start_index=start_index, length=length, metadata=annotation_md
         )
 
-    def set_last_calibration_time(self, last_cal_time):
-        self.sigmf_md.set_global_field(
-            "ntia-sensor:calibration_datetime", last_cal_time
-        )
-
     def add_to_global(self, key, value):
         self.sigmf_md.set_global_field(key, value)
 

From ed2458443c5f66bc0c1b88316e8eb2368bdd309f Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Fri, 22 Mar 2024 15:06:34 -0600
Subject: [PATCH 087/102] docstring updates.

---
 .../calibration/differential_calibration.py   | 32 +++++++++----------
 .../calibration/interfaces/calibration.py     | 17 +++++++---
 .../calibration/sensor_calibration.py         | 19 +++++++----
 3 files changed, 41 insertions(+), 27 deletions(-)

diff --git a/scos_actions/calibration/differential_calibration.py b/scos_actions/calibration/differential_calibration.py
index ac518737..de877f8a 100644
--- a/scos_actions/calibration/differential_calibration.py
+++ b/scos_actions/calibration/differential_calibration.py
@@ -1,19 +1,3 @@
-"""
-Dataclass implementation for "differential calibration" handling.
-
-A differential calibration provides loss values which represent excess loss
-between the ``SensorCalibration.calibration_reference`` reference point and
-another reference point. A typical usage would be for calibrating out measured
-cable losses which exist between the antenna and the Y-factor calibration terminal.
-At present, this is measured manually using a calibration probe consisting of a
-calibrated noise source and a programmable attenuator.
-
-The ``DifferentialCalibration.calibration_data`` entries should be dictionaries
-containing the key ``"loss"`` and a corresponding value in decibels (dB). A positive
-value of ``"loss"`` indicates a LOSS going FROM ``DifferentialCalibration.calibration_reference``
-TO ``SensorCalibration.calibration_reference``.
-"""
-
 from dataclasses import dataclass
 
 from scos_actions.calibration.interfaces.calibration import Calibration
@@ -21,6 +5,22 @@
 
 @dataclass
 class DifferentialCalibration(Calibration):
+    """
+    Dataclass implementation for "differential calibration" handling.
+
+    A differential calibration provides loss values which represent excess loss
+    between the ``SensorCalibration.calibration_reference`` reference point and
+    another reference point. A typical usage would be for calibrating out measured
+    cable losses which exist between the antenna and the Y-factor calibration terminal.
+    At present, this is measured manually using a calibration probe consisting of a
+    calibrated noise source and a programmable attenuator.
+
+    The ``DifferentialCalibration.calibration_data`` entries should be dictionaries
+    containing the key ``"loss"`` and a corresponding value in decibels (dB). A positive
+    value of ``"loss"`` indicates a LOSS going FROM ``DifferentialCalibration.calibration_reference``
+    TO ``SensorCalibration.calibration_reference``.
+    """
+
     def update(self):
         """
         SCOS Sensor should not update differential calibration files.
diff --git a/scos_actions/calibration/interfaces/calibration.py b/scos_actions/calibration/interfaces/calibration.py
index d89e1c50..70968ea6 100644
--- a/scos_actions/calibration/interfaces/calibration.py
+++ b/scos_actions/calibration/interfaces/calibration.py
@@ -1,7 +1,3 @@
-"""
-TODO
-"""
-
 import dataclasses
 import json
 import logging
@@ -19,6 +15,19 @@
 
 @dataclasses.dataclass
 class Calibration:
+    """
+    Base class to handle calibrated gains, noise figures, compression points, and losses.
+    The calibration_parameters defined the settings used to perform calibrations and the
+    order in which calibrations may be accessed in the calibration_data dictionary.
+    For example, if calibration_parameters where [frequency, sample_rate] then the
+    calibration for a particular frequency and sample rate would be accessed in
+    the calibration_data dictionary by the string value of the frequency and
+    sample rate, like calibration_data["3555000000.0"]["14000000.0"]. The
+    calibration_reference indicates the reference point for the calibration, e.d.,
+    antenna terminal, or noise source output. The file_path determines where
+    updates (if allowed) will be saved.
+    """
+
     calibration_parameters: List[str]
     calibration_data: dict
     calibration_reference: str
diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index 3224331e..90f32b7c 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -1,7 +1,3 @@
-"""
-TODO
-"""
-
 import logging
 from dataclasses import dataclass
 from datetime import datetime
@@ -18,6 +14,16 @@
 
 @dataclass
 class SensorCalibration(Calibration):
+    """
+    Extends the ``Calibration`` class to represent calibration
+    data that may be updated. Within scos-senso,``SensorCalibration``
+    instances are used to handle calibration files generated prior
+    to deployment through a lab-based calibration as well as the result
+    of calibrations that are performed by the sensor in the field. This
+    class provides an implementation fo the update method to allow calibration
+    data to be updated with new values.
+    """
+
     last_calibration_datetime: str
     clock_rate_lookup_by_sample_rate: List[Dict[str, float]]
     sensor_uid: str
@@ -41,8 +47,8 @@ def update(
         Update the calibration data by overwriting or adding an entry.
 
         This updates the instance variables of the ``SensorCalibration``
-        object and additionally writes these changes to the specified
-        output file.
+        object and additionally writes these changes to file specified
+        by the instance's file_path property.
 
         :param params: Parameters used for calibration. This must include
             entries for all of the ``Calibration.calibration_parameters``
@@ -52,7 +58,6 @@ def update(
         :param gain_dB: Gain value from calibration, in dB.
         :param noise_figure_dB: Noise figure value for calibration, in dB.
         :param temp_degC: Temperature at calibration time, in degrees Celsius.
-        :param file_path: File path for saving the updated calibration data.
         """
         logger.debug(f"Updating calibration file for params {params}")
         try:

From 924324306b1de1a7e24e06ef1dfaf8e8070be275 Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Fri, 22 Mar 2024 15:08:57 -0600
Subject: [PATCH 088/102] Use get_datetime_str_now()

---
 scos_actions/calibration/sensor_calibration.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index 90f32b7c..d8bcd16f 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -7,7 +7,7 @@
 
 from scos_actions.calibration.interfaces.calibration import Calibration
 from scos_actions.calibration.utils import CalibrationEntryMissingException
-from scos_actions.utils import parse_datetime_iso_format_str
+from scos_actions.utils import parse_datetime_iso_format_str, get_datetime_str_now
 
 logger = logging.getLogger(__name__)
 
@@ -100,7 +100,7 @@ def expired(self) -> bool:
         env = Env()
         time_limit = env.int("CALIBRATION_EXPIRATION_LIMIT", default=None)
         logger.debug("Checking if calibration has expired.")
-        now_string = datetime.utcnow().isoformat(timespec="milliseconds") + "Z"
+        now_string = get_datetime_str_now()
         now = parse_datetime_iso_format_str(now_string)
         if time_limit is None:
             return False

From ef0c7f7df236e79414aa57e33b05973c796faacc Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Fri, 22 Mar 2024 15:55:14 -0600
Subject: [PATCH 089/102] added assert in test.

---
 scos_actions/calibration/tests/test_sensor_calibration.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scos_actions/calibration/tests/test_sensor_calibration.py b/scos_actions/calibration/tests/test_sensor_calibration.py
index 3e3006f5..664c42c5 100644
--- a/scos_actions/calibration/tests/test_sensor_calibration.py
+++ b/scos_actions/calibration/tests/test_sensor_calibration.py
@@ -42,7 +42,7 @@ def check_duplicate(self, sr, f, g):
     def run_pytest_point(self, sr, f, g, reason, sr_m=False, f_m=False, g_m=False):
         """Test the calculated value against the algorithm
         Parameters:
-            sr, f, g -> Set values for the mock USRP
+            sr, f, g -> Set values for the mock signal analzyer
             reason: Test case string for failure reference
             sr_m, f_m, g_m -> Set values to use when calculating the expected value
                               May differ in from actual set points in edge cases
@@ -392,3 +392,4 @@ def test_has_expired_cal_data_multipledates_expired(self):
         expired = has_expired_cal_data(
             cal_data, parse_datetime_iso_format_str(now_date), 100
         )
+        assert expired == True

From 2004a0fb3030345ff6128ff80989e8e9387629f8 Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Fri, 22 Mar 2024 16:03:15 -0600
Subject: [PATCH 090/102] Update additional Sensor property getters with
 Optional returns.

---
 scos_actions/hardware/sensor.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scos_actions/hardware/sensor.py b/scos_actions/hardware/sensor.py
index 6179c421..47198b93 100644
--- a/scos_actions/hardware/sensor.py
+++ b/scos_actions/hardware/sensor.py
@@ -78,7 +78,7 @@ def gps(self, gps: GPSInterface):
         self._gps = gps
 
     @property
-    def preselector(self) -> Preselector:
+    def preselector(self) -> Optional[Preselector]:
         """
         RF front end that may include calibration sources, filters, and/or amplifiers.
         """
@@ -104,7 +104,7 @@ def switches(self, switches: Dict[str, WebRelay]):
         self._switches = switches
 
     @property
-    def location(self) -> dict:
+    def location(self) -> Optional[dict]:
         """
         The GeoJSON dictionary of the sensor's location.
         """
@@ -118,7 +118,7 @@ def location(self, loc: dict):
         self._location = loc
 
     @property
-    def capabilities(self) -> dict:
+    def capabilities(self) -> Optional[dict]:
         """
         A dictionary of the sensor's capabilities. The dictionary should
         include a 'sensor' key that maps to the ntia-sensor

From 8bba476790b77978de03cde6c3c4f05f56877f32 Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Fri, 22 Mar 2024 16:09:34 -0600
Subject: [PATCH 091/102] Update cal update test to use an original cal date in
 the past.

---
 scos_actions/calibration/tests/test_sensor_calibration.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scos_actions/calibration/tests/test_sensor_calibration.py b/scos_actions/calibration/tests/test_sensor_calibration.py
index 664c42c5..fc9ed010 100644
--- a/scos_actions/calibration/tests/test_sensor_calibration.py
+++ b/scos_actions/calibration/tests/test_sensor_calibration.py
@@ -305,7 +305,7 @@ def test_sf_no_interpolation_points(self):
                     break
 
     def test_update(self):
-        calibration_datetime = get_datetime_str_now()
+        calibration_datetime = "2024-03-17T19:16:55.172Z"
         calibration_params = ["sample_rate", "frequency"]
         calibration_data = {100.0: {200.0: {"noise_figure": 0, "gain": 0}}}
         test_cal_path = Path("test_calibration.json")

From 5c528ea28ec7c194c18af4584354b637cc13942e Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 25 Mar 2024 16:59:27 -0400
Subject: [PATCH 092/102] fix typo in variable name

---
 scos_actions/calibration/tests/test_sensor_calibration.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scos_actions/calibration/tests/test_sensor_calibration.py b/scos_actions/calibration/tests/test_sensor_calibration.py
index fc9ed010..88de63f6 100644
--- a/scos_actions/calibration/tests/test_sensor_calibration.py
+++ b/scos_actions/calibration/tests/test_sensor_calibration.py
@@ -69,7 +69,7 @@ def run_pytest_point(self, sr, f, g, reason, sr_m=False, f_m=False, g_m=False):
         interp_cal_data = self.sample_cal.get_calibration_dict(
             {"sample_rate": sr, "frequency": f, "gain": g}
         )
-        interp_gain_siggan = interp_cal_data["gain"]
+        interp_gain_sigan = interp_cal_data["gain"]
 
         # Save the point so we don't duplicate
         self.pytest_points.append(
@@ -86,7 +86,7 @@ def run_pytest_point(self, sr, f, g, reason, sr_m=False, f_m=False, g_m=False):
         tolerance = 1e-5
         msg = "Scale factor not correctly calculated!\r\n"
         msg = f"{msg}    Expected value:   {calc_gain_sigan}\r\n"
-        msg = f"{msg}    Calculated value: {interp_gain_siggan}\r\n"
+        msg = f"{msg}    Calculated value: {interp_gain_sigan}\r\n"
         msg = f"{msg}    Tolerance: {tolerance}\r\n"
         msg = f"{msg}    Test: {reason}\r\n"
         msg = f"{msg}    Sample Rate: {sr / 1e6}({sr_m / 1e6})\r\n"
@@ -97,12 +97,12 @@ def run_pytest_point(self, sr, f, g, reason, sr_m=False, f_m=False, g_m=False):
                 msg
             )
         )
-        if not isclose(calc_gain_sigan, interp_gain_siggan, abs_tol=tolerance):
+        if not isclose(calc_gain_sigan, interp_gain_sigan, abs_tol=tolerance):
             interp_cal_data = self.sample_cal.get_calibration_dict(
                 {"sample_rate": sr, "frequency": f, "gain": g}
             )
 
-        assert isclose(calc_gain_sigan, interp_gain_siggan, abs_tol=tolerance), msg
+        assert isclose(calc_gain_sigan, interp_gain_sigan, abs_tol=tolerance), msg
         return True
 
     @pytest.fixture(autouse=True)

From cf1175c08782f6e80336f700ac611b4e66f65280 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 25 Mar 2024 17:11:42 -0400
Subject: [PATCH 093/102] update docstring

---
 scos_actions/calibration/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scos_actions/calibration/utils.py b/scos_actions/calibration/utils.py
index fa977653..8aca940f 100644
--- a/scos_actions/calibration/utils.py
+++ b/scos_actions/calibration/utils.py
@@ -39,7 +39,8 @@ def filter_by_parameter(calibrations: dict, value: Union[float, int, bool]) -> d
     If ``value`` is a float or bool, ``str(value).lower()`` is used
     as the dictionary key. If ``value`` is an int, and the previous
     approach does not work, ``str(float(value))`` is attempted. This
-    allows for value ``1`` to match a key ``"1.0"``.
+    allows for value ``1`` to match a key ``"1.0"``, or a value of
+    ``1.0`` to match a key ``"1"``.
 
     :param calibrations: Calibration data dictionary.
     :param value: The parameter value for filtering. This value should

From ae416030e409b3946bed725ae07680e4ed428f6e Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 25 Mar 2024 17:15:10 -0400
Subject: [PATCH 094/102] Clarify filter_by_parameter debug message

---
 scos_actions/calibration/utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scos_actions/calibration/utils.py b/scos_actions/calibration/utils.py
index 8aca940f..e8275dcf 100644
--- a/scos_actions/calibration/utils.py
+++ b/scos_actions/calibration/utils.py
@@ -72,6 +72,7 @@ def filter_by_parameter(calibrations: dict, value: Union[float, int, bool]) -> d
             f"Could not locate calibration data at {value}"
             + f"\nAttempted lookup using key '{str(value).lower()}'"
             + f"{f' and {float(value)}' if isinstance(value, float) and value.is_integer() else ''}"
+            + f"{f' and {int(value)}' if isinstance(value, float) and value.is_integer() else ''}"
             + f"\nUsing calibration data: {calibrations}"
         )
         raise CalibrationEntryMissingException(msg)

From 59322059b106c70357b7938d5f09e41395775601 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 25 Mar 2024 17:17:13 -0400
Subject: [PATCH 095/102] Fix debug statement condition in filter_by_parameter

---
 scos_actions/calibration/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scos_actions/calibration/utils.py b/scos_actions/calibration/utils.py
index e8275dcf..a526e7f6 100644
--- a/scos_actions/calibration/utils.py
+++ b/scos_actions/calibration/utils.py
@@ -71,7 +71,7 @@ def filter_by_parameter(calibrations: dict, value: Union[float, int, bool]) -> d
         msg = (
             f"Could not locate calibration data at {value}"
             + f"\nAttempted lookup using key '{str(value).lower()}'"
-            + f"{f' and {float(value)}' if isinstance(value, float) and value.is_integer() else ''}"
+            + f"{f' and {float(value)}' if isinstance(value, int) else ''}"
             + f"{f' and {int(value)}' if isinstance(value, float) and value.is_integer() else ''}"
             + f"\nUsing calibration data: {calibrations}"
         )

From fc2d32ad44ebf1f33e60e04ff1a7edea8f531cab Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 25 Mar 2024 17:19:13 -0400
Subject: [PATCH 096/102] fix typo in class docstring

---
 scos_actions/calibration/sensor_calibration.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index d8bcd16f..6f1386bb 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -7,7 +7,7 @@
 
 from scos_actions.calibration.interfaces.calibration import Calibration
 from scos_actions.calibration.utils import CalibrationEntryMissingException
-from scos_actions.utils import parse_datetime_iso_format_str, get_datetime_str_now
+from scos_actions.utils import get_datetime_str_now, parse_datetime_iso_format_str
 
 logger = logging.getLogger(__name__)
 
@@ -16,7 +16,7 @@
 class SensorCalibration(Calibration):
     """
     Extends the ``Calibration`` class to represent calibration
-    data that may be updated. Within scos-senso,``SensorCalibration``
+    data that may be updated. Within SCOS Sensor,``SensorCalibration``
     instances are used to handle calibration files generated prior
     to deployment through a lab-based calibration as well as the result
     of calibrations that are performed by the sensor in the field. This

From bcea100f3287d0d6362350b4e0af36da9e0bf7c8 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 25 Mar 2024 17:20:12 -0400
Subject: [PATCH 097/102] fix typo in class docstring

---
 scos_actions/calibration/sensor_calibration.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index 6f1386bb..1ce985ee 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -20,7 +20,7 @@ class SensorCalibration(Calibration):
     instances are used to handle calibration files generated prior
     to deployment through a lab-based calibration as well as the result
     of calibrations that are performed by the sensor in the field. This
-    class provides an implementation fo the update method to allow calibration
+    class provides an implementation for the update method to allow calibration
     data to be updated with new values.
     """
 

From 5f9643d7bf755b401a76b940c907b3f29d6ee93b Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 25 Mar 2024 17:21:22 -0400
Subject: [PATCH 098/102] remove unused variable "sensor_cal"

---
 scos_actions/actions/interfaces/measurement_action.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scos_actions/actions/interfaces/measurement_action.py b/scos_actions/actions/interfaces/measurement_action.py
index c52f1724..e7ad088a 100644
--- a/scos_actions/actions/interfaces/measurement_action.py
+++ b/scos_actions/actions/interfaces/measurement_action.py
@@ -49,7 +49,6 @@ def create_capture_segment(
             overload=measurement_result["overload"],
             sigan_settings=sigan_settings,
         )
-        sensor_cal = self.sensor.sensor_calibration_data
         # Set calibration metadata if it exists
         cal_meta = self.get_calibration(measurement_result)
         if cal_meta is not None:

From 6087a8b4849b5b595aee4a012ffe36d4ea1224c3 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 25 Mar 2024 17:41:12 -0400
Subject: [PATCH 099/102] remove redundant type cast and formatting

---
 scos_actions/calibration/interfaces/calibration.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scos_actions/calibration/interfaces/calibration.py b/scos_actions/calibration/interfaces/calibration.py
index 70968ea6..f3ea522c 100644
--- a/scos_actions/calibration/interfaces/calibration.py
+++ b/scos_actions/calibration/interfaces/calibration.py
@@ -70,7 +70,7 @@ def get_calibration_dict(self, params: dict) -> dict:
             )
         cal_data = self.calibration_data
         for p_name in self.calibration_parameters:
-            p_value = str(params[p_name]).lower()
+            p_value = params[p_name]
             logger.debug(f"Looking up calibration data at {p_name}={p_value}")
             cal_data = filter_by_parameter(cal_data, p_value)
 

From 73fab880d4e065e7fecfcf2eb44f852071ed3d1c Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 25 Mar 2024 17:41:49 -0400
Subject: [PATCH 100/102] improve filter_by_parameter logging and tests

---
 .../tests/test_sensor_calibration.py          | 12 +++++++---
 scos_actions/calibration/tests/test_utils.py  | 22 ++++++++++++++-----
 scos_actions/calibration/utils.py             |  7 +++---
 3 files changed, 29 insertions(+), 12 deletions(-)

diff --git a/scos_actions/calibration/tests/test_sensor_calibration.py b/scos_actions/calibration/tests/test_sensor_calibration.py
index 88de63f6..aa29ba81 100644
--- a/scos_actions/calibration/tests/test_sensor_calibration.py
+++ b/scos_actions/calibration/tests/test_sensor_calibration.py
@@ -276,11 +276,17 @@ def test_get_calibration_dict_within_range(self):
             clock_rate_lookup_by_sample_rate=[],
             sensor_uid="TESTING",
         )
+
+        lookup_fail_value = 250.0
         with pytest.raises(CalibrationException) as e_info:
-            _ = cal.get_calibration_dict({"sample_rate": 100.0, "frequency": 250.0})
+            _ = cal.get_calibration_dict(
+                {"sample_rate": 100.0, "frequency": lookup_fail_value}
+            )
         assert e_info.value.args[0] == (
-            f"Could not locate calibration data at 250.0"
-            + f"\nAttempted lookup using key '250.0'"
+            f"Could not locate calibration data at {lookup_fail_value}"
+            + "\nAttempted lookup using keys: "
+            + f"\n\tstr({lookup_fail_value}).lower() = {str(lookup_fail_value).lower()}"
+            + f"\n\tstr(int({lookup_fail_value})) = {int(lookup_fail_value)}"
             + f"\nUsing calibration data: {cal.calibration_data['100.0']}"
         )
 
diff --git a/scos_actions/calibration/tests/test_utils.py b/scos_actions/calibration/tests/test_utils.py
index 15ab29cf..8b6eefcf 100644
--- a/scos_actions/calibration/tests/test_utils.py
+++ b/scos_actions/calibration/tests/test_utils.py
@@ -6,12 +6,17 @@
 class TestCalibrationUtils:
     def test_filter_by_parameter_out_of_range(self):
         calibrations = {200.0: {"some_cal_data"}, 300.0: {"more cal data"}}
+
+        # Also checks error output when missing value is an integer
+        test_value = 400
         with pytest.raises(CalibrationException) as e_info:
-            _ = filter_by_parameter(calibrations, 400.0)
+            _ = filter_by_parameter(calibrations, test_value)
         assert (
             e_info.value.args[0]
-            == f"Could not locate calibration data at 400.0"
-            + f"\nAttempted lookup using key '400.0' and 400.0"
+            == f"Could not locate calibration data at {test_value}"
+            + "\nAttempted lookup using keys: "
+            + f"\n\tstr({test_value}).lower() = {str(test_value).lower()}"
+            + f"\n\tstr(float({test_value})) = {float(test_value)}"
             + f"\nUsing calibration data: {calibrations}"
         )
 
@@ -20,11 +25,16 @@ def test_filter_by_parameter_in_range_requires_match(self):
             200.0: {"Gain": "Gain at 200.0"},
             300.0: {"Gain": "Gain at 300.0"},
         }
+
+        # Check looking up a missing value with a float
+        test_value = 150.0
         with pytest.raises(CalibrationException) as e_info:
-            _ = filter_by_parameter(calibrations, 150.0)
+            _ = filter_by_parameter(calibrations, test_value)
         assert e_info.value.args[0] == (
-            f"Could not locate calibration data at 150.0"
-            + f"\nAttempted lookup using key '150.0' and 150.0"
+            f"Could not locate calibration data at {test_value}"
+            + "\nAttempted lookup using keys: "
+            + f"\n\tstr({test_value}).lower() = {str(test_value).lower()}"
+            + f"\n\tstr(int({test_value})) = {int(test_value)}"
             + f"\nUsing calibration data: {calibrations}"
         )
 
diff --git a/scos_actions/calibration/utils.py b/scos_actions/calibration/utils.py
index a526e7f6..8e6a21dd 100644
--- a/scos_actions/calibration/utils.py
+++ b/scos_actions/calibration/utils.py
@@ -52,12 +52,15 @@ def filter_by_parameter(calibrations: dict, value: Union[float, int, bool]) -> d
     """
     try:
         filtered_data = calibrations.get(str(value).lower(), None)
+        attempts = f"\n\tstr({value}).lower() = {str(value).lower()}"
         if filtered_data is None and isinstance(value, int):
             # Try equivalent float for ints, i.e., match "1.0" to 1
             filtered_data = calibrations.get(str(float(value)), None)
+            attempts += f"\n\tstr(float({value})) = {str(float(value))}"
         if filtered_data is None and isinstance(value, float) and value.is_integer():
             # Check for, e.g., key '25' if value is '25.0'
             filtered_data = calibrations.get(str(int(value)), None)
+            attempts += f"\n\tstr(int({value})) = {str(int(value))}"
         if filtered_data is None:
             raise KeyError
         else:
@@ -70,9 +73,7 @@ def filter_by_parameter(calibrations: dict, value: Union[float, int, bool]) -> d
     except KeyError:
         msg = (
             f"Could not locate calibration data at {value}"
-            + f"\nAttempted lookup using key '{str(value).lower()}'"
-            + f"{f' and {float(value)}' if isinstance(value, int) else ''}"
-            + f"{f' and {int(value)}' if isinstance(value, float) and value.is_integer() else ''}"
+            + f"\nAttempted lookup using keys: {attempts}"
             + f"\nUsing calibration data: {calibrations}"
         )
         raise CalibrationEntryMissingException(msg)

From 15348fd4dbd2cc51be08d0a64a9b9c9c101ea394 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 25 Mar 2024 17:56:50 -0400
Subject: [PATCH 101/102] Remove outdated calibration unit testing code

---
 .../tests/test_sensor_calibration.py          | 103 ++----------------
 scos_actions/tests/resources/__init__.py      |   0
 scos_actions/tests/resources/utils.py         |  15 ---
 3 files changed, 8 insertions(+), 110 deletions(-)
 delete mode 100644 scos_actions/tests/resources/__init__.py
 delete mode 100644 scos_actions/tests/resources/utils.py

diff --git a/scos_actions/calibration/tests/test_sensor_calibration.py b/scos_actions/calibration/tests/test_sensor_calibration.py
index aa29ba81..bd4c610c 100644
--- a/scos_actions/calibration/tests/test_sensor_calibration.py
+++ b/scos_actions/calibration/tests/test_sensor_calibration.py
@@ -5,7 +5,6 @@
 import json
 import random
 from copy import deepcopy
-from math import isclose
 from pathlib import Path
 from typing import Dict, List
 
@@ -18,7 +17,6 @@
 )
 from scos_actions.calibration.tests.utils import recursive_check_keys
 from scos_actions.calibration.utils import CalibrationException
-from scos_actions.tests.resources.utils import easy_gain
 from scos_actions.utils import get_datetime_str_now, parse_datetime_iso_format_str
 
 
@@ -39,75 +37,14 @@ def check_duplicate(self, sr, f, g):
             if duplicate_f and duplicate_g and duplicate_sr:
                 return True
 
-    def run_pytest_point(self, sr, f, g, reason, sr_m=False, f_m=False, g_m=False):
-        """Test the calculated value against the algorithm
-        Parameters:
-            sr, f, g -> Set values for the mock signal analzyer
-            reason: Test case string for failure reference
-            sr_m, f_m, g_m -> Set values to use when calculating the expected value
-                              May differ in from actual set points in edge cases
-                              such as tuning in divisions or uncalibrated sample rate"""
-        # Check that the setup was completed
-        assert self.setup_complete, "Setup was not completed"
-
-        # If this point was tested before, skip it (triggering a new one)
-        if self.check_duplicate(sr, f, g):
-            return False
-
-        # If the point doesn't have modified inputs, use the algorithm ones
-        if not f_m:
-            f_m = f
-        if not g_m:
-            g_m = g
-        if not sr_m:
-            sr_m = sr
-
-        # Calculate what the scale factor should be
-        calc_gain_sigan = easy_gain(sr_m, f_m, g_m)
-
-        # Get the scale factor from the algorithm
-        interp_cal_data = self.sample_cal.get_calibration_dict(
-            {"sample_rate": sr, "frequency": f, "gain": g}
-        )
-        interp_gain_sigan = interp_cal_data["gain"]
-
-        # Save the point so we don't duplicate
-        self.pytest_points.append(
-            {
-                "sample_rate": int(sr),
-                "frequency": f,
-                "setting_value": g,
-                "gain": calc_gain_sigan,
-                "test": reason,
-            }
-        )
-
-        # Check if the point was calculated correctly
-        tolerance = 1e-5
-        msg = "Scale factor not correctly calculated!\r\n"
-        msg = f"{msg}    Expected value:   {calc_gain_sigan}\r\n"
-        msg = f"{msg}    Calculated value: {interp_gain_sigan}\r\n"
-        msg = f"{msg}    Tolerance: {tolerance}\r\n"
-        msg = f"{msg}    Test: {reason}\r\n"
-        msg = f"{msg}    Sample Rate: {sr / 1e6}({sr_m / 1e6})\r\n"
-        msg = f"{msg}    Frequency: {f / 1e6}({f_m / 1e6})\r\n"
-        msg = f"{msg}    Gain: {g}({g_m})\r\n"
-        msg = (
-            "{}    Formula: -1 * (Gain - Frequency[GHz] - Sample Rate[MHz])\r\n".format(
-                msg
-            )
-        )
-        if not isclose(calc_gain_sigan, interp_gain_sigan, abs_tol=tolerance):
-            interp_cal_data = self.sample_cal.get_calibration_dict(
-                {"sample_rate": sr, "frequency": f, "gain": g}
-            )
-
-        assert isclose(calc_gain_sigan, interp_gain_sigan, abs_tol=tolerance), msg
-        return True
-
     @pytest.fixture(autouse=True)
     def setup_calibration_file(self, tmp_path: Path):
-        """Create the dummy calibration file in the pytest temp directory"""
+        """
+        Create the dummy calibration file in the pytest temp directory
+
+        The gain values in each calibration data entry are set up as being
+        equal to the gain setting minus ``self.dummy_gain_scale_factor``
+        """
 
         # Only setup once
         if self.setup_complete:
@@ -117,6 +54,7 @@ def setup_calibration_file(self, tmp_path: Path):
         self.calibration_file = tmp_path / "dummy_cal_file.json"
 
         # Setup variables
+        self.dummy_gain_scale_factor = 5  # test data gain values are (gain setting - 5)
         self.dummy_noise_figure = 10
         self.dummy_compression = -20
         self.test_repeat_times = 3
@@ -164,14 +102,9 @@ def setup_calibration_file(self, tmp_path: Path):
             for i in range(len(frequencies)):
                 cal_data_g = {}
                 for j in range(len(gains)):
-                    # Create the scale factor that ensures easy interpolation
-                    gain_sigan = easy_gain(
-                        self.sample_rates[k], frequencies[i], gains[j]
-                    )
-
                     # Create the data point
                     cal_data_point = {
-                        "gain": gain_sigan,
+                        "gain": gains[j] - self.dummy_gain_scale_factor,
                         "noise_figure": self.dummy_noise_figure,
                         "1dB_compression_point": self.dummy_compression,
                     }
@@ -290,26 +223,6 @@ def test_get_calibration_dict_within_range(self):
             + f"\nUsing calibration data: {cal.calibration_data['100.0']}"
         )
 
-    def test_sf_bound_points(self):
-        """Test SF determination at boundary points"""
-        self.run_pytest_point(
-            self.srs[0], self.frequency_min, self.gain_min, "Testing boundary points"
-        )
-        self.run_pytest_point(
-            self.srs[0], self.frequency_max, self.gain_max, "Testing boundary points"
-        )
-
-    def test_sf_no_interpolation_points(self):
-        """Test points without interpolation"""
-        for i in range(4 * self.test_repeat_times):
-            while True:
-                g = self.g_s[self.rand_index(self.g_s)]
-                f = self.f_s[self.rand_index(self.f_s)]
-                if self.run_pytest_point(
-                    self.srs[0], f, g, "Testing no interpolation points"
-                ):
-                    break
-
     def test_update(self):
         calibration_datetime = "2024-03-17T19:16:55.172Z"
         calibration_params = ["sample_rate", "frequency"]
diff --git a/scos_actions/tests/resources/__init__.py b/scos_actions/tests/resources/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/scos_actions/tests/resources/utils.py b/scos_actions/tests/resources/utils.py
deleted file mode 100644
index ff26be2c..00000000
--- a/scos_actions/tests/resources/utils.py
+++ /dev/null
@@ -1,15 +0,0 @@
-def easy_gain(sample_rate, frequency, gain):
-    """Create an easily interpolated calibration gain value for testing.
-
-    :type sample_rate: float
-    :param sample_rate: Sample rate in samples per second
-
-    :type frequency: float
-    :param frequency: Frequency in hertz
-
-    :type gain: int
-    :param gain: Signal analyzer gain setting in dB
-
-    :rtype: float
-    """
-    return gain + (sample_rate / 1e6) + (frequency / 1e9)

From f8c9316ac2f671e8a85196faa004e7b0aafb20b6 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 25 Mar 2024 17:59:55 -0400
Subject: [PATCH 102/102] update pyupgrade and black pre-commit hooks

---
 .pre-commit-config.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 12ef9bfc..f6d8b0db 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -18,7 +18,7 @@ repos:
       - id: end-of-file-fixer
       - id: trailing-whitespace
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.15.1
+    rev: v3.15.2
     hooks:
       - id: pyupgrade
         args: ["--py38-plus"]
@@ -30,7 +30,7 @@ repos:
         types: [file, python]
         args: ["--profile", "black", "--filter-files", "--gitignore"]
   - repo: https://github.com/psf/black
-    rev: 24.2.0
+    rev: 24.3.0
     hooks:
       - id: black
         types: [file, python]