diff --git a/.github/workflows/github-actions-test.yml b/.github/workflows/github-actions-test.yml index ec289c2d..6aaa7561 100644 --- a/.github/workflows/github-actions-test.yml +++ b/.github/workflows/github-actions-test.yml @@ -75,8 +75,8 @@ jobs: source ./env.template export MOCK_SIGAN=1 export MOCK_SIGAN_RANDOM=1 - docker-compose build --no-cache - docker-compose up -d + docker compose build --no-cache + docker compose up -d - name: Wait for containers # wait for containers to finish starting run: sleep 45 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ab6aed0a..09e02539 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,24 +19,24 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/asottile/pyupgrade - rev: v3.15.0 + rev: v3.15.2 hooks: - id: pyupgrade args: ["--py38-plus"] - repo: https://github.com/pycqa/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort name: isort (python) types: [file, python] args: ["--profile", "black", "--filter-files", "--gitignore"] - repo: https://github.com/psf/black - rev: 23.10.1 + rev: 24.3.0 hooks: - id: black types: [file, python] - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.37.0 + rev: v0.39.0 hooks: - id: markdownlint types: [file, markdown] diff --git a/README.md b/README.md index 4bd31ab4..484caf8a 100644 --- a/README.md +++ b/README.md @@ -292,7 +292,9 @@ scos-sensor website from the same computer as where it is hosted. When running in a production environment or on a remote system, various settings will need to be configured. -## docker-compose.yml +### Compose File + +This section details configuration which takes place in `compose.yaml`: - shm_size: This setting is overriding the default setting of 64 mb. If using scos-sensor on a computer with lower memory, this may need to be decreased. This is @@ -321,6 +323,9 @@ settings in the environment file: By default, this is configured to use a version of `ghcr.io/ntia/scos-tekrsa/tekrsa_usb` to use a Tektronix signal analyzer. +- CALIBRATION_EXPIRATION_LIMIT: Number of seconds elapsed for a calibration result to + become expired. On startup, if existing calibration is expired, the action defined by + STARTUP_CALIBRATION_ACTION will be run to generate new calibration data. - CALLBACK_AUTHENTICATION: Sets how to authenticate to the callback URL. Supports `TOKEN` or `CERT`. - CALLBACK_SSL_VERIFICATION: Set to “true” in production environment. If false, the SSL @@ -353,6 +358,7 @@ settings in the environment file: certificate. - POSTGRES_PASSWORD: Sets password for the Postgres database for the “postgres” user. Change in production. The env.template file sets to a randomly generated value. +- RAY_INIT: If set to true, SCOS Sensor will ensure initializaiton of the Ray library. - REPO_ROOT: Root folder of the repository. Should be correctly set by default. - SCOS_SENSOR_GIT_TAG: The scos-sensor branch name. This value may be used in action metadata to capture the version of the software that produced the sigmf archive. @@ -379,6 +385,8 @@ settings in the environment file: scos-sensor repository with a valid certificate in production. - SSL_KEY_PATH: Path to server SSL private key. Use the private key for your valid certificate in production. +- STARTUP_CALIBRATION_ACTION: The name of an available action which will run on startup + if no unexpired calibration data is already present. - USB_DEVICE: Optional string used to search for available USB devices. By default, this is set to Tektronix to see if the Tektronix signal analyzer is available. If the specified value is not found in the output of lsusb, scos-sensor will attempt @@ -420,47 +428,46 @@ specific to the sensor you are using. } ``` -### Sensor Calibration File +### Calibration Files -By default, scos-sensor will use `configs/default_calibration.json` as the sensor -calibration file. However, if`configs/sensor_calibration.json` or -`configs/sigan_calibration.json` exist they will be used instead of the default -calibration file. Sensor calibration files allow scos-sensor to apply a gain based -on a laboratory calibration of the sensor and may also contain other useful -metadata that characterizes the sensor performance. For additional -information on the calibration data, see the -[NTIA-Sensor SigMF Calibration Object](https://github.com/NTIA/sigmf-ns-ntia/blob/master/ntia-sensor.sigmf-ext.md#08-the-calibration-object). -The default calibration file is shown below: +Calibration files allow SCOS Sensor to scale data based on a laboratory and/or +in-field calibration of the sensor, and may also contain other useful metadata that +characterizes the sensor performance. Two primary types of calibration files are used: +sensor calibration files and differential calibration files. -```json -{ - "calibration_data":{ - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 0, - "noise_figure": null, - "1db_compression_point": null, - "enbw": null, - "temperature": 26.85 - }, - "last_calibration_datetime": "1970-01-01T00:00:00.000000Z", - "calibration_parameters": [], - "clock_rate_lookup_by_sample_rate": [ - ], - "sensor_uid": "DEFAULT CALIBRATION", - "calibration_reference": "noise source output" -} -``` +Sensor calibration files may be provided upon sensor deployment or generated by onboard +calibration actions. If both exist, onboard calibration data takes priority. The +sensor will first attempt to load `configs/onboard_sensor_calibration.json`, and fall +back to `configs/sensor_calibration.json` if the first option fails. Next, SCOS determines +whether the loaded calibration data is expired, based on the threshold set by the +CALIBRATION_EXPIRATION_LIMIT threshold. If calibration data is expired, the sensor will +attempt to run the calibration action defined by STARTUP_CALIBRATION_ACTION. + +SCOS Sensor also supports an additional calibration file, called a differential +calibration. The differential calibration is used to provide additional scaling factors +to shift the reference point of data from the onboard calibration terminal to elsewhere +in the signal path. For instance, a differential calibration file can be provided with +scaling factors to shift the data reference point from the onboard calibration terminal +to the antenna port. The differential calibration is loaded separately, and used in +addition to, either the onboard or lab-provided sensor calibration file. + +#### Calibration File Contents + +In sensor calibration files, the unit of calibration data is defined by the +[NTIA-Sensor SigMF Calibration Object](https://github.com/NTIA/sigmf-ns-ntia/blob/master/ntia-sensor.sigmf-ext.md#08-the-calibration-object). +In differential calibration files, the only key in the calibration data is `loss`. The `calibration_parameters` key lists the parameters that will be used to obtain -the calibration data. In the case of the default calibration, there are no -`calibration_parameters` so the calibration data is found directly within the -`calibration_data` element and by default scos-sensor will not apply any additional -gain. Typically, a sensor would be calibrated at particular -sensing parameters. The calibration data for specific parameters should be listed -within the calibration_data object and accessed by the values of the settings -listed in the calibration_parameters element. For example, the calibration below -provides an example of a sensor calibrated at a sample rate of 14000000.0 samples -per second at several frequencies with a signal analyzer reference level setting of -25. +the calibration data. In the case of onboard calibration being generated from scratch, +the startup calibration action's signal analyzer settings are used as the `calibration_parameters`. +The calibration data is found directly within the `calibration_data` element and by +default SCOS Sensor will not apply any additional gain. Typically, a sensor would be +calibrated at particular sensing parameters. The calibration data for specific parameters +should be listed within the `calibration_data` object and accessed by the values of the +settings listed in the calibration_parameters element. For example, the sensor +calibration below provides an example of a sensor calibrated at a sample rate of +140000000 samples per second at several frequencies with a signal analyzer reference +level setting of -25. ```json { @@ -468,47 +475,33 @@ per second at several frequencies with a signal analyzer reference level setting "calibration_parameters": [ "sample_rate", "frequency", - "reference_level", - "preamp_enable", - "attenuation" + "reference_level" ], "clock_rate_lookup_by_sample_rate": [], "calibration_data": { "14000000.0": { "3545000000.0": { "-25": { - "true": { - "0": { - "datetime": "2023-10-23T14:38:02.882Z", - "gain": 30.09194805857024, - "noise_figure": 4.741521295220736, - "temperature": 15.6 - } - } + "datetime": "2023-10-23T14:38:02.882Z", + "gain": 30.09194805857024, + "noise_figure": 4.741521295220736, + "temperature": 15.6 } }, "3555000000.0": { "-25": { - "true": { - "0": { - "datetime": "2023-10-23T14:38:08.022Z", - "gain": 30.401008416406599, - "noise_figure": 4.394893979804061, - "temperature": 15.6 - } - } + "datetime": "2023-10-23T14:38:08.022Z", + "gain": 30.401008416406599, + "noise_figure": 4.394893979804061, + "temperature": 15.6 } }, "3565000000.0": { "-25": { - "true": { - "0": { - "datetime": "2023-10-23T14:38:11.922Z", - "gain": 30.848049817892105, - "noise_figure": 4.0751785215495819, - "temperature": 15.6 - } - } + "datetime": "2023-10-23T14:38:11.922Z", + "gain": 30.848049817892105, + "noise_figure": 4.0751785215495819, + "temperature": 15.6 } } } @@ -518,7 +511,7 @@ per second at several frequencies with a signal analyzer reference level setting When an action is run with the above calibration, SCOS will expect the action to have a sample_rate, frequency, and reference_level specified in the action config. The values -specified for these parameters will then be used to retrieve the calibration data. +specified for these parameters will then be used to retrieve the calibration data entry. ## Security @@ -844,7 +837,7 @@ for additional information. Be sure to re-source the environment file, update th requirements files, and prune any existing containers before rebuilding scos-sensor. -## Preselector Support +### Preselector Support Scos-sensor can be configured to support [preselectors](http://www.github.com/ntia/Preselector). @@ -858,7 +851,7 @@ in docker-compose.yaml to the python module that contains the preselector implementation you specify in PRESELECTOR_CLASS in docker-compose.yaml. -## Relay Support +### Relay Support Scos-sensor can be configured with zero or more [network controlled relays](https://www.controlbyweb.com/webrelay/). The default relay configuration directory is configs/switches. diff --git a/docker-compose.yml b/compose.yaml similarity index 97% rename from docker-compose.yml rename to compose.yaml index 4987bce3..8bce4c9b 100644 --- a/docker-compose.yml +++ b/compose.yaml @@ -1,5 +1,3 @@ -version: '3' - services: db: image: postgres:15-alpine @@ -45,6 +43,7 @@ services: - ADDITIONAL_USER_NAMES - ADDITIONAL_USER_PASSWORD - AUTHENTICATION + - CALIBRATION_EXPIRATION_LIMIT - CALLBACK_AUTHENTICATION - CALLBACK_SSL_VERIFICATION - CALLBACK_TIMEOUT @@ -70,6 +69,8 @@ services: - SIGAN_CLASS - SIGAN_POWER_SWITCH - SIGAN_POWER_CYCLE_STATES + - STARTUP_CALIBRATION_ACTION + - RAY_INIT - RUNNING_MIGRATIONS - USB_DEVICE expose: diff --git a/configs/default_calibration.json b/configs/default_calibration.json deleted file mode 100644 index 5b951871..00000000 --- a/configs/default_calibration.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "calibration_data":{ - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 0, - "noise_figure": 0.0, - "1db_compression_point": null, - "enbw": null, - "temperature": 26.85 - }, - "last_calibration_datetime": "1970-01-01T00:00:00.000000Z", - "calibration_parameters": [], - "clock_rate_lookup_by_sample_rate": [ - ], - "sensor_uid": "DEFAULT CALIBRATION", - "calibration_reference": "noise source output" -} diff --git a/env.template b/env.template index f78e123e..35dacf17 100644 --- a/env.template +++ b/env.template @@ -18,8 +18,7 @@ ADMIN_PASSWORD=password # set to CERT to enable scos-sensor certificate authentication AUTHENTICATION=TOKEN - -BASE_IMAGE=ghcr.io/ntia/scos-tekrsa/tekrsa_usb:0.2.3 +CALIBRATION_EXPIRATION_LIMIT=360 # Default callback api/results # Set to CERT for certificate authentication CALLBACK_AUTHENTICATION=TOKEN @@ -34,8 +33,6 @@ CALLBACK_TIMEOUT=2 # Use either true or false DEBUG=true -DEVICE_MODEL=RSA507A - # Use latest as default for local development DOCKER_TAG=latest @@ -75,8 +72,12 @@ SCOS_SENSOR_GIT_TAG="$(git describe --tags)" # SECURITY WARNING: generate unique key with `manage.py generate_secret_key` SECRET_KEY="$(python3 -c 'import secrets; print(secrets.token_urlsafe(64))')" +# Signal analyzer selection/setup SIGAN_CLASS=TekRSASigan SIGAN_MODULE=scos_tekrsa.hardware.tekrsa_sigan +USB_DEVICE=Tektronix +DEVICE_MODEL=RSA507A +BASE_IMAGE=ghcr.io/ntia/scos-tekrsa/tekrsa_usb:latest # SECURITY WARNING: You should be using certs from a trusted authority. # If you don't have any, try letsencrypt or a similar service. @@ -86,8 +87,11 @@ SSL_CA_PATH=scos_test_ca.crt SSL_CERT_PATH=sensor01.pem SSL_KEY_PATH=sensor01.pem -USB_DEVICE=Tektronix - +# Calibration action selection +# The action specified here will be used to attempt an onboard +# sensor calibration on startup, if no onboard calibration data +# is available on startup. The specified action must be available. +STARTUP_CALIBRATION_ACTION=SEA_CBRS_Calibrate_Baseline # Debug dependant settings if $DEBUG; then diff --git a/gunicorn/config.py b/gunicorn/config.py index 5efc64fb..0b68ad33 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -2,7 +2,6 @@ import sys from multiprocessing import cpu_count - bind = ":8000" workers = 1 worker_class = "gthread" diff --git a/src/handlers/apps.py b/src/handlers/apps.py index 8afad0e2..3dffb135 100644 --- a/src/handlers/apps.py +++ b/src/handlers/apps.py @@ -5,7 +5,7 @@ from scos_actions.signals import ( location_action_completed, measurement_action_completed, - trigger_api_restart + trigger_api_restart, ) logger = logging.getLogger(__name__) @@ -20,7 +20,6 @@ def ready(self): db_location_deleted, db_location_updated, location_action_completed_callback, - ) from handlers.measurement_handler import measurement_action_completed_callback @@ -40,5 +39,3 @@ def ready(self): trigger_api_restart.connect(trigger_api_restart_callback) logger.debug("trigger_api_restart_callback registered to trigger_api_restart") - - diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index e97def1e..82822e38 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -1,10 +1,18 @@ +import importlib import logging -import sys -import types +import time +from os import path from pathlib import Path -from subprocess import check_output +from typing import Optional, Union from django.conf import settings +from its_preselector.configuration_exception import ConfigurationException +from its_preselector.controlbyweb_web_relay import ControlByWebWebRelay +from its_preselector.preselector import Preselector +from scos_actions.calibration.differential_calibration import DifferentialCalibration +from scos_actions.calibration.sensor_calibration import SensorCalibration +from scos_actions.hardware.utils import power_cycle_sigan +from scos_actions.utils import load_from_json from utils.signals import register_component_with_status @@ -12,20 +20,71 @@ from .capabilities_loader import CapabilitiesLoader from .sensor_loader import SensorLoader from .status_monitor import StatusMonitor +from .utils import get_usb_device_exists, set_container_unhealthy logger = logging.getLogger(__name__) status_monitor = StatusMonitor() -def get_usb_device_exists() -> bool: - logger.debug("Checking for USB...") - if not settings.RUNNING_TESTS and settings.USB_DEVICE is not None: - usb_devices = check_output("lsusb").decode(sys.stdout.encoding) - logger.debug("Checking for " + settings.USB_DEVICE) - logger.debug("Found " + usb_devices) - return settings.USB_DEVICE in usb_devices - return True +def load_preselector_from_file( + preselector_module, preselector_class, preselector_config_file: Path +): + if preselector_config_file is None: + return None + else: + try: + preselector_config = load_from_json(preselector_config_file) + return load_preselector( + preselector_config, preselector_module, preselector_class + ) + except ConfigurationException: + logger.exception( + f"Unable to create preselector defined in: {preselector_config_file}" + ) + return None + + +def load_preselector( + preselector_config: str, + module: str, + preselector_class_name: str, + sensor_definition: dict, +) -> Preselector: + logger.debug( + f"loading {preselector_class_name} from {module} with config: {preselector_config}" + ) + if module is not None and preselector_class_name is not None: + preselector_module = importlib.import_module(module) + preselector_constructor = getattr(preselector_module, preselector_class_name) + preselector_config = load_from_json(preselector_config) + ps = preselector_constructor(sensor_definition, preselector_config) + register_component_with_status.send(ps, component=ps) + else: + ps = None + return ps + + +def load_switches(switch_dir: Path) -> dict: + logger.debug(f"Loading switches in {switch_dir}") + switch_dict = {} + try: + if switch_dir is not None and switch_dir.is_dir(): + for f in switch_dir.iterdir(): + file_path = f.resolve() + logger.debug(f"loading switch config {file_path}") + conf = load_from_json(file_path) + try: + switch = ControlByWebWebRelay(conf) + logger.debug(f"Adding {switch.id}") + switch_dict[switch.id] = switch + logger.debug(f"Registering switch status for {switch.name}") + register_component_with_status.send(__name__, component=switch) + except ConfigurationException: + logger.error(f"Unable to configure switch defined in: {file_path}") + except Exception as ex: + logger.error(f"Unable to load switches {ex}") + return switch_dict def status_registration_handler(sender, **kwargs): @@ -42,32 +101,109 @@ def set_container_unhealthy(): Path(settings.SDR_HEALTHCHECK_FILE).touch() +def get_calibration( + cal_file_path: str, cal_type: str +) -> Optional[Union[DifferentialCalibration, SensorCalibration]]: + """ + Load calibration data from file. + + :param cal_file_path: Path to the JSON calibration file. + :param cal_type: Calibration type to load: "sensor" or "differential" + :return: The ``Calibration`` object, if loaded, or ``None`` if loading failed. + """ + try: + cal = None + if cal_file_path is None or cal_file_path == "": + logger.error("No calibration file specified, reverting to none.") + elif not path.exists(cal_file_path): + logger.error(f"{cal_file_path} does not exist, reverting to none.") + else: + logger.debug(f"Loading calibration file: {cal_file_path}") + # Create calibration object + cal_file_path = Path(cal_file_path) + if cal_type.lower() in ["sensor", "onboard"]: + cal = SensorCalibration.from_json(cal_file_path) + elif cal_type.lower() == "differential": + cal = DifferentialCalibration.from_json(cal_file_path) + else: + logger.error(f"Unknown calibration type: {cal_type}") + raise ValueError + except Exception: + cal = None + logger.exception( + f"Unable to load {cal_type} calibration file from {cal_file_path}." + + " Reverting to none" + ) + finally: + return cal + + try: + sensor_loader = None register_component_with_status.connect(status_registration_handler) - usb_device_exists = get_usb_device_exists() - if usb_device_exists: - action_loader = ActionLoader() - logger.debug(f"Actions ActionLoader has {len(action_loader.actions)} actions") - capabilities_loader = CapabilitiesLoader() - logger.debug("Calling sensor loader.") - sensor_loader = SensorLoader(capabilities_loader.capabilities) + action_loader = ActionLoader() + logger.debug(f"Actions ActionLoader has {len(action_loader.actions)} actions") + capabilities_loader = CapabilitiesLoader() + switches = load_switches(settings.SWITCH_CONFIGS_DIR) + preselector = load_preselector( + settings.PRESELECTOR_CONFIG, + settings.PRESELECTOR_MODULE, + settings.PRESELECTOR_CLASS, + capabilities_loader.capabilities["sensor"], + ) + + if get_usb_device_exists(): + logger.debug("Initializing Sensor...") + sensor_loader = SensorLoader( + capabilities_loader.capabilities, switches, preselector + ) + + else: + logger.debug("Power cycling sigan") + try: + power_cycle_sigan(switches) + except Exception as power_cycle_exception: + logger.error(f"Unable to power cycle sigan: {power_cycle_exception}") + set_container_unhealthy() + time.sleep(60) + + if not settings.RUNNING_MIGRATIONS: if ( - not settings.RUNNING_MIGRATIONS - and not sensor_loader.sensor.signal_analyzer.healthy() + sensor_loader.sensor.signal_analyzer is None + or not sensor_loader.sensor.signal_analyzer.healthy() ): + try: + power_cycle_sigan(switches) + except Exception as power_cycle_exception: + logger.error(f"Unable to power cycle sigan: {power_cycle_exception}") set_container_unhealthy() - else: - action_loader = types.SimpleNamespace() - action_loader.actions = {} - capabilities_loader = types.SimpleNamespace() - capabilities_loader.capabilities = {} - sensor_loader = types.SimpleNamespace() - sensor_loader.sensor = types.SimpleNamespace() - sensor_loader.sensor.signal_analyzer = None - sensor_loader.preselector = None - sensor_loader.switches = {} - sensor_loader.capabilities = {} - logger.warning("Usb is not ready. Marking container as unhealthy") - set_container_unhealthy() -except: - logger.exception("Error during initialization") + time.sleep(60) + + # Calibration loading + if not settings.RUNNING_TESTS: + # Load the onboard cal file as the sensor calibration, if it exists + onboard_cal = get_calibration(settings.ONBOARD_CALIBRATION_FILE, "sensor") + if onboard_cal is not None: + sensor_loader.sensor.sensor_calibration = onboard_cal + else: + # Otherwise, try using the sensor calibration file + sensor_cal = get_calibration(settings.SENSOR_CALIBRATION_FILE, "sensor") + if sensor_cal is not None: + sensor_loader.sensor.sensor_calibration = sensor_cal + + # Now load the differential calibration, if it exists + differential_cal = get_calibration( + settings.DIFFERENTIAL_CALIBRATION_FILE, + "differential", + ) + sensor_loader.sensor.differential_calibration = differential_cal + + import ray + + if settings.RAY_INIT and not ray.is_initialized(): + # Dashboard is only enabled if ray[default] is installed + logger.debug("Initializing ray.") + ray.init() +except BaseException as error: + logger.exception(f"Error during initialization: {error}") + set_container_unhealthy() diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index 4ce54936..5a911743 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -1,33 +1,35 @@ import importlib import logging -from os import path -from pathlib import Path from django.conf import settings -from its_preselector.configuration_exception import ConfigurationException -from its_preselector.controlbyweb_web_relay import ControlByWebWebRelay +from environs import Env from its_preselector.preselector import Preselector -from scos_actions import utils -from scos_actions.calibration.calibration import Calibration, load_from_json from scos_actions.hardware.sensor import Sensor from scos_actions.metadata.utils import construct_geojson_point from utils.signals import register_component_with_status +from .utils import get_usb_device_exists, set_container_unhealthy + logger = logging.getLogger(__name__) +env = Env() class SensorLoader: _instance = None - def __init__(self, sensor_capabilities: dict): + def __init__( + self, sensor_capabilities: dict, switches: dict, preselector: Preselector + ): if not hasattr(self, "sensor"): logger.debug("Sensor has not been loaded. Loading...") - self._sensor = load_sensor(sensor_capabilities) + self._sensor = load_sensor(sensor_capabilities, switches, preselector) else: logger.debug("Already loaded sensor. ") - def __new__(cls, sensor_capabilities): + def __new__( + cls, sensor_capabilities: dict, switches: dict, preselector: Preselector + ): if cls._instance is None: logger.debug("Creating the SensorLoader") cls._instance = super().__new__(cls) @@ -38,11 +40,9 @@ def sensor(self) -> Sensor: return self._sensor -def load_sensor(sensor_capabilities: dict) -> Sensor: - switches = {} - sigan_cal = None - sensor_cal = None - preselector = None +def load_sensor( + sensor_capabilities: dict, switches: dict, preselector: Preselector +) -> Sensor: location = None if not settings.RUNNING_TESTS: # Remove location from sensor definition and convert to geojson. @@ -55,45 +55,38 @@ def load_sensor(sensor_capabilities: dict) -> Sensor: sensor_loc["y"], sensor_loc["z"] if "z" in sensor_loc else None, ) - switches = load_switches(settings.SWITCH_CONFIGS_DIR) - sensor_cal = get_sensor_calibration( - settings.SENSOR_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE - ) - sigan_cal = get_sigan_calibration( - settings.SIGAN_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE - ) - preselector = load_preselector( - settings.PRESELECTOR_CONFIG, - settings.PRESELECTOR_MODULE, - settings.PRESELECTOR_CLASS, - sensor_capabilities["sensor"], - ) sigan = None try: if not settings.RUNNING_MIGRATIONS: - check_for_required_sigan_settings() - sigan_module_setting = settings.SIGAN_MODULE - sigan_module = importlib.import_module(sigan_module_setting) - logger.info( - "Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE - ) - sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) - sigan = sigan_constructor( - sensor_cal=sensor_cal, sigan_cal=sigan_cal, switches=switches - ) - register_component_with_status.send(sigan, component=sigan) + if get_usb_device_exists(): + check_for_required_sigan_settings() + sigan_module_setting = settings.SIGAN_MODULE + sigan_module = importlib.import_module(sigan_module_setting) + logger.info( + f"Creating {settings.SIGAN_CLASS} from {settings.SIGAN_MODULE}" + ) + sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) + sigan = sigan_constructor(switches=switches) + register_component_with_status.send(sigan, component=sigan) + else: + logger.warning("Required USB Device does not exist.") else: logger.info("Running migrations. Not loading signal analyzer.") - except Exception as ex: + except BaseException as ex: logger.warning(f"unable to create signal analyzer: {ex}") + set_container_unhealthy() + # Create sensor before handling calibrations sensor = Sensor( signal_analyzer=sigan, + # TODO GPS Not Implemented capabilities=sensor_capabilities, preselector=preselector, switches=switches, location=location, + sensor_cal=None, + differential_cal=None, ) return sensor @@ -109,145 +102,3 @@ def check_for_required_sigan_settings(): error += "SIGAN_CLASS environment variable. " if raise_exception: raise Exception(error) - - -def load_switches(switch_dir: Path) -> dict: - logger.debug(f"Loading switches in {switch_dir}") - switch_dict = {} - try: - if switch_dir is not None and switch_dir.is_dir(): - for f in switch_dir.iterdir(): - file_path = f.resolve() - logger.debug(f"loading switch config {file_path}") - conf = utils.load_from_json(file_path) - try: - switch = ControlByWebWebRelay(conf) - logger.debug(f"Adding {switch.id}") - switch_dict[switch.id] = switch - logger.debug(f"Registering switch status for {switch.name}") - register_component_with_status.send(__name__, component=switch) - except ConfigurationException: - logger.error(f"Unable to configure switch defined in: {file_path}") - except Exception as ex: - logger.error(f"Unable to load switches {ex}") - return switch_dict - - -def load_preselector_from_file( - preselector_module, preselector_class, preselector_config_file: Path -): - if preselector_config_file is None: - return None - else: - try: - preselector_config = utils.load_from_json(preselector_config_file) - return load_preselector( - preselector_config, preselector_module, preselector_class - ) - except ConfigurationException: - logger.exception( - f"Unable to create preselector defined in: {preselector_config_file}" - ) - return None - - -def load_preselector( - preselector_config: str, - module: str, - preselector_class_name: str, - sensor_definition: dict, -) -> Preselector: - logger.debug( - f"loading {preselector_class_name} from {module} with config: {preselector_config}" - ) - if module is not None and preselector_class_name is not None: - preselector_module = importlib.import_module(module) - preselector_constructor = getattr(preselector_module, preselector_class_name) - preselector_config = utils.load_from_json(preselector_config) - ps = preselector_constructor(sensor_definition, preselector_config) - register_component_with_status.send(ps, component=ps) - else: - ps = None - return ps - - -def get_sigan_calibration( - sigan_cal_file_path: str, default_cal_file_path: str -) -> Calibration: - """ - Load signal analyzer calibration data from file. - - :param sigan_cal_file_path: Path to JSON file containing signal - analyzer calibration data. - :param default_cal_file_path: Path to the default cal file. - :return: The signal analyzer ``Calibration`` object. - """ - try: - sigan_cal = None - if sigan_cal_file_path is None or sigan_cal_file_path == "": - logger.warning( - "No sigan calibration file specified. Not loading calibration file." - ) - elif not path.exists(sigan_cal_file_path): - logger.warning( - sigan_cal_file_path - + " does not exist. Not loading sigan calibration file." - ) - else: - logger.debug(f"Loading sigan cal file: {sigan_cal_file_path}") - default = check_for_default_calibration( - sigan_cal_file_path, default_cal_file_path, "Sigan" - ) - sigan_cal = load_from_json(sigan_cal_file_path, default) - sigan_cal.is_default = default - except Exception: - sigan_cal = None - logger.exception("Unable to load sigan calibration data, reverting to none") - return sigan_cal - - -def get_sensor_calibration( - sensor_cal_file_path: str, default_cal_file_path: str -) -> Calibration: - """ - Load sensor calibration data from file. - - :param sensor_cal_file_path: Path to JSON file containing sensor - calibration data. - :param default_cal_file_path: Name of the default calibration file. - :return: The sensor ``Calibration`` object. - """ - try: - sensor_cal = None - if sensor_cal_file_path is None or sensor_cal_file_path == "": - logger.warning( - "No sensor calibration file specified. Not loading calibration file." - ) - elif not path.exists(sensor_cal_file_path): - logger.warning( - sensor_cal_file_path - + " does not exist. Not loading sensor calibration file." - ) - else: - logger.debug(f"Loading sensor cal file: {sensor_cal_file_path}") - default = check_for_default_calibration( - sensor_cal_file_path, default_cal_file_path, "Sensor" - ) - sensor_cal = load_from_json(sensor_cal_file_path, default) - sensor_cal.is_default = default - except Exception: - sensor_cal = None - logger.exception("Unable to load sensor calibration data, reverting to none") - return sensor_cal - - -def check_for_default_calibration( - cal_file_path: str, default_cal_path: str, cal_type: str -) -> bool: - default_cal = False - if cal_file_path == default_cal_path: - default_cal = True - logger.warning( - f"***************LOADING DEFAULT {cal_type} CALIBRATION***************" - ) - return default_cal diff --git a/src/initialization/tests/test_initialization.py b/src/initialization/tests/test_initialization.py index 706a2e15..bc2b92f8 100644 --- a/src/initialization/tests/test_initialization.py +++ b/src/initialization/tests/test_initialization.py @@ -1,7 +1,7 @@ import logging import os -from initialization.sensor_loader import load_preselector +from initialization import load_preselector logger = logging.getLogger(__name__) diff --git a/src/initialization/utils.py b/src/initialization/utils.py new file mode 100644 index 00000000..9eff0253 --- /dev/null +++ b/src/initialization/utils.py @@ -0,0 +1,26 @@ +import logging +import sys +from pathlib import Path +from subprocess import check_output + +from django.conf import settings + +logger = logging.getLogger(__name__) + + +def set_container_unhealthy(): + if settings.IN_DOCKER: + logger.warning("Marking container for restart.") + Path(settings.SDR_HEALTHCHECK_FILE).touch() + + +def get_usb_device_exists() -> bool: + logger.debug("Checking for USB...") + if not settings.MOCK_SIGAN and settings.USB_DEVICE is not None: + usb_devices = check_output("lsusb").decode(sys.stdout.encoding) + logger.debug("Checking for " + settings.USB_DEVICE) + logger.debug("Found " + usb_devices) + return settings.USB_DEVICE in usb_devices + else: + logger.debug("Skipping check for USB device") + return True diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index df30ffd1..7d955bd4 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -59,9 +59,7 @@ colorama==0.4.6 colorful==0.5.5 # via ray coverage[toml]==7.3.2 - # via - # coverage - # pytest-cov + # via pytest-cov cryptography==42.0.4 # via -r requirements.txt defusedxml==0.7.1 @@ -315,11 +313,11 @@ scipy==1.10.1 # via # -r requirements.txt # scos-actions -scos-actions @ git+https://github.com/NTIA/scos-actions@8.0.0 +scos-actions @ git+https://github.com/NTIA/scos-actions@9.0.0 # via # -r requirements.txt # scos-tekrsa -scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@5.0.0 +scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@6.0.0 # via -r requirements.txt sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive # via diff --git a/src/requirements.in b/src/requirements.in index a2362238..01179100 100644 --- a/src/requirements.in +++ b/src/requirements.in @@ -11,7 +11,7 @@ packaging>=23.0, <24.0 psycopg2-binary>=2.0, <3.0 requests-mock>=1.0, <2.0 requests_oauthlib>=1.0, <2.0 -scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@5.0.0 +scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@6.0.0 # The following are sub-dependencies for which SCOS Sensor enforces a # higher minimum patch version than the dependencies which require them. diff --git a/src/requirements.txt b/src/requirements.txt index 315f56cf..363648b3 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -149,9 +149,9 @@ ruamel-yaml-clib==0.2.8 # via ruamel-yaml scipy==1.10.1 # via scos-actions -scos-actions @ git+https://github.com/NTIA/scos-actions@8.0.0 +scos-actions @ git+https://github.com/NTIA/scos-actions@9.0.0 # via scos-tekrsa -scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@5.0.0 +scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@6.0.0 # via -r requirements.in sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive # via scos-actions diff --git a/src/scheduler/scheduler.py b/src/scheduler/scheduler.py index a264dd90..9f392079 100644 --- a/src/scheduler/scheduler.py +++ b/src/scheduler/scheduler.py @@ -1,4 +1,5 @@ """Queue and run tasks.""" + import json import logging import threading @@ -11,7 +12,7 @@ from scos_actions.hardware.sensor import Sensor from scos_actions.signals import trigger_api_restart -from initialization import sensor_loader +from initialization import action_loader, sensor_loader from schedule.models import ScheduleEntry from tasks.consts import MAX_DETAIL_LEN from tasks.models import TaskResult @@ -91,6 +92,7 @@ def run(self, blocking=True): pass try: + self.calibrate_if_needed() while True: with minimum_duration(blocking): self._consume_schedule(blocking) @@ -335,6 +337,38 @@ def __repr__(self): s = "running" if self.running else "stopped" return f"<{self.__class__.__name__} status={s}>" + def calibrate_if_needed(self): + # Now run the calibration action defined in the environment + # This will create an onboard_cal file if needed, and set it + # as the sensor's sensor_calibration. + if settings.MOCK_SIGAN: + logger.debug("Skipping startup calibration when using mock sigan") + return + if not settings.RUNNING_MIGRATIONS: + if ( + sensor_loader.sensor.sensor_calibration is None + or sensor_loader.sensor.sensor_calibration.expired() + ): + if settings.STARTUP_CALIBRATION_ACTION is None: + logger.error("No STARTUP_CALIBRATION_ACTION set.") + else: + logger.info("Performing startup calibration...") + try: + cal_action = action_loader.actions[ + settings.STARTUP_CALIBRATION_ACTION + ] + cal_action( + sensor=sensor_loader.sensor, + schedule_entry=None, + task_id=None, + ) + except BaseException as cal_error: + logger.error(f"Error during startup calibration: {cal_error}") + else: + logger.debug( + "Skipping startup calibration since sensor_calibration exists and has not expired." + ) + @contextmanager def minimum_duration(blocking): diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 1bc97746..d1a46a9f 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -9,6 +9,7 @@ https://docs.djangoproject.com/en/1.11/ref/settings/ """ + import logging import os import sys @@ -69,23 +70,35 @@ ACTIONS_DIR = path.join(CONFIG_DIR, "actions") DRIVERS_DIR = path.join(REPO_ROOT, "drivers") -DEFAULT_CALIBRATION_FILE = path.join(CONFIG_DIR, "default_calibration.json") -os.environ["DEFAULT_CALIBRATION_FILE"] = str(DEFAULT_CALIBRATION_FILE) -# JSON configs -if path.exists(path.join(CONFIG_DIR, "sensor_calibration.json")): - SENSOR_CALIBRATION_FILE = path.join(CONFIG_DIR, "sensor_calibration.json") -else: - SENSOR_CALIBRATION_FILE = DEFAULT_CALIBRATION_FILE +########### Calibration Files ################## -if path.exists(path.join(CONFIG_DIR, "sigan_calibration.json")): - SIGAN_CALIBRATION_FILE = path.join(CONFIG_DIR, "sigan_calibration.json") +# Onboard calibration file; should only be autogenerated by actions. +# If present and parseable, gain and noise figure values will be used in +# place of any present in the SENSOR_CALIBRATION_FILE. +ONBOARD_CALIBRATION_FILE = path.join(CONFIG_DIR, "onboard_sensor_calibration.json") +os.environ["ONBOARD_CALIBRATION_FILE"] = ONBOARD_CALIBRATION_FILE + +# Sensor calibration file; should be provided manually. Will not be +# overwritten by calibration actions. +if path.exists(sensor_cal_path := path.join(CONFIG_DIR, "sensor_calibration.json")): + SENSOR_CALIBRATION_FILE = sensor_cal_path +else: + SENSOR_CALIBRATION_FILE = None +os.environ["SENSOR_CALIBRATION_FILE"] = sensor_cal_path + +# Differential calibration file; should be provided manually. If present, +# this will be used to apply additional scaling on top of the sensor or onboard +# calibration. This can be used to shift the data reference point from the onboard +# calibration terminal to another point in the signal path. +if path.exists(diff_cal_path := path.join(CONFIG_DIR, "differential_calibration.json")): + DIFFERENTIAL_CALIBRATION_FILE = diff_cal_path else: - SIGAN_CALIBRATION_FILE = DEFAULT_CALIBRATION_FILE + DIFFERENTIAL_CALIBRATION_FILE = None +os.environ["DIFFERENTIAL_CALIBRATION_FILE"] = diff_cal_path -if path.exists(path.join(CONFIG_DIR, "sensor_definition.json")): - SENSOR_DEFINITION_FILE = path.join(CONFIG_DIR, "sensor_definition.json") -os.environ["SENSOR_CALIBRATION_FILE"] = SENSOR_CALIBRATION_FILE -os.environ["SIGAN_CALIBRATION_FILE"] = SIGAN_CALIBRATION_FILE +# Sensor Definition File +if path.exists(sensor_def_path := path.join(CONFIG_DIR, "sensor_definition.json")): + SENSOR_DEFINITION_FILE = sensor_def_path MEDIA_ROOT = path.join(REPO_ROOT, "files") PRESELECTOR_CONFIG = path.join(CONFIG_DIR, "preselector_config.json") @@ -438,3 +451,5 @@ MAX_FAILURES = env("MAX_FAILURES", default=2) os.environ["RUNNING_TESTS"] = str(RUNNING_TESTS) USB_DEVICE = env("USB_DEVICE", default=None) +STARTUP_CALIBRATION_ACTION = env("STARTUP_CALIBRATION_ACTION", default=None) +RAY_INIT = env.bool("RAY_INIT", default=False) diff --git a/src/sensor/utils.py b/src/sensor/utils.py index 3b2cd92d..b71ad05b 100644 --- a/src/sensor/utils.py +++ b/src/sensor/utils.py @@ -17,5 +17,3 @@ def get_timestamp_from_datetime(dt: datetime) -> int: def parse_datetime_str(d: str) -> datetime: return datetime.strptime(d, settings.DATETIME_FORMAT) - - diff --git a/src/sensor/wsgi.py b/src/sensor/wsgi.py index 610456e7..703f9628 100644 --- a/src/sensor/wsgi.py +++ b/src/sensor/wsgi.py @@ -30,4 +30,4 @@ if not settings.IN_DOCKER: # Normally scheduler is started by gunicorn worker process - scheduler.thread.start() \ No newline at end of file + scheduler.thread.start() diff --git a/src/status/views.py b/src/status/views.py index 64878835..c7d070d1 100644 --- a/src/status/views.py +++ b/src/status/views.py @@ -1,25 +1,25 @@ import datetime import logging import platform -import shutil import sys -from initialization import sensor_loader, status_monitor +from django.conf import settings from its_preselector import __version__ as PRESELECTOR_API_VERSION from its_preselector.preselector import Preselector from its_preselector.web_relay import WebRelay from rest_framework.decorators import api_view from rest_framework.response import Response -from scheduler import scheduler from scos_actions import __version__ as SCOS_ACTIONS_VERSION from scos_actions.hardware.sigan_iface import SignalAnalyzerInterface -from scos_actions.metadata.structs import ntia_diagnostics -from scos_actions.settings import SCOS_SENSOR_GIT_TAG from scos_actions.utils import ( convert_datetime_to_millisecond_iso_format, get_datetime_str_now, + get_disk_usage, ) +from initialization import sensor_loader, status_monitor +from scheduler import scheduler + from . import start_time from .serializers import LocationSerializer from .utils import get_location @@ -37,14 +37,6 @@ def serialize_location(): return None -def disk_usage(): - """Return the total disk usage as a percentage.""" - usage = shutil.disk_usage("/") - percent_used = round(100 * usage.used / usage.total) - logger.debug(str(percent_used) + " disk used") - return round(percent_used, 2) - - def get_days_up(): """Return the number of days SCOS has been running.""" elapsed = datetime.datetime.utcnow() - start_time @@ -58,18 +50,27 @@ def get_software_version(): software_version = { "system_platform": platform.platform(), "python_version": sys.version.split()[0], - "scos_sensor_version": SCOS_SENSOR_GIT_TAG, + "scos_sensor_version": settings.SCOS_SENSOR_GIT_TAG, "scos_actions_version": SCOS_ACTIONS_VERSION, "preselector_api_version": PRESELECTOR_API_VERSION, } - if sensor_loader.sensor is not None and sensor_loader.sensor.signal_analyzer is not None: + if ( + sensor_loader.sensor is not None + and sensor_loader.sensor.signal_analyzer is not None + ): if sensor_loader.sensor.signal_analyzer.firmware_version is not None: - software_version["sigan_firmware_version"] = sensor_loader.sensor.signal_analyzer.firmware_version + software_version["sigan_firmware_version"] = ( + sensor_loader.sensor.signal_analyzer.firmware_version + ) if sensor_loader.sensor.signal_analyzer.api_version is not None: - software_version["sigan_api_version"] = sensor_loader.sensor.signal_analyzer.api_version + software_version["sigan_api_version"] = ( + sensor_loader.sensor.signal_analyzer.api_version + ) if sensor_loader.sensor.signal_analyzer.plugin_version is not None: - software_version["scos_sigan_plugin"] = sensor_loader.sensor.signal_analyzer.plugin_version + software_version["scos_sigan_plugin"] = ( + sensor_loader.sensor.signal_analyzer.plugin_version + ) logger.debug(software_version) return software_version @@ -84,19 +85,16 @@ def status(request, version, format=None): "location": serialize_location(), "system_time": get_datetime_str_now(), "start_time": convert_datetime_to_millisecond_iso_format(start_time), - "disk_usage": disk_usage(), + "disk_usage": get_disk_usage(), "days_up": get_days_up(), "software": get_software_version(), } if ( sensor_loader.sensor is not None - and sensor_loader.sensor.signal_analyzer is not None - and sensor_loader.sensor.signal_analyzer.sensor_calibration is not None + and sensor_loader.sensor.sensor_calibration is not None ): - status_json[ - "last_calibration_datetime" - ] = ( - sensor_loader.sensor.signal_analyzer.sensor_calibration.last_calibration_datetime + status_json["last_calibration_datetime"] = ( + sensor_loader.sensor.sensor_calibration.last_calibration_datetime ) for component in status_monitor.status_components: component_status = component.get_status() diff --git a/src/tasks/__init__.py b/src/tasks/__init__.py index 8302e1a6..725c1d12 100644 --- a/src/tasks/__init__.py +++ b/src/tasks/__init__.py @@ -1,6 +1,4 @@ import logging - logger = logging.getLogger(__name__) logger.debug("********** Initializing tasks **********") - diff --git a/src/utils/signals.py b/src/utils/signals.py index 225298b5..c6753f6a 100644 --- a/src/utils/signals.py +++ b/src/utils/signals.py @@ -1,4 +1,4 @@ from django.dispatch import Signal -#provides component -register_component_with_status = Signal() \ No newline at end of file +# provides component +register_component_with_status = Signal()