From 19a1cb37eef51882c35b408dd7fa06e183b6c339 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 2 Feb 2022 13:32:44 +0000 Subject: [PATCH 01/84] FEA: Use a thread pool to read from the serial port --- data_gateway/cli.py | 29 ++++++++++++++++------------- data_gateway/exceptions.py | 4 ++++ setup.py | 2 +- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/data_gateway/cli.py b/data_gateway/cli.py index 4d26224f..51a49439 100644 --- a/data_gateway/cli.py +++ b/data_gateway/cli.py @@ -12,7 +12,7 @@ from data_gateway.configuration import Configuration from data_gateway.dummy_serial import DummySerial -from data_gateway.exceptions import DataMustBeSavedError, WrongNumberOfSensorCoordinatesError +from data_gateway.exceptions import DataMustBeSavedError, PacketReaderStopError, WrongNumberOfSensorCoordinatesError from data_gateway.routine import Routine @@ -166,7 +166,7 @@ def start( [startBaros, startMics, startIMU, getBattery, stop]. """ import sys - import threading + from concurrent.futures import ThreadPoolExecutor from data_gateway.packet_reader import PacketReader @@ -207,13 +207,14 @@ def start( window_size, ) - # Start packet reader in a separate thread so commands can be sent to it in real time in interactive mode or by a - # routine. - reader_thread = threading.Thread(target=packet_reader.read_packets, args=(serial_port,), daemon=True) - reader_thread.setName("ReaderThread") - reader_thread.start() + # Start packet reader in a thread pool for parallelised reading and so commands can be sent to the serial port in + # real time. + reader_thread_pool = ThreadPoolExecutor(thread_name_prefix="ReaderThread") try: + for _ in range(reader_thread_pool._max_workers): + reader_thread_pool.submit(packet_reader.read_packets, args=[serial_port]) + if interactive: # Keep a record of the commands given. commands_record_file = os.path.join( @@ -231,8 +232,7 @@ def start( if line.startswith("sleep") and line.endswith("\n"): time.sleep(int(line.split(" ")[-1].strip())) elif line == "stop\n": - packet_reader.stop = True - break + raise PacketReaderStopError() # Send the command to the node serial_port.write(line.encode("utf_8")) @@ -241,11 +241,14 @@ def start( if routine is not None: routine.run() - except KeyboardInterrupt: - packet_reader.stop = True + except (KeyboardInterrupt, PacketReaderStopError): + pass - logger.info("Stopping gateway.") - packet_reader.writer.force_persist() + finally: + logger.info("Stopping gateway.") + packet_reader.stop = True + reader_thread_pool.shutdown(wait=False, cancel_futures=True) + packet_reader.writer.force_persist() @gateway_cli.command() diff --git a/data_gateway/exceptions.py b/data_gateway/exceptions.py index 99bb2012..54731959 100644 --- a/data_gateway/exceptions.py +++ b/data_gateway/exceptions.py @@ -18,3 +18,7 @@ class WrongNumberOfSensorCoordinatesError(GatewayError, ValueError): class DataMustBeSavedError(GatewayError, ValueError): """Raise if options are given to the packet reader that mean no data will be saved locally or uploaded to the cloud.""" + + +class PacketReaderStopError(GatewayError): + pass diff --git a/setup.py b/setup.py index 40bb6332..62f45ec2 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ setup( name="data_gateway", - version="0.9.0", + version="0.10.0", install_requires=[ "click>=7.1.2", "pyserial==3.5", From dd5ad5d4db44d19df41d67d1b53bb1412e234f81 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 2 Feb 2022 13:41:20 +0000 Subject: [PATCH 02/84] FIX: Remove python3.9-only argument in ThreadPoolExecutor.shutdown --- data_gateway/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_gateway/cli.py b/data_gateway/cli.py index 51a49439..c2157c34 100644 --- a/data_gateway/cli.py +++ b/data_gateway/cli.py @@ -247,7 +247,7 @@ def start( finally: logger.info("Stopping gateway.") packet_reader.stop = True - reader_thread_pool.shutdown(wait=False, cancel_futures=True) + reader_thread_pool.shutdown(wait=False) packet_reader.writer.force_persist() From d3d2930b5b9b189ec84d2aee4aaf5eece4144736 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 2 Feb 2022 15:36:13 +0000 Subject: [PATCH 03/84] FIX: Pass args to threads correctly --- data_gateway/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_gateway/cli.py b/data_gateway/cli.py index c2157c34..d9f7f4f1 100644 --- a/data_gateway/cli.py +++ b/data_gateway/cli.py @@ -213,7 +213,7 @@ def start( try: for _ in range(reader_thread_pool._max_workers): - reader_thread_pool.submit(packet_reader.read_packets, args=[serial_port]) + reader_thread_pool.submit(packet_reader.read_packets, serial_port) if interactive: # Keep a record of the commands given. From 3d67438300a238f2fd079c5eda16c51d08bbe659 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 2 Feb 2022 15:42:30 +0000 Subject: [PATCH 04/84] ENH: Log the start-up of reader threads at debug level --- data_gateway/packet_reader.py | 1 + 1 file changed, 1 insertion(+) diff --git a/data_gateway/packet_reader.py b/data_gateway/packet_reader.py index e0dc3334..c7f2f726 100644 --- a/data_gateway/packet_reader.py +++ b/data_gateway/packet_reader.py @@ -85,6 +85,7 @@ def read_packets(self, serial_port, stop_when_no_more_data=False): :param bool stop_when_no_more_data: stop reading when no more data is received from the port (for testing) :return None: """ + logger.debug("Beginning reading for packets.") self._persist_configuration() previous_timestamp = {} From b1399494bc7c35d777e417063fec9d6a38cd8985 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 2 Feb 2022 15:45:50 +0000 Subject: [PATCH 05/84] ENH: Enumerate parser threads according to their reader thread number --- data_gateway/packet_reader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data_gateway/packet_reader.py b/data_gateway/packet_reader.py index c7f2f726..e70801aa 100644 --- a/data_gateway/packet_reader.py +++ b/data_gateway/packet_reader.py @@ -109,7 +109,8 @@ def read_packets(self, serial_port, stop_when_no_more_data=False): daemon=True, ) - parser_thread.setName("ParserThread") + current_reader_thread_number = threading.current_thread().name.split("_")[-1] + parser_thread.setName(f"ParserThread_{current_reader_thread_number}") parser_thread.start() while not self.stop: From 324d2ac221ff3af7118ee494bdbaad6ffb662ffd Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 2 Feb 2022 16:11:37 +0000 Subject: [PATCH 06/84] DOC: Update docstring --- data_gateway/persistence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_gateway/persistence.py b/data_gateway/persistence.py index 58f966f7..ffce5485 100644 --- a/data_gateway/persistence.py +++ b/data_gateway/persistence.py @@ -64,7 +64,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): self.force_persist() def add_to_current_window(self, sensor_name, data): - """Add serialised data (a string) to the current window for the given sensor name. + """Add data to the current window for the given sensor name. :param str sensor_name: name of sensor :param iter data: data to add to window From e2d10cf02741ff032b317b45ffb76b0a12d970ac Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 2 Feb 2022 16:34:28 +0000 Subject: [PATCH 07/84] REF: Move logic for starting gateway into new DataGateway class --- data_gateway/cli.py | 192 +++---------------------------- data_gateway/data_gateway.py | 216 +++++++++++++++++++++++++++++++++++ 2 files changed, 233 insertions(+), 175 deletions(-) create mode 100644 data_gateway/data_gateway.py diff --git a/data_gateway/cli.py b/data_gateway/cli.py index d9f7f4f1..ede2af0c 100644 --- a/data_gateway/cli.py +++ b/data_gateway/cli.py @@ -1,19 +1,15 @@ import json import logging import os -import time import click import pkg_resources import requests -import serial from requests import HTTPError from slugify import slugify -from data_gateway.configuration import Configuration -from data_gateway.dummy_serial import DummySerial -from data_gateway.exceptions import DataMustBeSavedError, PacketReaderStopError, WrongNumberOfSensorCoordinatesError -from data_gateway.routine import Routine +from data_gateway.data_gateway import DataGateway +from data_gateway.exceptions import WrongNumberOfSensorCoordinatesError SUPERVISORD_PROGRAM_NAME = "AerosenseGateway" @@ -165,90 +161,23 @@ def start( nodes/sensors via the serial port by typing them into stdin and pressing enter. These commands are: [startBaros, startMics, startIMU, getBattery, stop]. """ - import sys - from concurrent.futures import ThreadPoolExecutor - - from data_gateway.packet_reader import PacketReader - - if not save_locally and no_upload_to_cloud: - raise DataMustBeSavedError( - "Data from the gateway must either be saved locally or uploaded to the cloud. Please adjust the CLI " - "options provided." - ) - - config = _load_configuration(configuration_path=config_file) - config.session_data["label"] = label - - serial_port = _get_serial_port(serial_port, configuration=config, use_dummy_serial_port=use_dummy_serial_port) - routine = _load_routine(routine_path=routine_file, interactive=interactive, serial_port=serial_port) - output_directory = _update_and_create_output_directory(output_directory_path=output_dir) - - # Start a new thread to parse the serial data while the main thread stays ready to take in commands from stdin. - packet_reader = PacketReader( - save_locally=save_locally, - upload_to_cloud=not no_upload_to_cloud, - output_directory=output_directory, - window_size=window_size, - project_name=gcp_project_name, - bucket_name=gcp_bucket_name, - configuration=config, - save_csv_files=save_csv_files, + data_gateway = DataGateway( + serial_port, + config_file, + routine_file, + save_locally, + no_upload_to_cloud, + interactive, + output_dir, + window_size, + gcp_project_name, + gcp_bucket_name, + label, + save_csv_files, + use_dummy_serial_port, ) - logger.info("Starting packet reader.") - - if not no_upload_to_cloud: - logger.info("Files will be uploaded to cloud storage at intervals of %s seconds.", window_size) - - if save_locally: - logger.info( - "Files will be saved locally to disk at %r at intervals of %s seconds.", - os.path.join(packet_reader.output_directory, packet_reader.session_subdirectory), - window_size, - ) - - # Start packet reader in a thread pool for parallelised reading and so commands can be sent to the serial port in - # real time. - reader_thread_pool = ThreadPoolExecutor(thread_name_prefix="ReaderThread") - - try: - for _ in range(reader_thread_pool._max_workers): - reader_thread_pool.submit(packet_reader.read_packets, serial_port) - - if interactive: - # Keep a record of the commands given. - commands_record_file = os.path.join( - packet_reader.output_directory, packet_reader.session_subdirectory, "commands.txt" - ) - - os.makedirs(os.path.join(packet_reader.output_directory, packet_reader.session_subdirectory), exist_ok=True) - - while not packet_reader.stop: - for line in sys.stdin: - - with open(commands_record_file, "a") as f: - f.write(line) - - if line.startswith("sleep") and line.endswith("\n"): - time.sleep(int(line.split(" ")[-1].strip())) - elif line == "stop\n": - raise PacketReaderStopError() - - # Send the command to the node - serial_port.write(line.encode("utf_8")) - - else: - if routine is not None: - routine.run() - - except (KeyboardInterrupt, PacketReaderStopError): - pass - - finally: - logger.info("Stopping gateway.") - packet_reader.stop = True - reader_thread_pool.shutdown(wait=False) - packet_reader.writer.force_persist() + data_gateway.start() @gateway_cli.command() @@ -334,92 +263,5 @@ def supervisord_conf(config_file): return 0 -def _load_configuration(configuration_path): - """Load a configuration from the path if it exists, otherwise load the default configuration. - - :param str configuration_path: - :return data_gateway.configuration.Configuration: - """ - if os.path.exists(configuration_path): - with open(configuration_path) as f: - configuration = Configuration.from_dict(json.load(f)) - - logger.info("Loaded configuration file from %r.", configuration_path) - return configuration - - configuration = Configuration() - logger.info("No configuration file provided - using default configuration.") - return configuration - - -def _get_serial_port(serial_port, configuration, use_dummy_serial_port): - """Get the serial port or a dummy serial port if specified. - - :param str serial_port: - :param data_gateway.configuration.Configuration configuration: - :param bool use_dummy_serial_port: - :return serial.Serial: - """ - if not use_dummy_serial_port: - serial_port = serial.Serial(port=serial_port, baudrate=configuration.baudrate) - else: - serial_port = DummySerial(port=serial_port, baudrate=configuration.baudrate) - - # The buffer size can only be set on Windows. - if os.name == "nt": - serial_port.set_buffer_size( - rx_size=configuration.serial_buffer_rx_size, - tx_size=configuration.serial_buffer_tx_size, - ) - else: - logger.warning("Serial port buffer size can only be set on Windows.") - - return serial_port - - -def _load_routine(routine_path, interactive, serial_port): - """Load a sensor commands routine from the path if exists, otherwise return no routine. If in interactive mode, the - routine file is ignored. Note that "\n" has to be added to the end of each command sent to the serial port for it to - be executed - this is done automatically in this method. - - :param str routine_path: - :param bool interactive: - :param serial.Serial serial_port: - :return data_gateway.routine.Routine|None: - """ - if os.path.exists(routine_path): - if interactive: - logger.warning("Sensor command routine files are ignored in interactive mode.") - return - else: - with open(routine_path) as f: - routine = Routine( - **json.load(f), - action=lambda command: serial_port.write((command + "\n").encode("utf_8")), - ) - - logger.info("Loaded routine file from %r.", routine_path) - return routine - - logger.info( - "No routine file found at %r - no commands will be sent to the sensors unless given in interactive mode.", - routine_path, - ) - - -def _update_and_create_output_directory(output_directory_path): - """Set the output directory to a path relative to the current directory if the path does not start with "/" and - create it if it does not already exist. - - :param str output_directory_path: - :return str: - """ - if not output_directory_path.startswith("/"): - output_directory_path = os.path.join(".", output_directory_path) - - os.makedirs(output_directory_path, exist_ok=True) - return output_directory_path - - if __name__ == "__main__": gateway_cli() diff --git a/data_gateway/data_gateway.py b/data_gateway/data_gateway.py new file mode 100644 index 00000000..dee63c8f --- /dev/null +++ b/data_gateway/data_gateway.py @@ -0,0 +1,216 @@ +import json +import logging +import os +import sys +import time +from concurrent.futures import ThreadPoolExecutor + +import serial + +from data_gateway.configuration import Configuration +from data_gateway.dummy_serial import DummySerial +from data_gateway.exceptions import DataMustBeSavedError, PacketReaderStopError +from data_gateway.packet_reader import PacketReader +from data_gateway.routine import Routine + + +logger = logging.getLogger(__name__) + + +class DataGateway: + def __init__( + self, + serial_port, + config_file, + routine_file, + save_locally, + no_upload_to_cloud, + interactive, + output_dir, + window_size, + gcp_project_name, + gcp_bucket_name, + label, + save_csv_files, + use_dummy_serial_port, + ): + if not save_locally and no_upload_to_cloud: + raise DataMustBeSavedError( + "Data from the gateway must either be saved locally or uploaded to the cloud. Please adjust the CLI " + "options provided." + ) + + self.serial_port = serial_port + self.interactive = interactive + self.no_upload_to_cloud = no_upload_to_cloud + self.save_locally = save_locally + self.window_size = window_size + + packet_reader_configuration = self._load_configuration(configuration_path=config_file) + packet_reader_configuration.session_data["label"] = label + + self.serial_port = self._get_serial_port( + serial_port, + configuration=packet_reader_configuration, + use_dummy_serial_port=use_dummy_serial_port, + ) + + self.routine = self._load_routine(routine_path=routine_file) + + # Start a new thread to parse the serial data while the main thread stays ready to take in commands from stdin. + self.packet_reader = PacketReader( + save_locally=save_locally, + upload_to_cloud=not no_upload_to_cloud, + output_directory=self._update_and_create_output_directory(output_directory_path=output_dir), + window_size=window_size, + project_name=gcp_project_name, + bucket_name=gcp_bucket_name, + configuration=packet_reader_configuration, + save_csv_files=save_csv_files, + ) + + def start(self): + """Begin reading and persisting data from the serial port for the sensors at the installation defined in + the configuration. In interactive mode, commands can be sent to the nodes/sensors via the serial port by typing + them into stdin and pressing enter. These commands are: [startBaros, startMics, startIMU, getBattery, stop]. + + :return None: + """ + logger.info("Starting packet reader.") + + if not self.no_upload_to_cloud: + logger.info("Files will be uploaded to cloud storage at intervals of %s seconds.", self.window_size) + + if self.save_locally: + logger.info( + "Files will be saved locally to disk at %r at intervals of %s seconds.", + os.path.join(self.packet_reader.output_directory, self.packet_reader.session_subdirectory), + self.window_size, + ) + + # Start packet reader in a thread pool for parallelised reading and so commands can be sent to the serial port + # in real time. + reader_thread_pool = ThreadPoolExecutor(thread_name_prefix="ReaderThread") + + try: + for _ in range(reader_thread_pool._max_workers): + reader_thread_pool.submit(self.packet_reader.read_packets, self.serial_port) + + if self.interactive: + # Keep a record of the commands given. + commands_record_file = os.path.join( + self.packet_reader.output_directory, self.packet_reader.session_subdirectory, "commands.txt" + ) + + os.makedirs( + os.path.join(self.packet_reader.output_directory, self.packet_reader.session_subdirectory), + exist_ok=True, + ) + + while not self.packet_reader.stop: + for line in sys.stdin: + + with open(commands_record_file, "a") as f: + f.write(line) + + if line.startswith("sleep") and line.endswith("\n"): + time.sleep(int(line.split(" ")[-1].strip())) + elif line == "stop\n": + raise PacketReaderStopError() + + # Send the command to the node + self.serial_port.write(line.encode("utf_8")) + + else: + if self.routine is not None: + self.routine.run() + + except (KeyboardInterrupt, PacketReaderStopError): + pass + + finally: + logger.info("Stopping gateway.") + self.packet_reader.stop = True + reader_thread_pool.shutdown(wait=False) + self.packet_reader.writer.force_persist() + + def _load_configuration(self, configuration_path): + """Load a configuration from the path if it exists, otherwise load the default configuration. + + :param str configuration_path: + :return data_gateway.configuration.Configuration: + """ + if os.path.exists(configuration_path): + with open(configuration_path) as f: + configuration = Configuration.from_dict(json.load(f)) + + logger.debug("Loaded configuration file from %r.", configuration_path) + return configuration + + configuration = Configuration() + logger.info("No configuration file provided - using default configuration.") + return configuration + + def _get_serial_port(self, serial_port, configuration, use_dummy_serial_port): + """Get the serial port or a dummy serial port if specified. + + :param str serial_port: + :param data_gateway.configuration.Configuration configuration: + :param bool use_dummy_serial_port: + :return serial.Serial: + """ + if not use_dummy_serial_port: + serial_port = serial.Serial(port=serial_port, baudrate=configuration.baudrate) + else: + serial_port = DummySerial(port=serial_port, baudrate=configuration.baudrate) + + # The buffer size can only be set on Windows. + if os.name == "nt": + serial_port.set_buffer_size( + rx_size=configuration.serial_buffer_rx_size, + tx_size=configuration.serial_buffer_tx_size, + ) + else: + logger.warning("Serial port buffer size can only be set on Windows.") + + return serial_port + + def _load_routine(self, routine_path): + """Load a sensor commands routine from the path if exists, otherwise return no routine. If in interactive mode, + the routine file is ignored. Note that "\n" has to be added to the end of each command sent to the serial port + for it to be executed - this is done automatically in this method. + + :param str routine_path: + :return data_gateway.routine.Routine|None: + """ + if os.path.exists(routine_path): + if self.interactive: + logger.warning("Sensor command routine files are ignored in interactive mode.") + return + else: + with open(routine_path) as f: + routine = Routine( + **json.load(f), + action=lambda command: self.serial_port.write((command + "\n").encode("utf_8")), + ) + + logger.info("Loaded routine file from %r.", routine_path) + return routine + + logger.info( + "No routine file found at %r - no commands will be sent to the sensors unless given in interactive mode.", + routine_path, + ) + + def _update_and_create_output_directory(self, output_directory_path): + """Set the output directory to a path relative to the current directory if the path does not start with "/" and + create it if it does not already exist. + + :param str output_directory_path: + :return str: + """ + if not output_directory_path.startswith("/"): + output_directory_path = os.path.join(".", output_directory_path) + + os.makedirs(output_directory_path, exist_ok=True) + return output_directory_path From a33a4a521dd7dba39f3648f4db4ec047ae02e6ed Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 2 Feb 2022 17:06:07 +0000 Subject: [PATCH 08/84] ENH: Send all packets from reader threads to single parser thread --- data_gateway/data_gateway.py | 66 ++++++++------ data_gateway/packet_reader.py | 157 ++++++++++++++++------------------ 2 files changed, 115 insertions(+), 108 deletions(-) diff --git a/data_gateway/data_gateway.py b/data_gateway/data_gateway.py index dee63c8f..fdc4239c 100644 --- a/data_gateway/data_gateway.py +++ b/data_gateway/data_gateway.py @@ -1,7 +1,9 @@ import json import logging import os +import queue import sys +import threading import time from concurrent.futures import ThreadPoolExecutor @@ -88,38 +90,27 @@ def start(self): self.window_size, ) - # Start packet reader in a thread pool for parallelised reading and so commands can be sent to the serial port - # in real time. + self.packet_reader._persist_configuration() + reader_thread_pool = ThreadPoolExecutor(thread_name_prefix="ReaderThread") + packet_queue = queue.Queue() + error_queue = queue.Queue() try: for _ in range(reader_thread_pool._max_workers): - reader_thread_pool.submit(self.packet_reader.read_packets, self.serial_port) - - if self.interactive: - # Keep a record of the commands given. - commands_record_file = os.path.join( - self.packet_reader.output_directory, self.packet_reader.session_subdirectory, "commands.txt" - ) - - os.makedirs( - os.path.join(self.packet_reader.output_directory, self.packet_reader.session_subdirectory), - exist_ok=True, - ) + reader_thread_pool.submit(self.packet_reader.read_packets, self.serial_port, packet_queue, error_queue) - while not self.packet_reader.stop: - for line in sys.stdin: - - with open(commands_record_file, "a") as f: - f.write(line) + parser_thread = threading.Thread( + target=self.packet_reader.parse_payload, + kwargs={"packet_queue": packet_queue, "error_queue": error_queue}, + daemon=True, + ) - if line.startswith("sleep") and line.endswith("\n"): - time.sleep(int(line.split(" ")[-1].strip())) - elif line == "stop\n": - raise PacketReaderStopError() + parser_thread.setName("ParserThread") + parser_thread.start() - # Send the command to the node - self.serial_port.write(line.encode("utf_8")) + if self.interactive: + self._send_commands_to_sensors() else: if self.routine is not None: @@ -134,6 +125,31 @@ def start(self): reader_thread_pool.shutdown(wait=False) self.packet_reader.writer.force_persist() + def _send_commands_to_sensors(self): + # Keep a record of the commands given. + commands_record_file = os.path.join( + self.packet_reader.output_directory, self.packet_reader.session_subdirectory, "commands.txt" + ) + + os.makedirs( + os.path.join(self.packet_reader.output_directory, self.packet_reader.session_subdirectory), + exist_ok=True, + ) + + while not self.packet_reader.stop: + for line in sys.stdin: + + with open(commands_record_file, "a") as f: + f.write(line) + + if line.startswith("sleep") and line.endswith("\n"): + time.sleep(int(line.split(" ")[-1].strip())) + elif line == "stop\n": + raise PacketReaderStopError() + + # Send the command to the node + self.serial_port.write(line.encode("utf_8")) + def _load_configuration(self, configuration_path): """Load a configuration from the path if it exists, otherwise load the default configuration. diff --git a/data_gateway/packet_reader.py b/data_gateway/packet_reader.py index e70801aa..2ac4b802 100644 --- a/data_gateway/packet_reader.py +++ b/data_gateway/packet_reader.py @@ -2,9 +2,7 @@ import json import logging import os -import queue import struct -import threading from octue.cloud import storage @@ -77,7 +75,7 @@ def __init__( else: self.writer = NoOperationContextManager() - def read_packets(self, serial_port, stop_when_no_more_data=False): + def read_packets(self, serial_port, packet_queue, error_queue, stop_when_no_more_data=False): """Read packets from a serial port and send them to a separate thread that will parse and upload them to Google Cloud storage and/or write them to disk. @@ -85,8 +83,7 @@ def read_packets(self, serial_port, stop_when_no_more_data=False): :param bool stop_when_no_more_data: stop reading when no more data is received from the port (for testing) :return None: """ - logger.debug("Beginning reading for packets.") - self._persist_configuration() + logger.debug("Beginning reading packets from serial port.") previous_timestamp = {} data = {} @@ -98,61 +95,47 @@ def read_packets(self, serial_port, stop_when_no_more_data=False): for _ in range(self.config.number_of_sensors[sensor_name]) ] - with self.uploader: - with self.writer: - packet_queue = queue.Queue() - error_queue = queue.Queue() + while not self.stop: + if not error_queue.empty(): + raise error_queue.get() - parser_thread = threading.Thread( - target=self._parse_payload, - kwargs={"packet_queue": packet_queue, "error_queue": error_queue}, - daemon=True, - ) + serial_data = serial_port.read() - current_reader_thread_number = threading.current_thread().name.split("_")[-1] - parser_thread.setName(f"ParserThread_{current_reader_thread_number}") - parser_thread.start() - - while not self.stop: - if not error_queue.empty(): - raise error_queue.get() - - serial_data = serial_port.read() - - if len(serial_data) == 0: - if stop_when_no_more_data: - break - continue - - if serial_data[0] != self.config.packet_key: - continue - - packet_type = str(int.from_bytes(serial_port.read(), self.config.endian)) - length = int.from_bytes(serial_port.read(), self.config.endian) - payload = serial_port.read(length) - - if packet_type == str(self.config.type_handle_def): - self.update_handles(payload) - continue - - # Check for bytes in serial input buffer. A full buffer results in overflow. - if serial_port.in_waiting == self.config.serial_buffer_rx_size: - logger.warning( - "Buffer is full: %d bytes waiting. Re-opening serial port, to avoid overflow", - serial_port.in_waiting, - ) - serial_port.close() - serial_port.open() - continue - - packet_queue.put( - { - "packet_type": packet_type, - "payload": payload, - "data": data, - "previous_timestamp": previous_timestamp, - } - ) + if len(serial_data) == 0: + if stop_when_no_more_data: + break + continue + + if serial_data[0] != self.config.packet_key: + continue + + packet_type = str(int.from_bytes(serial_port.read(), self.config.endian)) + length = int.from_bytes(serial_port.read(), self.config.endian) + payload = serial_port.read(length) + logger.debug("Packet received.") + + if packet_type == str(self.config.type_handle_def): + self.update_handles(payload) + continue + + # Check for bytes in serial input buffer. A full buffer results in overflow. + if serial_port.in_waiting == self.config.serial_buffer_rx_size: + logger.warning( + "Buffer is full: %d bytes waiting. Re-opening serial port, to avoid overflow", + serial_port.in_waiting, + ) + serial_port.close() + serial_port.open() + continue + + packet_queue.put( + { + "packet_type": packet_type, + "payload": payload, + "data": data, + "previous_timestamp": previous_timestamp, + } + ) def update_handles(self, payload): """Update the Bluetooth handles object. Handles are updated every time a new Bluetooth connection is @@ -209,7 +192,7 @@ def _persist_configuration(self): ), ) - def _parse_payload(self, packet_queue, error_queue): + def parse_payload(self, packet_queue, error_queue): """Get packets from a thread-safe packet queue, check if a full payload has been received (i.e. correct length) with the correct packet type handle, then parse the payload. After parsing/processing, upload them to Google Cloud storage and/or write them to disk. If any errors are raised, put them on the error queue for the reader @@ -219,33 +202,41 @@ def _parse_payload(self, packet_queue, error_queue): :param queue.Queue error_queue: a thread-safe queue to put any exceptions on to for the reader thread to handle :return None: """ - try: - while not self.stop: - packet_type, payload, data, previous_timestamp = packet_queue.get().values() - - if packet_type not in self.handles: - logger.error("Received packet with unknown type: %s", packet_type) - raise exceptions.UnknownPacketTypeError("Received packet with unknown type: {}".format(packet_type)) - - if len(payload) == 244: # If the full data payload is received, proceed parsing it - timestamp = int.from_bytes(payload[240:244], self.config.endian, signed=False) / (2 ** 16) - - data, sensor_names = self._parse_sensor_packet_data(self.handles[packet_type], payload, data) + with self.uploader: + with self.writer: + try: + while not self.stop: + packet_type, payload, data, previous_timestamp = packet_queue.get().values() + + if packet_type not in self.handles: + logger.error("Received packet with unknown type: %s", packet_type) + raise exceptions.UnknownPacketTypeError( + "Received packet with unknown type: {}".format(packet_type) + ) - for sensor_name in sensor_names: - self._check_for_packet_loss(sensor_name, timestamp, previous_timestamp) - self._timestamp_and_persist_data(data, sensor_name, timestamp, self.config.period[sensor_name]) + if len(payload) == 244: # If the full data payload is received, proceed parsing it + timestamp = int.from_bytes(payload[240:244], self.config.endian, signed=False) / (2 ** 16) - elif len(payload) >= 1 and self.handles[packet_type] in [ - "Mic 1", - "Cmd Decline", - "Sleep State", - "Info Message", - ]: - self._parse_info_packet(self.handles[packet_type], payload) + data, sensor_names = self._parse_sensor_packet_data( + self.handles[packet_type], payload, data + ) - except Exception as e: - error_queue.put(e) + for sensor_name in sensor_names: + self._check_for_packet_loss(sensor_name, timestamp, previous_timestamp) + self._timestamp_and_persist_data( + data, sensor_name, timestamp, self.config.period[sensor_name] + ) + + elif len(payload) >= 1 and self.handles[packet_type] in [ + "Mic 1", + "Cmd Decline", + "Sleep State", + "Info Message", + ]: + self._parse_info_packet(self.handles[packet_type], payload) + + except Exception as e: + error_queue.put(e) def _parse_sensor_packet_data(self, packet_type, payload, data): """Parse sensor data type payloads. From f2e4a633d1fc6756dbacc73d9c0f1003d3c9f7a8 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 2 Feb 2022 18:23:41 +0000 Subject: [PATCH 09/84] FIX: Raise errors from reader and parser threads in main thread --- data_gateway/data_gateway.py | 87 +++++++++++++++----------- data_gateway/packet_reader.py | 112 +++++++++++++++++----------------- 2 files changed, 109 insertions(+), 90 deletions(-) diff --git a/data_gateway/data_gateway.py b/data_gateway/data_gateway.py index fdc4239c..6c3c80d5 100644 --- a/data_gateway/data_gateway.py +++ b/data_gateway/data_gateway.py @@ -23,18 +23,18 @@ class DataGateway: def __init__( self, serial_port, - config_file, - routine_file, - save_locally, - no_upload_to_cloud, - interactive, - output_dir, - window_size, - gcp_project_name, - gcp_bucket_name, - label, - save_csv_files, - use_dummy_serial_port, + config_file="config.json", + routine_file="routine.json", + save_locally=False, + no_upload_to_cloud=False, + interactive=False, + output_directory="data_gateway", + window_size=600, + project_name=None, + bucket_name=None, + label=None, + save_csv_files=False, + use_dummy_serial_port=False, ): if not save_locally and no_upload_to_cloud: raise DataMustBeSavedError( @@ -63,10 +63,10 @@ def __init__( self.packet_reader = PacketReader( save_locally=save_locally, upload_to_cloud=not no_upload_to_cloud, - output_directory=self._update_and_create_output_directory(output_directory_path=output_dir), + output_directory=self._update_and_create_output_directory(output_directory_path=output_directory), window_size=window_size, - project_name=gcp_project_name, - bucket_name=gcp_bucket_name, + project_name=project_name, + bucket_name=bucket_name, configuration=packet_reader_configuration, save_csv_files=save_csv_files, ) @@ -101,20 +101,35 @@ def start(self): reader_thread_pool.submit(self.packet_reader.read_packets, self.serial_port, packet_queue, error_queue) parser_thread = threading.Thread( + name="ParserThread", target=self.packet_reader.parse_payload, kwargs={"packet_queue": packet_queue, "error_queue": error_queue}, daemon=True, ) - parser_thread.setName("ParserThread") parser_thread.start() if self.interactive: - self._send_commands_to_sensors() + commands_thread = threading.Thread( + name="SensorCommandsThread", + target=self._send_commands_to_sensors, + daemon=True, + ) - else: - if self.routine is not None: - self.routine.run() + commands_thread.start() + + elif self.routine is not None: + commands_thread = threading.Thread( + name="SensorCommandsThread", + target=self.routine.run, + daemon=True, + ) + + commands_thread.start() + + while not self.packet_reader.stop: + if not error_queue.empty(): + raise error_queue.get() except (KeyboardInterrupt, PacketReaderStopError): pass @@ -128,7 +143,9 @@ def start(self): def _send_commands_to_sensors(self): # Keep a record of the commands given. commands_record_file = os.path.join( - self.packet_reader.output_directory, self.packet_reader.session_subdirectory, "commands.txt" + self.packet_reader.output_directory, + self.packet_reader.session_subdirectory, + "commands.txt", ) os.makedirs( @@ -138,7 +155,6 @@ def _send_commands_to_sensors(self): while not self.packet_reader.stop: for line in sys.stdin: - with open(commands_record_file, "a") as f: f.write(line) @@ -175,19 +191,20 @@ def _get_serial_port(self, serial_port, configuration, use_dummy_serial_port): :param bool use_dummy_serial_port: :return serial.Serial: """ - if not use_dummy_serial_port: - serial_port = serial.Serial(port=serial_port, baudrate=configuration.baudrate) - else: - serial_port = DummySerial(port=serial_port, baudrate=configuration.baudrate) - - # The buffer size can only be set on Windows. - if os.name == "nt": - serial_port.set_buffer_size( - rx_size=configuration.serial_buffer_rx_size, - tx_size=configuration.serial_buffer_tx_size, - ) - else: - logger.warning("Serial port buffer size can only be set on Windows.") + if isinstance(serial_port, str): + if not use_dummy_serial_port: + serial_port = serial.Serial(port=serial_port, baudrate=configuration.baudrate) + else: + serial_port = DummySerial(port=serial_port, baudrate=configuration.baudrate) + + # The buffer size can only be set on Windows. + if os.name == "nt": + serial_port.set_buffer_size( + rx_size=configuration.serial_buffer_rx_size, + tx_size=configuration.serial_buffer_tx_size, + ) + else: + logger.warning("Serial port buffer size can only be set on Windows.") return serial_port diff --git a/data_gateway/packet_reader.py b/data_gateway/packet_reader.py index 2ac4b802..b20faad2 100644 --- a/data_gateway/packet_reader.py +++ b/data_gateway/packet_reader.py @@ -83,59 +83,60 @@ def read_packets(self, serial_port, packet_queue, error_queue, stop_when_no_more :param bool stop_when_no_more_data: stop reading when no more data is received from the port (for testing) :return None: """ - logger.debug("Beginning reading packets from serial port.") + try: + logger.debug("Beginning reading packets from serial port.") - previous_timestamp = {} - data = {} + previous_timestamp = {} + data = {} - for sensor_name in self.config.sensor_names: - previous_timestamp[sensor_name] = -1 - data[sensor_name] = [ - ([0] * self.config.samples_per_packet[sensor_name]) - for _ in range(self.config.number_of_sensors[sensor_name]) - ] - - while not self.stop: - if not error_queue.empty(): - raise error_queue.get() - - serial_data = serial_port.read() - - if len(serial_data) == 0: - if stop_when_no_more_data: - break - continue - - if serial_data[0] != self.config.packet_key: - continue - - packet_type = str(int.from_bytes(serial_port.read(), self.config.endian)) - length = int.from_bytes(serial_port.read(), self.config.endian) - payload = serial_port.read(length) - logger.debug("Packet received.") - - if packet_type == str(self.config.type_handle_def): - self.update_handles(payload) - continue - - # Check for bytes in serial input buffer. A full buffer results in overflow. - if serial_port.in_waiting == self.config.serial_buffer_rx_size: - logger.warning( - "Buffer is full: %d bytes waiting. Re-opening serial port, to avoid overflow", - serial_port.in_waiting, + for sensor_name in self.config.sensor_names: + previous_timestamp[sensor_name] = -1 + data[sensor_name] = [ + ([0] * self.config.samples_per_packet[sensor_name]) + for _ in range(self.config.number_of_sensors[sensor_name]) + ] + + while not self.stop: + serial_data = serial_port.read() + + if len(serial_data) == 0: + if stop_when_no_more_data: + break + continue + + if serial_data[0] != self.config.packet_key: + continue + + packet_type = str(int.from_bytes(serial_port.read(), self.config.endian)) + length = int.from_bytes(serial_port.read(), self.config.endian) + payload = serial_port.read(length) + logger.debug("Packet received.") + + if packet_type == str(self.config.type_handle_def): + self.update_handles(payload) + continue + + # Check for bytes in serial input buffer. A full buffer results in overflow. + if serial_port.in_waiting == self.config.serial_buffer_rx_size: + logger.warning( + "Buffer is full: %d bytes waiting. Re-opening serial port, to avoid overflow", + serial_port.in_waiting, + ) + serial_port.close() + serial_port.open() + continue + + packet_queue.put( + { + "packet_type": packet_type, + "payload": payload, + "data": data, + "previous_timestamp": previous_timestamp, + } ) - serial_port.close() - serial_port.open() - continue - - packet_queue.put( - { - "packet_type": packet_type, - "payload": payload, - "data": data, - "previous_timestamp": previous_timestamp, - } - ) + + except Exception as e: + error_queue.put(e) def update_handles(self, payload): """Update the Bluetooth handles object. Handles are updated every time a new Bluetooth connection is @@ -202,14 +203,15 @@ def parse_payload(self, packet_queue, error_queue): :param queue.Queue error_queue: a thread-safe queue to put any exceptions on to for the reader thread to handle :return None: """ - with self.uploader: - with self.writer: - try: + try: + with self.uploader: + with self.writer: while not self.stop: packet_type, payload, data, previous_timestamp = packet_queue.get().values() if packet_type not in self.handles: logger.error("Received packet with unknown type: %s", packet_type) + raise exceptions.UnknownPacketTypeError( "Received packet with unknown type: {}".format(packet_type) ) @@ -235,8 +237,8 @@ def parse_payload(self, packet_queue, error_queue): ]: self._parse_info_packet(self.handles[packet_type], payload) - except Exception as e: - error_queue.put(e) + except Exception as e: + error_queue.put(e) def _parse_sensor_packet_data(self, packet_type, payload, data): """Parse sensor data type payloads. From 2d03e01c152ef4a4db4587311e72bf3313c15f68 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 2 Feb 2022 18:35:37 +0000 Subject: [PATCH 10/84] FIX: Re-enable stopping when no more data is received from serial port --- data_gateway/data_gateway.py | 10 ++++++++-- data_gateway/packet_reader.py | 1 + 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/data_gateway/data_gateway.py b/data_gateway/data_gateway.py index 6c3c80d5..7f41881a 100644 --- a/data_gateway/data_gateway.py +++ b/data_gateway/data_gateway.py @@ -71,7 +71,7 @@ def __init__( save_csv_files=save_csv_files, ) - def start(self): + def start(self, stop_when_no_more_data=False): """Begin reading and persisting data from the serial port for the sensors at the installation defined in the configuration. In interactive mode, commands can be sent to the nodes/sensors via the serial port by typing them into stdin and pressing enter. These commands are: [startBaros, startMics, startIMU, getBattery, stop]. @@ -98,7 +98,13 @@ def start(self): try: for _ in range(reader_thread_pool._max_workers): - reader_thread_pool.submit(self.packet_reader.read_packets, self.serial_port, packet_queue, error_queue) + reader_thread_pool.submit( + self.packet_reader.read_packets, + self.serial_port, + packet_queue, + error_queue, + stop_when_no_more_data, + ) parser_thread = threading.Thread( name="ParserThread", diff --git a/data_gateway/packet_reader.py b/data_gateway/packet_reader.py index b20faad2..36b1f8db 100644 --- a/data_gateway/packet_reader.py +++ b/data_gateway/packet_reader.py @@ -101,6 +101,7 @@ def read_packets(self, serial_port, packet_queue, error_queue, stop_when_no_more if len(serial_data) == 0: if stop_when_no_more_data: + self.stop = True break continue From c499ba0e6e0daa04dd66ed1ed34acc715e2e4e4f Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 2 Feb 2022 18:55:01 +0000 Subject: [PATCH 11/84] FIX: Add missing force persist to cloud when data gateway stops --- data_gateway/data_gateway.py | 1 + 1 file changed, 1 insertion(+) diff --git a/data_gateway/data_gateway.py b/data_gateway/data_gateway.py index 7f41881a..1829ac05 100644 --- a/data_gateway/data_gateway.py +++ b/data_gateway/data_gateway.py @@ -145,6 +145,7 @@ def start(self, stop_when_no_more_data=False): self.packet_reader.stop = True reader_thread_pool.shutdown(wait=False) self.packet_reader.writer.force_persist() + self.packet_reader.uploader.force_persist() def _send_commands_to_sensors(self): # Keep a record of the commands given. From 3400f74f319934804364f25fbf11acfa81eb9bc7 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 2 Feb 2022 19:08:35 +0000 Subject: [PATCH 12/84] FIX: Add force_persist method to NoOperationContextManager --- data_gateway/persistence.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/data_gateway/persistence.py b/data_gateway/persistence.py index ffce5485..da65ee4a 100644 --- a/data_gateway/persistence.py +++ b/data_gateway/persistence.py @@ -30,6 +30,13 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): pass + def force_persist(self): + """Do nothing. + + :return None: + """ + pass + class TimeBatcher: """A batcher that groups the given data into time windows. From ab904d7574b76821e054609b6d970195ae77c0d9 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 2 Feb 2022 19:11:07 +0000 Subject: [PATCH 13/84] TST: Update tests to use DataGateway --- tests/test_packet_reader.py | 217 ++++++++++++++++++++++-------------- 1 file changed, 134 insertions(+), 83 deletions(-) diff --git a/tests/test_packet_reader.py b/tests/test_packet_reader.py index 21d90d25..4d6fba9e 100644 --- a/tests/test_packet_reader.py +++ b/tests/test_packet_reader.py @@ -8,8 +8,8 @@ from data_gateway import exceptions from data_gateway.configuration import Configuration +from data_gateway.data_gateway import DataGateway from data_gateway.dummy_serial import DummySerial -from data_gateway.packet_reader import PacketReader from tests import LENGTH, PACKET_KEY, RANDOM_BYTES, TEST_BUCKET_NAME, TEST_PROJECT_NAME from tests.base import BaseTestCase @@ -73,16 +73,16 @@ def test_error_is_raised_if_unknown_sensor_type_packet_is_received(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[1]))) with tempfile.TemporaryDirectory() as temporary_directory: - packet_reader = PacketReader( + data_gateway = DataGateway( + serial_port=serial_port, save_locally=True, - upload_to_cloud=True, output_directory=temporary_directory, window_size=self.WINDOW_SIZE, project_name=TEST_PROJECT_NAME, bucket_name=TEST_BUCKET_NAME, ) with self.assertRaises(exceptions.UnknownPacketTypeError): - packet_reader.read_packets(serial_port, stop_when_no_more_data=False) + data_gateway.start() def test_configuration_file_is_persisted(self): """Test that the configuration file is persisted.""" @@ -93,18 +93,19 @@ def test_configuration_file_is_persisted(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[1]))) with tempfile.TemporaryDirectory() as temporary_directory: - packet_reader = PacketReader( + data_gateway = DataGateway( + serial_port=serial_port, save_locally=True, - upload_to_cloud=True, output_directory=temporary_directory, window_size=self.WINDOW_SIZE, project_name=TEST_PROJECT_NAME, bucket_name=TEST_BUCKET_NAME, ) - packet_reader.read_packets(serial_port, stop_when_no_more_data=True) + + data_gateway.start(stop_when_no_more_data=True) configuration_path = os.path.join( - temporary_directory, packet_reader.session_subdirectory, "configuration.json" + temporary_directory, data_gateway.packet_reader.session_subdirectory, "configuration.json" ) # Check configuration file is present and valid locally. @@ -115,7 +116,9 @@ def test_configuration_file_is_persisted(self): configuration = self.storage_client.download_as_string( bucket_name=TEST_BUCKET_NAME, path_in_bucket=storage.path.join( - packet_reader.uploader.output_directory, packet_reader.session_subdirectory, "configuration.json" + data_gateway.packet_reader.uploader.output_directory, + data_gateway.packet_reader.session_subdirectory, + "configuration.json", ), ) @@ -136,18 +139,18 @@ def test_update_handles_fails_if_start_and_end_handles_are_incorrect(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, payload))) with tempfile.TemporaryDirectory() as temporary_directory: - packet_reader = PacketReader( + data_gateway = DataGateway( + serial_port, save_locally=True, - upload_to_cloud=False, output_directory=temporary_directory, window_size=self.WINDOW_SIZE, project_name=TEST_PROJECT_NAME, bucket_name=TEST_BUCKET_NAME, ) - with patch("data_gateway.packet_reader.logger") as mock_logger: - packet_reader.read_packets(serial_port, stop_when_no_more_data=True) - self.assertIn("Handle error", mock_logger.method_calls[0].args[0]) + with self.assertLogs() as logging_context: + data_gateway.start(stop_when_no_more_data=True) + self.assertIn("Handle error", logging_context.output[3]) def test_update_handles(self): """Test that the handles can be updated.""" @@ -163,20 +166,21 @@ def test_update_handles(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, payload))) with tempfile.TemporaryDirectory() as temporary_directory: - packet_reader = PacketReader( + data_gateway = DataGateway( + serial_port, save_locally=True, - upload_to_cloud=False, + no_upload_to_cloud=True, output_directory=temporary_directory, window_size=self.WINDOW_SIZE, project_name=TEST_PROJECT_NAME, bucket_name=TEST_BUCKET_NAME, ) - with patch("data_gateway.packet_reader.logger") as mock_logger: - packet_reader.read_packets(serial_port, stop_when_no_more_data=True) - self.assertIn("Successfully updated handles", mock_logger.method_calls[0].args[0]) + with self.assertLogs() as logging_context: + data_gateway.start(stop_when_no_more_data=True) + self.assertIn("Successfully updated handles", logging_context.output[2]) - def test_packet_reader_with_baros_p_sensor(self): + def test_data_gateway_with_baros_p_sensor(self): """Test that the packet reader works with the Baro_P sensor.""" serial_port = DummySerial(port="test") packet_type = bytes([34]) @@ -185,20 +189,25 @@ def test_packet_reader_with_baros_p_sensor(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[1]))) with tempfile.TemporaryDirectory() as temporary_directory: - packet_reader = PacketReader( + data_gateway = DataGateway( + serial_port, save_locally=True, - upload_to_cloud=True, output_directory=temporary_directory, window_size=self.WINDOW_SIZE, project_name=TEST_PROJECT_NAME, bucket_name=TEST_BUCKET_NAME, ) - packet_reader.read_packets(serial_port, stop_when_no_more_data=True) - self._check_data_is_written_to_files(packet_reader, temporary_directory, sensor_names=["Baros_P"]) - self._check_windows_are_uploaded_to_cloud(packet_reader, sensor_names=["Baros_P"], number_of_windows_to_check=1) + data_gateway.start(stop_when_no_more_data=True) + self._check_data_is_written_to_files( + data_gateway.packet_reader, temporary_directory, sensor_names=["Baros_P"] + ) + + self._check_windows_are_uploaded_to_cloud( + data_gateway.packet_reader, sensor_names=["Baros_P"], number_of_windows_to_check=1 + ) - def test_packet_reader_with_baros_t_sensor(self): + def test_data_gateway_with_baros_t_sensor(self): """Test that the packet reader works with the Baro_T sensor.""" serial_port = DummySerial(port="test") packet_type = bytes([34]) @@ -207,20 +216,24 @@ def test_packet_reader_with_baros_t_sensor(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[1]))) with tempfile.TemporaryDirectory() as temporary_directory: - packet_reader = PacketReader( + data_gateway = DataGateway( + serial_port, save_locally=True, - upload_to_cloud=True, output_directory=temporary_directory, window_size=self.WINDOW_SIZE, project_name=TEST_PROJECT_NAME, bucket_name=TEST_BUCKET_NAME, ) - packet_reader.read_packets(serial_port, stop_when_no_more_data=True) - self._check_data_is_written_to_files(packet_reader, temporary_directory, sensor_names=["Baros_T"]) + data_gateway.start(stop_when_no_more_data=True) + self._check_data_is_written_to_files( + data_gateway.packet_reader, temporary_directory, sensor_names=["Baros_T"] + ) - self._check_windows_are_uploaded_to_cloud(packet_reader, sensor_names=["Baros_T"], number_of_windows_to_check=1) + self._check_windows_are_uploaded_to_cloud( + data_gateway.packet_reader, sensor_names=["Baros_T"], number_of_windows_to_check=1 + ) - def test_packet_reader_with_diff_baros_sensor(self): + def test_data_gateway_with_diff_baros_sensor(self): """Test that the packet reader works with the Diff_Baros sensor.""" serial_port = DummySerial(port="test") packet_type = bytes([36]) @@ -229,24 +242,26 @@ def test_packet_reader_with_diff_baros_sensor(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[1]))) with tempfile.TemporaryDirectory() as temporary_directory: - packet_reader = PacketReader( + data_gateway = DataGateway( + serial_port, save_locally=True, - upload_to_cloud=True, output_directory=temporary_directory, window_size=self.WINDOW_SIZE, project_name=TEST_PROJECT_NAME, bucket_name=TEST_BUCKET_NAME, ) - packet_reader.read_packets(serial_port, stop_when_no_more_data=True) - self._check_data_is_written_to_files(packet_reader, temporary_directory, sensor_names=["Diff_Baros"]) + data_gateway.start(stop_when_no_more_data=True) + self._check_data_is_written_to_files( + data_gateway.packet_reader, temporary_directory, sensor_names=["Diff_Baros"] + ) self._check_windows_are_uploaded_to_cloud( - packet_reader, + data_gateway.packet_reader, sensor_names=["Diff_Baros"], number_of_windows_to_check=1, ) - def test_packet_reader_with_mic_sensor(self): + def test_data_gateway_with_mic_sensor(self): """Test that the packet reader works with the mic sensor.""" serial_port = DummySerial(port="test") packet_type = bytes([38]) @@ -255,20 +270,22 @@ def test_packet_reader_with_mic_sensor(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[1]))) with tempfile.TemporaryDirectory() as temporary_directory: - packet_reader = PacketReader( + data_gateway = DataGateway( + serial_port, save_locally=True, - upload_to_cloud=True, output_directory=temporary_directory, window_size=self.WINDOW_SIZE, project_name=TEST_PROJECT_NAME, bucket_name=TEST_BUCKET_NAME, ) - packet_reader.read_packets(serial_port, stop_when_no_more_data=True) - self._check_data_is_written_to_files(packet_reader, temporary_directory, sensor_names=["Mics"]) + data_gateway.start(stop_when_no_more_data=True) + self._check_data_is_written_to_files(data_gateway.packet_reader, temporary_directory, sensor_names=["Mics"]) - self._check_windows_are_uploaded_to_cloud(packet_reader, sensor_names=["Mics"], number_of_windows_to_check=1) + self._check_windows_are_uploaded_to_cloud( + data_gateway.packet_reader, sensor_names=["Mics"], number_of_windows_to_check=1 + ) - def test_packet_reader_with_acc_sensor(self): + def test_data_gateway_with_acc_sensor(self): """Test that the packet reader works with the acc sensor.""" serial_port = DummySerial(port="test") packet_type = bytes([42]) @@ -277,20 +294,22 @@ def test_packet_reader_with_acc_sensor(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[1]))) with tempfile.TemporaryDirectory() as temporary_directory: - packet_reader = PacketReader( + data_gateway = DataGateway( + serial_port, save_locally=True, - upload_to_cloud=True, output_directory=temporary_directory, window_size=self.WINDOW_SIZE, project_name=TEST_PROJECT_NAME, bucket_name=TEST_BUCKET_NAME, ) - packet_reader.read_packets(serial_port, stop_when_no_more_data=True) - self._check_data_is_written_to_files(packet_reader, temporary_directory, sensor_names=["Acc"]) + data_gateway.start(stop_when_no_more_data=True) + self._check_data_is_written_to_files(data_gateway.packet_reader, temporary_directory, sensor_names=["Acc"]) - self._check_windows_are_uploaded_to_cloud(packet_reader, sensor_names=["Acc"], number_of_windows_to_check=1) + self._check_windows_are_uploaded_to_cloud( + data_gateway.packet_reader, sensor_names=["Acc"], number_of_windows_to_check=1 + ) - def test_packet_reader_with_gyro_sensor(self): + def test_data_gateway_with_gyro_sensor(self): """Test that the packet reader works with the gyro sensor.""" serial_port = DummySerial(port="test") packet_type = bytes([44]) @@ -299,20 +318,22 @@ def test_packet_reader_with_gyro_sensor(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[1]))) with tempfile.TemporaryDirectory() as temporary_directory: - packet_reader = PacketReader( + data_gateway = DataGateway( + serial_port, save_locally=True, - upload_to_cloud=True, output_directory=temporary_directory, window_size=self.WINDOW_SIZE, project_name=TEST_PROJECT_NAME, bucket_name=TEST_BUCKET_NAME, ) - packet_reader.read_packets(serial_port, stop_when_no_more_data=True) - self._check_data_is_written_to_files(packet_reader, temporary_directory, sensor_names=["Gyro"]) + data_gateway.start(stop_when_no_more_data=True) + self._check_data_is_written_to_files(data_gateway.packet_reader, temporary_directory, sensor_names=["Gyro"]) - self._check_windows_are_uploaded_to_cloud(packet_reader, sensor_names=["Gyro"], number_of_windows_to_check=1) + self._check_windows_are_uploaded_to_cloud( + data_gateway.packet_reader, sensor_names=["Gyro"], number_of_windows_to_check=1 + ) - def test_packet_reader_with_mag_sensor(self): + def test_data_gateway_with_mag_sensor(self): """Test that the packet reader works with the mag sensor.""" serial_port = DummySerial(port="test") packet_type = bytes([46]) @@ -321,20 +342,22 @@ def test_packet_reader_with_mag_sensor(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[1]))) with tempfile.TemporaryDirectory() as temporary_directory: - packet_reader = PacketReader( + data_gateway = DataGateway( + serial_port, save_locally=True, - upload_to_cloud=True, output_directory=temporary_directory, window_size=self.WINDOW_SIZE, project_name=TEST_PROJECT_NAME, bucket_name=TEST_BUCKET_NAME, ) - packet_reader.read_packets(serial_port, stop_when_no_more_data=True) - self._check_data_is_written_to_files(packet_reader, temporary_directory, sensor_names=["Mag"]) + data_gateway.start(stop_when_no_more_data=True) + self._check_data_is_written_to_files(data_gateway.packet_reader, temporary_directory, sensor_names=["Mag"]) - self._check_windows_are_uploaded_to_cloud(packet_reader, sensor_names=["Mag"], number_of_windows_to_check=1) + self._check_windows_are_uploaded_to_cloud( + data_gateway.packet_reader, sensor_names=["Mag"], number_of_windows_to_check=1 + ) - def test_packet_reader_with_connections_statistics(self): + def test_data_gateway_with_connections_statistics(self): """Test that the packet reader works with the connection statistics "sensor".""" serial_port = DummySerial(port="test") packet_type = bytes([52]) @@ -343,21 +366,25 @@ def test_packet_reader_with_connections_statistics(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[1]))) with tempfile.TemporaryDirectory() as temporary_directory: - packet_reader = PacketReader( + data_gateway = DataGateway( + serial_port, save_locally=True, - upload_to_cloud=True, output_directory=temporary_directory, window_size=self.WINDOW_SIZE, project_name=TEST_PROJECT_NAME, bucket_name=TEST_BUCKET_NAME, ) - packet_reader.read_packets(serial_port, stop_when_no_more_data=True) + data_gateway.start(stop_when_no_more_data=True) - self._check_data_is_written_to_files(packet_reader, temporary_directory, sensor_names=["Constat"]) + self._check_data_is_written_to_files( + data_gateway.packet_reader, temporary_directory, sensor_names=["Constat"] + ) - self._check_windows_are_uploaded_to_cloud(packet_reader, sensor_names=["Constat"], number_of_windows_to_check=1) + self._check_windows_are_uploaded_to_cloud( + data_gateway.packet_reader, sensor_names=["Constat"], number_of_windows_to_check=1 + ) - def test_packet_reader_with_connections_statistics_in_sleep_mode(self): + def test_data_gateway_with_connections_statistics_in_sleep_mode(self): """Test that the packet reader works with the connection statistics "sensor" in sleep state. Normally, randomly generated payloads would trigger packet loss warning in logger. Check that this warning is suppressed in sleep mode. @@ -372,9 +399,10 @@ def test_packet_reader_with_connections_statistics_in_sleep_mode(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[1]))) with tempfile.TemporaryDirectory() as temporary_directory: - packet_reader = PacketReader( + data_gateway = DataGateway( + serial_port, save_locally=True, - upload_to_cloud=False, + no_upload_to_cloud=True, output_directory=temporary_directory, window_size=self.WINDOW_SIZE, project_name=TEST_PROJECT_NAME, @@ -382,9 +410,11 @@ def test_packet_reader_with_connections_statistics_in_sleep_mode(self): ) with patch("data_gateway.packet_reader.logger") as mock_logger: - packet_reader.read_packets(serial_port, stop_when_no_more_data=True) + data_gateway.start(stop_when_no_more_data=True) - self._check_data_is_written_to_files(packet_reader, temporary_directory, sensor_names=["Constat"]) + self._check_data_is_written_to_files( + data_gateway.packet_reader, temporary_directory, sensor_names=["Constat"] + ) self.assertEqual(0, mock_logger.warning.call_count) def test_all_sensors_together(self): @@ -398,23 +428,27 @@ def test_all_sensors_together(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[1]))) with tempfile.TemporaryDirectory() as temporary_directory: - packet_reader = PacketReader( + data_gateway = DataGateway( + serial_port, save_locally=True, - upload_to_cloud=True, output_directory=temporary_directory, window_size=self.WINDOW_SIZE, project_name=TEST_PROJECT_NAME, bucket_name=TEST_BUCKET_NAME, ) - packet_reader.read_packets(serial_port, stop_when_no_more_data=True) + data_gateway.start(stop_when_no_more_data=True) - self._check_data_is_written_to_files(packet_reader, temporary_directory, sensor_names=sensor_names) + self._check_data_is_written_to_files( + data_gateway.packet_reader, temporary_directory, sensor_names=sensor_names + ) self._check_windows_are_uploaded_to_cloud( - packet_reader, sensor_names=sensor_names, number_of_windows_to_check=1 + data_gateway.packet_reader, + sensor_names=sensor_names, + number_of_windows_to_check=1, ) - def test_packet_reader_with_info_packets(self): + def test_data_gateway_with_info_packets(self): """Test that the packet reader works with info packets.""" serial_port = DummySerial(port="test") @@ -432,15 +466,32 @@ def test_packet_reader_with_info_packets(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, bytes([1]), payload))) with tempfile.TemporaryDirectory() as temporary_directory: - packet_reader = PacketReader( + data_gateway = DataGateway( + serial_port, save_locally=True, - upload_to_cloud=False, + no_upload_to_cloud=True, output_directory=temporary_directory, window_size=self.WINDOW_SIZE, project_name=TEST_PROJECT_NAME, bucket_name=TEST_BUCKET_NAME, ) - with patch("data_gateway.packet_reader.logger") as mock_logger: - packet_reader.read_packets(serial_port, stop_when_no_more_data=True) - self.assertEqual(11, len(mock_logger.method_calls)) + with self.assertLogs() as logging_context: + data_gateway.start(stop_when_no_more_data=True) + + log_messages_combined = "\n".join(logging_context.output) + + for message in [ + "Microphone data reading done", + "Microphone data erasing done", + "Microphones started ", + "Command declined, Bad block detection ongoing", + "Command declined, Task already registered, cannot register again", + "Command declined, Task is not registered, cannot de-register", + "Command declined, Connection Parameter update unfinished", + "\nExiting sleep\n", + "\nEntering sleep\n", + "Battery info", + "Voltage : 0.000000V\n Cycle count: 0.000000\nState of charge: 0.000000%", + ]: + self.assertIn(message, log_messages_combined) From 9394e99d2a8c2f45b050f67587c31dd13282b041 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 2 Feb 2022 19:13:00 +0000 Subject: [PATCH 14/84] TST: Rename test module and class --- ..._packet_reader.py => test_data_gateway.py} | 78 +++++++++---------- 1 file changed, 39 insertions(+), 39 deletions(-) rename tests/{test_packet_reader.py => test_data_gateway.py} (98%) diff --git a/tests/test_packet_reader.py b/tests/test_data_gateway.py similarity index 98% rename from tests/test_packet_reader.py rename to tests/test_data_gateway.py index 4d6fba9e..086bd37c 100644 --- a/tests/test_packet_reader.py +++ b/tests/test_data_gateway.py @@ -14,10 +14,10 @@ from tests.base import BaseTestCase -class TestPacketReader(BaseTestCase): - """Test packet reader with different sensors. NOTE: The payloads are generated randomly. Consequently, - two consecutive packets are extremely unlikely to have consecutive timestamps. This will trigger lost packet - warning during tests. +class TestDataGateway(BaseTestCase): + """Test `DataGateway` with different sensors. NOTE: The payloads are generated randomly. Consequently, two + consecutive packets are extremely unlikely to have consecutive timestamps. This will trigger lost packet warning + during tests. """ @classmethod @@ -29,41 +29,6 @@ def setUpClass(cls): cls.WINDOW_SIZE = 10 cls.storage_client = GoogleCloudStorageClient(project_name=TEST_PROJECT_NAME) - def _check_windows_are_uploaded_to_cloud(self, packet_reader, sensor_names, number_of_windows_to_check=5): - """Check that non-trivial windows from a packet reader for a particular sensor are uploaded to cloud storage.""" - number_of_windows = packet_reader.uploader._window_number - self.assertTrue(number_of_windows > 0) - - for i in range(number_of_windows_to_check): - data = json.loads( - self.storage_client.download_as_string( - bucket_name=TEST_BUCKET_NAME, - path_in_bucket=storage.path.join( - packet_reader.uploader.output_directory, - packet_reader.uploader._session_subdirectory, - f"window-{i}.json", - ), - ) - ) - - for name in sensor_names: - lines = data["sensor_data"][name] - self.assertTrue(len(lines[0]) > 1) - - def _check_data_is_written_to_files(self, packet_reader, temporary_directory, sensor_names): - """Check that non-trivial data is written to the given file.""" - window_directory = os.path.join(temporary_directory, packet_reader.writer._session_subdirectory) - windows = [file for file in os.listdir(window_directory) if file.startswith(packet_reader.writer._file_prefix)] - self.assertTrue(len(windows) > 0) - - for window in windows: - with open(os.path.join(window_directory, window)) as f: - data = json.load(f) - - for name in sensor_names: - lines = data["sensor_data"][name] - self.assertTrue(len(lines[0]) > 1) - def test_error_is_raised_if_unknown_sensor_type_packet_is_received(self): """Test that an `UnknownPacketTypeException` is raised if an unknown sensor type packet is received.""" serial_port = DummySerial(port="test") @@ -495,3 +460,38 @@ def test_data_gateway_with_info_packets(self): "Voltage : 0.000000V\n Cycle count: 0.000000\nState of charge: 0.000000%", ]: self.assertIn(message, log_messages_combined) + + def _check_windows_are_uploaded_to_cloud(self, packet_reader, sensor_names, number_of_windows_to_check=5): + """Check that non-trivial windows from a packet reader for a particular sensor are uploaded to cloud storage.""" + number_of_windows = packet_reader.uploader._window_number + self.assertTrue(number_of_windows > 0) + + for i in range(number_of_windows_to_check): + data = json.loads( + self.storage_client.download_as_string( + bucket_name=TEST_BUCKET_NAME, + path_in_bucket=storage.path.join( + packet_reader.uploader.output_directory, + packet_reader.uploader._session_subdirectory, + f"window-{i}.json", + ), + ) + ) + + for name in sensor_names: + lines = data["sensor_data"][name] + self.assertTrue(len(lines[0]) > 1) + + def _check_data_is_written_to_files(self, packet_reader, temporary_directory, sensor_names): + """Check that non-trivial data is written to the given file.""" + window_directory = os.path.join(temporary_directory, packet_reader.writer._session_subdirectory) + windows = [file for file in os.listdir(window_directory) if file.startswith(packet_reader.writer._file_prefix)] + self.assertTrue(len(windows) > 0) + + for window in windows: + with open(os.path.join(window_directory, window)) as f: + data = json.load(f) + + for name in sensor_names: + lines = data["sensor_data"][name] + self.assertTrue(len(lines[0]) > 1) From 16c84e90b441e49bed78f302a73a26cdbde8dca1 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 2 Feb 2022 21:45:31 +0000 Subject: [PATCH 15/84] FIX: Stop packet reader when receiving stop signal --- data_gateway/data_gateway.py | 8 ++++---- data_gateway/exceptions.py | 4 ---- tests/test_cli.py | 4 ++-- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/data_gateway/data_gateway.py b/data_gateway/data_gateway.py index 1829ac05..59def968 100644 --- a/data_gateway/data_gateway.py +++ b/data_gateway/data_gateway.py @@ -11,7 +11,7 @@ from data_gateway.configuration import Configuration from data_gateway.dummy_serial import DummySerial -from data_gateway.exceptions import DataMustBeSavedError, PacketReaderStopError +from data_gateway.exceptions import DataMustBeSavedError from data_gateway.packet_reader import PacketReader from data_gateway.routine import Routine @@ -137,7 +137,7 @@ def start(self, stop_when_no_more_data=False): if not error_queue.empty(): raise error_queue.get() - except (KeyboardInterrupt, PacketReaderStopError): + except KeyboardInterrupt: pass finally: @@ -168,7 +168,7 @@ def _send_commands_to_sensors(self): if line.startswith("sleep") and line.endswith("\n"): time.sleep(int(line.split(" ")[-1].strip())) elif line == "stop\n": - raise PacketReaderStopError() + self.packet_reader.stop = True # Send the command to the node self.serial_port.write(line.encode("utf_8")) @@ -183,7 +183,7 @@ def _load_configuration(self, configuration_path): with open(configuration_path) as f: configuration = Configuration.from_dict(json.load(f)) - logger.debug("Loaded configuration file from %r.", configuration_path) + logger.info("Loaded configuration file from %r.", configuration_path) return configuration configuration = Configuration() diff --git a/data_gateway/exceptions.py b/data_gateway/exceptions.py index 54731959..99bb2012 100644 --- a/data_gateway/exceptions.py +++ b/data_gateway/exceptions.py @@ -18,7 +18,3 @@ class WrongNumberOfSensorCoordinatesError(GatewayError, ValueError): class DataMustBeSavedError(GatewayError, ValueError): """Raise if options are given to the packet reader that mean no data will be saved locally or uploaded to the cloud.""" - - -class PacketReaderStopError(GatewayError): - pass diff --git a/tests/test_cli.py b/tests/test_cli.py index d0085ce1..02c0fcf1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -249,7 +249,7 @@ def test_save_locally(self): def test_start_with_config_file(self): """Ensure a configuration file can be provided via the CLI.""" with EnvironmentVariableRemover("GOOGLE_APPLICATION_CREDENTIALS"): - with mock.patch("logging.StreamHandler.emit") as mock_local_logger_emit: + with self.assertLogs() as logging_context: with tempfile.TemporaryDirectory() as temporary_directory: result = CliRunner().invoke( gateway_cli, @@ -267,7 +267,7 @@ def test_start_with_config_file(self): self.assertIsNone(result.exception) self.assertEqual(result.exit_code, 0) - self.assertIn("Loaded configuration file", mock_local_logger_emit.call_args_list[0][0][0].msg) + self.assertIn("Loaded configuration file", logging_context.output[0]) class TestCreateInstallation(BaseTestCase): From 00d99b27225af841289532f567aaa15d6aef6a38 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 7 Feb 2022 13:22:33 +0000 Subject: [PATCH 16/84] TST: Use simpler values for RUN_DEPLOYMENT_TESTS environment variable --- tests/test_cloud_functions/test_deployment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cloud_functions/test_deployment.py b/tests/test_cloud_functions/test_deployment.py index 95f3ff5c..0abc0226 100644 --- a/tests/test_cloud_functions/test_deployment.py +++ b/tests/test_cloud_functions/test_deployment.py @@ -13,7 +13,7 @@ @unittest.skipUnless( - condition=os.getenv("RUN_DEPLOYMENT_TESTS", "").lower() == "true", + condition=os.getenv("RUN_DEPLOYMENT_TESTS", "0") == "1", reason="'RUN_DEPLOYMENT_TESTS' environment variable is False or not present.", ) class TestDeployment(unittest.TestCase, DatasetMixin): From e84ff7276891f7cbf08e41a7bde3b4a6b93331b5 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 7 Feb 2022 13:24:26 +0000 Subject: [PATCH 17/84] TST: Avoid potentially missing environment variable when skipping test --- tests/test_cloud_functions/test_deployment.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_cloud_functions/test_deployment.py b/tests/test_cloud_functions/test_deployment.py index 0abc0226..7e2ed041 100644 --- a/tests/test_cloud_functions/test_deployment.py +++ b/tests/test_cloud_functions/test_deployment.py @@ -17,7 +17,9 @@ reason="'RUN_DEPLOYMENT_TESTS' environment variable is False or not present.", ) class TestDeployment(unittest.TestCase, DatasetMixin): - storage_client = GoogleCloudStorageClient(os.environ["TEST_PROJECT_NAME"]) + if os.getenv("RUN_DEPLOYMENT_TESTS", "0") == "1": + # The client must be instantiated here to avoid the storage emulator. + storage_client = GoogleCloudStorageClient(os.environ["TEST_PROJECT_NAME"]) def test_clean_and_upload_window(self): """Test that a window can be uploaded to a cloud bucket, its data processed by the test cloud function, and the From d8573f89baaf2def0a461eb5e419cbc745681bb4 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 7 Feb 2022 14:53:53 +0000 Subject: [PATCH 18/84] TST: Add missing bucket and project arguments in CLI tests --- tests/test_cli.py | 75 +++++++++++++++++++++++++---------------------- 1 file changed, 40 insertions(+), 35 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 02c0fcf1..44a2b94c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -10,7 +10,7 @@ from data_gateway.cli import CREATE_INSTALLATION_CLOUD_FUNCTION_URL, gateway_cli from data_gateway.dummy_serial import DummySerial from data_gateway.exceptions import DataMustBeSavedError -from tests import LENGTH, PACKET_KEY, RANDOM_BYTES +from tests import LENGTH, PACKET_KEY, RANDOM_BYTES, TEST_BUCKET_NAME, TEST_PROJECT_NAME from tests.base import BaseTestCase @@ -73,7 +73,8 @@ def test_start(self): [ "start", "--interactive", - "--save-locally", + f"--gcp-project-name={TEST_PROJECT_NAME}", + f"--gcp-bucket-name={TEST_BUCKET_NAME}", "--use-dummy-serial-port", f"--output-dir={temporary_directory}", ], @@ -88,22 +89,26 @@ def test_start_with_default_output_directory(self): initial_directory = os.getcwd() with tempfile.TemporaryDirectory() as temporary_directory: - os.chdir(temporary_directory) + try: + os.chdir(temporary_directory) - result = CliRunner().invoke( - gateway_cli, - [ - "start", - "--interactive", - "--save-locally", - "--use-dummy-serial-port", - ], - input="sleep 2\nstop\n", - ) + result = CliRunner().invoke( + gateway_cli, + [ + "start", + "--interactive", + f"--gcp-project-name={TEST_PROJECT_NAME}", + f"--gcp-bucket-name={TEST_BUCKET_NAME}", + "--use-dummy-serial-port", + ], + input="sleep 2\nstop\n", + ) - self.assertIsNone(result.exception) - self.assertEqual(result.exit_code, 0) - os.chdir(initial_directory) + self.assertIsNone(result.exception) + self.assertEqual(result.exit_code, 0) + + finally: + os.chdir(initial_directory) def test_commands_are_recorded_in_interactive_mode(self): """Ensure commands given in interactive mode are recorded.""" @@ -248,26 +253,26 @@ def test_save_locally(self): def test_start_with_config_file(self): """Ensure a configuration file can be provided via the CLI.""" - with EnvironmentVariableRemover("GOOGLE_APPLICATION_CREDENTIALS"): - with self.assertLogs() as logging_context: - with tempfile.TemporaryDirectory() as temporary_directory: - result = CliRunner().invoke( - gateway_cli, - [ - "start", - "--interactive", - "--save-locally", - "--no-upload-to-cloud", - "--use-dummy-serial-port", - f"--config-file={CONFIGURATION_PATH}", - f"--output-dir={temporary_directory}", - ], - input="stop\n", - ) + # with EnvironmentVariableRemover("GOOGLE_APPLICATION_CREDENTIALS"): + with self.assertLogs() as logging_context: + with tempfile.TemporaryDirectory() as temporary_directory: + result = CliRunner().invoke( + gateway_cli, + [ + "start", + "--interactive", + f"--gcp-project-name={TEST_PROJECT_NAME}", + f"--gcp-bucket-name={TEST_BUCKET_NAME}", + "--use-dummy-serial-port", + f"--config-file={CONFIGURATION_PATH}", + f"--output-dir={temporary_directory}", + ], + input="stop\n", + ) - self.assertIsNone(result.exception) - self.assertEqual(result.exit_code, 0) - self.assertIn("Loaded configuration file", logging_context.output[0]) + self.assertIsNone(result.exception) + self.assertEqual(result.exit_code, 0) + self.assertIn("Loaded configuration file", logging_context.output[0]) class TestCreateInstallation(BaseTestCase): From 3a9d14a4fea29a5dc8d4584ef94f6f5ff5234af0 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 7 Feb 2022 14:58:35 +0000 Subject: [PATCH 19/84] FIX: Stop packet reader when routine finishes --- data_gateway/data_gateway.py | 5 +++-- data_gateway/routine.py | 11 +++++++---- tests/test_routine.py | 34 ++++++++++++++++++++++++++++------ 3 files changed, 38 insertions(+), 12 deletions(-) diff --git a/data_gateway/data_gateway.py b/data_gateway/data_gateway.py index 59def968..40fdeeae 100644 --- a/data_gateway/data_gateway.py +++ b/data_gateway/data_gateway.py @@ -57,8 +57,6 @@ def __init__( use_dummy_serial_port=use_dummy_serial_port, ) - self.routine = self._load_routine(routine_path=routine_file) - # Start a new thread to parse the serial data while the main thread stays ready to take in commands from stdin. self.packet_reader = PacketReader( save_locally=save_locally, @@ -71,6 +69,8 @@ def __init__( save_csv_files=save_csv_files, ) + self.routine = self._load_routine(routine_path=routine_file) + def start(self, stop_when_no_more_data=False): """Begin reading and persisting data from the serial port for the sensors at the installation defined in the configuration. In interactive mode, commands can be sent to the nodes/sensors via the serial port by typing @@ -232,6 +232,7 @@ def _load_routine(self, routine_path): routine = Routine( **json.load(f), action=lambda command: self.serial_port.write((command + "\n").encode("utf_8")), + packet_reader=self.packet_reader, ) logger.info("Loaded routine file from %r.", routine_path) diff --git a/data_gateway/routine.py b/data_gateway/routine.py index 90606500..9c0d9014 100644 --- a/data_gateway/routine.py +++ b/data_gateway/routine.py @@ -19,9 +19,10 @@ class Routine: :return None: """ - def __init__(self, commands, action, period=None, stop_after=None): + def __init__(self, commands, action, packet_reader, period=None, stop_after=None): self.commands = commands self.action = self._wrap_action_with_logger(action) + self.packet_reader = packet_reader self.period = period self.stop_after = stop_after @@ -46,7 +47,7 @@ def run(self): scheduler = sched.scheduler(time.perf_counter) start_time = time.perf_counter() - while True: + while not self.packet_reader.stop: cycle_start_time = time.perf_counter() for command, delay in self.commands: @@ -55,14 +56,16 @@ def run(self): scheduler.run(blocking=True) if self.period is None: - break + self.packet_reader.stop = True + return elapsed_time = time.perf_counter() - cycle_start_time time.sleep(self.period - elapsed_time) if self.stop_after: if time.perf_counter() - start_time >= self.stop_after: - break + self.packet_reader.stop = True + return def _wrap_action_with_logger(self, action): """Wrap the given action so that when it's run on a command, the command is logged. diff --git a/tests/test_routine.py b/tests/test_routine.py index ece6eda5..4123329c 100644 --- a/tests/test_routine.py +++ b/tests/test_routine.py @@ -1,6 +1,7 @@ import time from unittest import TestCase +from data_gateway.packet_reader import PacketReader from data_gateway.routine import Routine @@ -12,7 +13,11 @@ def test_routine_with_no_period_runs_commands_once(self): def record_commands(command): recorded_commands.append((command, time.perf_counter())) - routine = Routine(commands=[("first-command", 0.1), ("second-command", 0.3)], action=record_commands) + routine = Routine( + commands=[("first-command", 0.1), ("second-command", 0.3)], + action=record_commands, + packet_reader=PacketReader(save_locally=False, upload_to_cloud=False), + ) start_time = time.perf_counter() routine.run() @@ -26,20 +31,36 @@ def record_commands(command): def test_error_raised_if_any_delay_is_greater_than_period(self): """Test that an error is raised if any of the command delays is greater than the period.""" with self.assertRaises(ValueError): - Routine(commands=[("first-command", 10), ("second-command", 0.3)], action=None, period=1) + Routine( + commands=[("first-command", 10), ("second-command", 0.3)], + action=None, + packet_reader=PacketReader(save_locally=False, upload_to_cloud=False), + period=1, + ) def test_error_raised_if_stop_after_time_is_less_than_period(self): """Test that an error is raised if the `stop_after` time is less than the period.""" with self.assertRaises(ValueError): - Routine(commands=[("first-command", 0.1), ("second-command", 0.3)], action=None, period=1, stop_after=0.5) + Routine( + commands=[("first-command", 0.1), ("second-command", 0.3)], + action=None, + packet_reader=PacketReader(save_locally=False, upload_to_cloud=False), + period=1, + stop_after=0.5, + ) def test_warning_raised_if_stop_after_time_provided_without_a_period(self): """Test that a warning is raised if the `stop_after` time is provided without a period.""" - with self.assertLogs() as logs_context: - Routine(commands=[("first-command", 10), ("second-command", 0.3)], action=None, stop_after=0.5) + with self.assertLogs() as logging_context: + Routine( + commands=[("first-command", 10), ("second-command", 0.3)], + action=None, + packet_reader=PacketReader(save_locally=False, upload_to_cloud=False), + stop_after=0.5, + ) self.assertEqual( - logs_context.output[0], + logging_context.output[1], "WARNING:data_gateway.routine:The `stop_after` parameter is ignored unless `period` is also given.", ) @@ -53,6 +74,7 @@ def record_commands(command): routine = Routine( commands=[("first-command", 0.1), ("second-command", 0.3)], action=record_commands, + packet_reader=PacketReader(save_locally=False, upload_to_cloud=False), period=0.4, stop_after=1, ) From 7f7b67f625630b7b4afa420f1d103fd0e4eefef4 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 7 Feb 2022 16:23:08 +0000 Subject: [PATCH 20/84] REF: Tidy up DataGateway --- data_gateway/cli.py | 26 +++---- data_gateway/data_gateway.py | 125 ++++++++++++++++++++-------------- data_gateway/packet_reader.py | 15 ++-- data_gateway/persistence.py | 2 +- tests/test_data_gateway.py | 6 +- 5 files changed, 97 insertions(+), 77 deletions(-) diff --git a/data_gateway/cli.py b/data_gateway/cli.py index ede2af0c..42d32952 100644 --- a/data_gateway/cli.py +++ b/data_gateway/cli.py @@ -162,19 +162,19 @@ def start( [startBaros, startMics, startIMU, getBattery, stop]. """ data_gateway = DataGateway( - serial_port, - config_file, - routine_file, - save_locally, - no_upload_to_cloud, - interactive, - output_dir, - window_size, - gcp_project_name, - gcp_bucket_name, - label, - save_csv_files, - use_dummy_serial_port, + serial_port=serial_port, + configuration_path=config_file, + routine_path=routine_file, + save_locally=save_locally, + upload_to_cloud=not no_upload_to_cloud, + interactive=interactive, + output_directory=output_dir, + window_size=window_size, + project_name=gcp_project_name, + bucket_name=gcp_bucket_name, + label=label, + save_csv_files=save_csv_files, + use_dummy_serial_port=use_dummy_serial_port, ) data_gateway.start() diff --git a/data_gateway/data_gateway.py b/data_gateway/data_gateway.py index 40fdeeae..eab6f38f 100644 --- a/data_gateway/data_gateway.py +++ b/data_gateway/data_gateway.py @@ -20,13 +20,34 @@ class DataGateway: + """A class for running the data gateway for wind turbine sensor data. The gateway is run with multiple threads + reading from the serial port which put the packets they read into a queue for a single parser thread to process and + persist. An additional thread is run for sending commands to the sensors either interactively or via a routine. If + a "stop" signal is sent as a command, all threads are stopped and any data in the current window is persisted. + + :param str|serial.Serial serial_port: the name of the serial port to use or a `serial.Serial` instance + :param str configuration_path: the path to a JSON configuration file for reading and parsing data + :param str routine_path: the path to a JSON routine file containing sensor commands to be run automatically + :param bool save_locally: if `True`, save data windows locally + :param bool upload_to_cloud: if `True`, upload data windows to Google cloud + :param bool interactive: if `True`, allow commands to be sent to the sensors automatically + :param str output_directory: the directory in which to save data in the cloud bucket or local file system + :param float window_size: the period in seconds at which data is persisted + :param str|None project_name: the name of Google Cloud project to upload to + :param str|None bucket_name: the name of Google Cloud bucket to upload to + :param str|None label: a label to be associated with the data collected in this run of the data gateway + :param bool save_csv_files: if `True`, also save windows locally as CSV files for debugging + :param bool use_dummy_serial_port: if `True` use a dummy serial port for testing + :return None: + """ + def __init__( self, serial_port, - config_file="config.json", - routine_file="routine.json", + configuration_path="config.json", + routine_path="routine.json", save_locally=False, - no_upload_to_cloud=False, + upload_to_cloud=True, interactive=False, output_directory="data_gateway", window_size=600, @@ -36,19 +57,15 @@ def __init__( save_csv_files=False, use_dummy_serial_port=False, ): - if not save_locally and no_upload_to_cloud: + if not save_locally and not upload_to_cloud: raise DataMustBeSavedError( "Data from the gateway must either be saved locally or uploaded to the cloud. Please adjust the CLI " "options provided." ) - self.serial_port = serial_port self.interactive = interactive - self.no_upload_to_cloud = no_upload_to_cloud - self.save_locally = save_locally - self.window_size = window_size - packet_reader_configuration = self._load_configuration(configuration_path=config_file) + packet_reader_configuration = self._load_configuration(configuration_path=configuration_path) packet_reader_configuration.session_data["label"] = label self.serial_port = self._get_serial_port( @@ -60,7 +77,7 @@ def __init__( # Start a new thread to parse the serial data while the main thread stays ready to take in commands from stdin. self.packet_reader = PacketReader( save_locally=save_locally, - upload_to_cloud=not no_upload_to_cloud, + upload_to_cloud=upload_to_cloud, output_directory=self._update_and_create_output_directory(output_directory_path=output_directory), window_size=window_size, project_name=project_name, @@ -69,7 +86,7 @@ def __init__( save_csv_files=save_csv_files, ) - self.routine = self._load_routine(routine_path=routine_file) + self.routine = self._load_routine(routine_path=routine_path) def start(self, stop_when_no_more_data=False): """Begin reading and persisting data from the serial port for the sensors at the installation defined in @@ -80,17 +97,19 @@ def start(self, stop_when_no_more_data=False): """ logger.info("Starting packet reader.") - if not self.no_upload_to_cloud: - logger.info("Files will be uploaded to cloud storage at intervals of %s seconds.", self.window_size) + if self.packet_reader.upload_to_cloud: + logger.info( + "Files will be uploaded to cloud storage at intervals of %s seconds.", self.packet_reader.window_size + ) - if self.save_locally: + if self.packet_reader.save_locally: logger.info( "Files will be saved locally to disk at %r at intervals of %s seconds.", os.path.join(self.packet_reader.output_directory, self.packet_reader.session_subdirectory), - self.window_size, + self.packet_reader.window_size, ) - self.packet_reader._persist_configuration() + self.packet_reader.persist_configuration() reader_thread_pool = ThreadPoolExecutor(thread_name_prefix="ReaderThread") packet_queue = queue.Queue() @@ -116,22 +135,17 @@ def start(self, stop_when_no_more_data=False): parser_thread.start() if self.interactive: - commands_thread = threading.Thread( - name="SensorCommandsThread", + interactive_commands_thread = threading.Thread( + name="InteractiveCommandsThread", target=self._send_commands_to_sensors, daemon=True, ) - commands_thread.start() + interactive_commands_thread.start() elif self.routine is not None: - commands_thread = threading.Thread( - name="SensorCommandsThread", - target=self.routine.run, - daemon=True, - ) - - commands_thread.start() + routine_thread = threading.Thread(name="RoutineCommandsThread", target=self.routine.run, daemon=True) + routine_thread.start() while not self.packet_reader.stop: if not error_queue.empty(): @@ -147,32 +161,6 @@ def start(self, stop_when_no_more_data=False): self.packet_reader.writer.force_persist() self.packet_reader.uploader.force_persist() - def _send_commands_to_sensors(self): - # Keep a record of the commands given. - commands_record_file = os.path.join( - self.packet_reader.output_directory, - self.packet_reader.session_subdirectory, - "commands.txt", - ) - - os.makedirs( - os.path.join(self.packet_reader.output_directory, self.packet_reader.session_subdirectory), - exist_ok=True, - ) - - while not self.packet_reader.stop: - for line in sys.stdin: - with open(commands_record_file, "a") as f: - f.write(line) - - if line.startswith("sleep") and line.endswith("\n"): - time.sleep(int(line.split(" ")[-1].strip())) - elif line == "stop\n": - self.packet_reader.stop = True - - # Send the command to the node - self.serial_port.write(line.encode("utf_8")) - def _load_configuration(self, configuration_path): """Load a configuration from the path if it exists, otherwise load the default configuration. @@ -193,7 +181,7 @@ def _load_configuration(self, configuration_path): def _get_serial_port(self, serial_port, configuration, use_dummy_serial_port): """Get the serial port or a dummy serial port if specified. - :param str serial_port: + :param str|serial.Serial serial_port: :param data_gateway.configuration.Configuration configuration: :param bool use_dummy_serial_port: :return serial.Serial: @@ -255,3 +243,34 @@ def _update_and_create_output_directory(self, output_directory_path): os.makedirs(output_directory_path, exist_ok=True) return output_directory_path + + def _send_commands_to_sensors(self): + """Send commands from `stdin` to the sensors until the "stop" command is received or the packet reader is + otherwise stopped. A record is kept of the commands sent to the sensors as a text file in the session + subdirectory. Available commands: [startBaros, startMics, startIMU, getBattery, stop]. + + :return None: + """ + commands_record_file = os.path.join( + self.packet_reader.output_directory, + self.packet_reader.session_subdirectory, + "commands.txt", + ) + + os.makedirs( + os.path.join(self.packet_reader.output_directory, self.packet_reader.session_subdirectory), + exist_ok=True, + ) + + while not self.packet_reader.stop: + for line in sys.stdin: + with open(commands_record_file, "a") as f: + f.write(line) + + if line.startswith("sleep") and line.endswith("\n"): + time.sleep(int(line.split(" ")[-1].strip())) + elif line == "stop\n": + self.packet_reader.stop = True + + # Send the command to the node + self.serial_port.write(line.encode("utf_8")) diff --git a/data_gateway/packet_reader.py b/data_gateway/packet_reader.py index 36b1f8db..377b632e 100644 --- a/data_gateway/packet_reader.py +++ b/data_gateway/packet_reader.py @@ -19,11 +19,11 @@ class PacketReader: :param bool save_locally: save data windows locally :param bool upload_to_cloud: upload data windows to Google cloud - :param str|None output_directory: - :param float window_size: length of time window in seconds + :param str|None output_directory: the directory in which to save data in the cloud bucket or local file system + :param float window_size: the period in seconds at which data is persisted. :param str|None project_name: name of Google Cloud project to upload to - :param str|None bucket_name: name of Google Cloud project to upload to - :param data_gateway.configuration.Configuration|None configuration: + :param str|None bucket_name: name of Google Cloud bucket to upload to + :param data_gateway.configuration.Configuration|None configuration: the configuration for reading and parsing data :param bool save_csv_files: save sensor data to .csv when in interactive mode :return None: """ @@ -42,6 +42,7 @@ def __init__( self.save_locally = save_locally self.upload_to_cloud = upload_to_cloud self.output_directory = output_directory + self.window_size = window_size self.config = configuration or Configuration() self.handles = self.config.default_handles self.sleep = False @@ -56,7 +57,7 @@ def __init__( sensor_names=self.config.sensor_names, project_name=project_name, bucket_name=bucket_name, - window_size=window_size, + window_size=self.window_size, session_subdirectory=self.session_subdirectory, output_directory=output_directory, metadata={"data_gateway__configuration": self.config.to_dict()}, @@ -67,7 +68,7 @@ def __init__( if save_locally: self.writer = BatchingFileWriter( sensor_names=self.config.sensor_names, - window_size=window_size, + window_size=self.window_size, session_subdirectory=self.session_subdirectory, output_directory=output_directory, save_csv_files=save_csv_files, @@ -171,7 +172,7 @@ def update_handles(self, payload): logger.error("Handle error: %s %s", start_handle, end_handle) - def _persist_configuration(self): + def persist_configuration(self): """Persist the configuration to disk and/or cloud storage. :return None: diff --git a/data_gateway/persistence.py b/data_gateway/persistence.py index da65ee4a..fad3ccc2 100644 --- a/data_gateway/persistence.py +++ b/data_gateway/persistence.py @@ -137,7 +137,7 @@ class BatchingFileWriter(TimeBatcher): """A file writer that groups the given into time windows, saving each window to disk. :param iter(str) sensor_names: names of sensors to make windows for - :param float window_size: :param float window_size: length of time window in seconds + :param float window_size: length of time window in seconds :param str session_subdirectory: directory within output directory to persist into :param str output_directory: directory to write windows to :param int storage_limit: storage limit in bytes (default is 1 GB) diff --git a/tests/test_data_gateway.py b/tests/test_data_gateway.py index 086bd37c..c1b636e8 100644 --- a/tests/test_data_gateway.py +++ b/tests/test_data_gateway.py @@ -134,7 +134,7 @@ def test_update_handles(self): data_gateway = DataGateway( serial_port, save_locally=True, - no_upload_to_cloud=True, + upload_to_cloud=False, output_directory=temporary_directory, window_size=self.WINDOW_SIZE, project_name=TEST_PROJECT_NAME, @@ -367,7 +367,7 @@ def test_data_gateway_with_connections_statistics_in_sleep_mode(self): data_gateway = DataGateway( serial_port, save_locally=True, - no_upload_to_cloud=True, + upload_to_cloud=False, output_directory=temporary_directory, window_size=self.WINDOW_SIZE, project_name=TEST_PROJECT_NAME, @@ -434,7 +434,7 @@ def test_data_gateway_with_info_packets(self): data_gateway = DataGateway( serial_port, save_locally=True, - no_upload_to_cloud=True, + upload_to_cloud=False, output_directory=temporary_directory, window_size=self.WINDOW_SIZE, project_name=TEST_PROJECT_NAME, From bff321a4835ccd352a978f818064948fe3a3bf46 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 7 Feb 2022 16:42:41 +0000 Subject: [PATCH 21/84] REF: Remove unnecessary except block --- data_gateway/data_gateway.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/data_gateway/data_gateway.py b/data_gateway/data_gateway.py index eab6f38f..351205ff 100644 --- a/data_gateway/data_gateway.py +++ b/data_gateway/data_gateway.py @@ -151,9 +151,6 @@ def start(self, stop_when_no_more_data=False): if not error_queue.empty(): raise error_queue.get() - except KeyboardInterrupt: - pass - finally: logger.info("Stopping gateway.") self.packet_reader.stop = True From 0dd8b8ff264bdfd0b6af65bcabde9c7aa10af79c Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 7 Feb 2022 16:45:18 +0000 Subject: [PATCH 22/84] REF: Rename PacketReader.parse_payload to parse_packets --- data_gateway/data_gateway.py | 2 +- data_gateway/packet_reader.py | 101 +++++++++++++++++----------------- 2 files changed, 52 insertions(+), 51 deletions(-) diff --git a/data_gateway/data_gateway.py b/data_gateway/data_gateway.py index 351205ff..044c7005 100644 --- a/data_gateway/data_gateway.py +++ b/data_gateway/data_gateway.py @@ -127,7 +127,7 @@ def start(self, stop_when_no_more_data=False): parser_thread = threading.Thread( name="ParserThread", - target=self.packet_reader.parse_payload, + target=self.packet_reader.parse_packets, kwargs={"packet_queue": packet_queue, "error_queue": error_queue}, daemon=True, ) diff --git a/data_gateway/packet_reader.py b/data_gateway/packet_reader.py index 377b632e..783ad301 100644 --- a/data_gateway/packet_reader.py +++ b/data_gateway/packet_reader.py @@ -77,11 +77,12 @@ def __init__( self.writer = NoOperationContextManager() def read_packets(self, serial_port, packet_queue, error_queue, stop_when_no_more_data=False): - """Read packets from a serial port and send them to a separate thread that will parse and upload them to Google - Cloud storage and/or write them to disk. + """Read packets from a serial port and send them to the parser thread for processing and persistence. :param serial.Serial serial_port: name of serial port to read from - :param bool stop_when_no_more_data: stop reading when no more data is received from the port (for testing) + :param queue.Queue packet_queue: a thread-safe queue to put packets on to for the parser thread to pick up + :param queue.Queue error_queue: a thread-safe queue to put any exceptions on to for the main thread to handle + :param bool stop_when_no_more_data: if `True`, stop reading when no more data is received from the port (for testing) :return None: """ try: @@ -140,6 +141,53 @@ def read_packets(self, serial_port, packet_queue, error_queue, stop_when_no_more except Exception as e: error_queue.put(e) + def parse_packets(self, packet_queue, error_queue): + """Get packets from a thread-safe packet queue, check if a full payload has been received (i.e. correct length) + with the correct packet type handle, then parse the payload. After parsing/processing, upload them to Google + Cloud storage and/or write them to disk. If any errors are raised, put them on the error queue for the main + thread to handle. + + :param queue.Queue packet_queue: a thread-safe queue of packets provided by a reader thread + :param queue.Queue error_queue: a thread-safe queue to put any exceptions on to for the main thread to handle + :return None: + """ + try: + with self.uploader: + with self.writer: + while not self.stop: + packet_type, payload, data, previous_timestamp = packet_queue.get().values() + + if packet_type not in self.handles: + logger.error("Received packet with unknown type: %s", packet_type) + + raise exceptions.UnknownPacketTypeError( + "Received packet with unknown type: {}".format(packet_type) + ) + + if len(payload) == 244: # If the full data payload is received, proceed parsing it + timestamp = int.from_bytes(payload[240:244], self.config.endian, signed=False) / (2 ** 16) + + data, sensor_names = self._parse_sensor_packet_data( + self.handles[packet_type], payload, data + ) + + for sensor_name in sensor_names: + self._check_for_packet_loss(sensor_name, timestamp, previous_timestamp) + self._timestamp_and_persist_data( + data, sensor_name, timestamp, self.config.period[sensor_name] + ) + + elif len(payload) >= 1 and self.handles[packet_type] in [ + "Mic 1", + "Cmd Decline", + "Sleep State", + "Info Message", + ]: + self._parse_info_packet(self.handles[packet_type], payload) + + except Exception as e: + error_queue.put(e) + def update_handles(self, payload): """Update the Bluetooth handles object. Handles are updated every time a new Bluetooth connection is established. @@ -195,53 +243,6 @@ def persist_configuration(self): ), ) - def parse_payload(self, packet_queue, error_queue): - """Get packets from a thread-safe packet queue, check if a full payload has been received (i.e. correct length) - with the correct packet type handle, then parse the payload. After parsing/processing, upload them to Google - Cloud storage and/or write them to disk. If any errors are raised, put them on the error queue for the reader - thread to handle. - - :param queue.Queue packet_queue: a thread-safe queue of packets provided by the reader thread - :param queue.Queue error_queue: a thread-safe queue to put any exceptions on to for the reader thread to handle - :return None: - """ - try: - with self.uploader: - with self.writer: - while not self.stop: - packet_type, payload, data, previous_timestamp = packet_queue.get().values() - - if packet_type not in self.handles: - logger.error("Received packet with unknown type: %s", packet_type) - - raise exceptions.UnknownPacketTypeError( - "Received packet with unknown type: {}".format(packet_type) - ) - - if len(payload) == 244: # If the full data payload is received, proceed parsing it - timestamp = int.from_bytes(payload[240:244], self.config.endian, signed=False) / (2 ** 16) - - data, sensor_names = self._parse_sensor_packet_data( - self.handles[packet_type], payload, data - ) - - for sensor_name in sensor_names: - self._check_for_packet_loss(sensor_name, timestamp, previous_timestamp) - self._timestamp_and_persist_data( - data, sensor_name, timestamp, self.config.period[sensor_name] - ) - - elif len(payload) >= 1 and self.handles[packet_type] in [ - "Mic 1", - "Cmd Decline", - "Sleep State", - "Info Message", - ]: - self._parse_info_packet(self.handles[packet_type], payload) - - except Exception as e: - error_queue.put(e) - def _parse_sensor_packet_data(self, packet_type, payload, data): """Parse sensor data type payloads. From 8aec58e19c49bfa523a4bb85ecf7d6435f36bedc Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 7 Feb 2022 16:52:18 +0000 Subject: [PATCH 23/84] TST: Remove commented-out code --- tests/test_cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 44a2b94c..4bfc2ca5 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -253,7 +253,6 @@ def test_save_locally(self): def test_start_with_config_file(self): """Ensure a configuration file can be provided via the CLI.""" - # with EnvironmentVariableRemover("GOOGLE_APPLICATION_CREDENTIALS"): with self.assertLogs() as logging_context: with tempfile.TemporaryDirectory() as temporary_directory: result = CliRunner().invoke( From 77b4fa257229c7a8c1aef432a33d43e188a026d7 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 7 Feb 2022 17:24:14 +0000 Subject: [PATCH 24/84] ENH: Lower some log messages to debug; optimise packet reader stopping --- data_gateway/data_gateway.py | 74 +++++++++++++------------ tests/test_cli.py | 9 ++- tests/test_cloud_functions/test_main.py | 5 +- tests/test_data_gateway.py | 4 +- 4 files changed, 50 insertions(+), 42 deletions(-) diff --git a/data_gateway/data_gateway.py b/data_gateway/data_gateway.py index 044c7005..3105d2c6 100644 --- a/data_gateway/data_gateway.py +++ b/data_gateway/data_gateway.py @@ -20,21 +20,22 @@ class DataGateway: - """A class for running the data gateway for wind turbine sensor data. The gateway is run with multiple threads - reading from the serial port which put the packets they read into a queue for a single parser thread to process and - persist. An additional thread is run for sending commands to the sensors either interactively or via a routine. If - a "stop" signal is sent as a command, all threads are stopped and any data in the current window is persisted. + """A class for running the data gateway for collecting wind turbine sensor data. The gateway is run with multiple + threads reading from the serial port which put the packets they read into a queue for a single parser thread to + process and persist. An additional thread is run for sending commands to the sensors either interactively or via a + routine. If a "stop" signal is sent as a command, all threads are stopped and any data in the current window is + persisted. :param str|serial.Serial serial_port: the name of the serial port to use or a `serial.Serial` instance :param str configuration_path: the path to a JSON configuration file for reading and parsing data :param str routine_path: the path to a JSON routine file containing sensor commands to be run automatically :param bool save_locally: if `True`, save data windows locally - :param bool upload_to_cloud: if `True`, upload data windows to Google cloud - :param bool interactive: if `True`, allow commands to be sent to the sensors automatically + :param bool upload_to_cloud: if `True`, upload data windows to Google Cloud Storage + :param bool interactive: if `True`, allow commands entered into `stdin` to be sent to the sensors in real time :param str output_directory: the directory in which to save data in the cloud bucket or local file system :param float window_size: the period in seconds at which data is persisted - :param str|None project_name: the name of Google Cloud project to upload to - :param str|None bucket_name: the name of Google Cloud bucket to upload to + :param str|None project_name: the name of the Google Cloud project to upload to + :param str|None bucket_name: the name of the Google Cloud bucket to upload to :param str|None label: a label to be associated with the data collected in this run of the data gateway :param bool save_csv_files: if `True`, also save windows locally as CSV files for debugging :param bool use_dummy_serial_port: if `True` use a dummy serial port for testing @@ -74,7 +75,6 @@ def __init__( use_dummy_serial_port=use_dummy_serial_port, ) - # Start a new thread to parse the serial data while the main thread stays ready to take in commands from stdin. self.packet_reader = PacketReader( save_locally=save_locally, upload_to_cloud=upload_to_cloud, @@ -91,21 +91,21 @@ def __init__( def start(self, stop_when_no_more_data=False): """Begin reading and persisting data from the serial port for the sensors at the installation defined in the configuration. In interactive mode, commands can be sent to the nodes/sensors via the serial port by typing - them into stdin and pressing enter. These commands are: [startBaros, startMics, startIMU, getBattery, stop]. + them into `stdin` and pressing enter. These commands are: [startBaros, startMics, startIMU, getBattery, stop]. :return None: """ logger.info("Starting packet reader.") if self.packet_reader.upload_to_cloud: - logger.info( - "Files will be uploaded to cloud storage at intervals of %s seconds.", self.packet_reader.window_size + logger.debug( + "Files will be uploaded to cloud storage at intervals of %s seconds.", + self.packet_reader.window_size, ) if self.packet_reader.save_locally: - logger.info( - "Files will be saved locally to disk at %r at intervals of %s seconds.", - os.path.join(self.packet_reader.output_directory, self.packet_reader.session_subdirectory), + logger.debug( + "Files will be saved locally to disk at intervals of %s seconds.", self.packet_reader.window_size, ) @@ -137,7 +137,7 @@ def start(self, stop_when_no_more_data=False): if self.interactive: interactive_commands_thread = threading.Thread( name="InteractiveCommandsThread", - target=self._send_commands_to_sensors, + target=self._send_commands_from_stdin_to_sensors, daemon=True, ) @@ -147,6 +147,7 @@ def start(self, stop_when_no_more_data=False): routine_thread = threading.Thread(name="RoutineCommandsThread", target=self.routine.run, daemon=True) routine_thread.start() + # Raise any errors from the reader threads and parser thread. while not self.packet_reader.stop: if not error_queue.empty(): raise error_queue.get() @@ -161,27 +162,28 @@ def start(self, stop_when_no_more_data=False): def _load_configuration(self, configuration_path): """Load a configuration from the path if it exists, otherwise load the default configuration. - :param str configuration_path: + :param str configuration_path: path to the configuration JSON file :return data_gateway.configuration.Configuration: """ if os.path.exists(configuration_path): with open(configuration_path) as f: configuration = Configuration.from_dict(json.load(f)) - logger.info("Loaded configuration file from %r.", configuration_path) + logger.debug("Loaded configuration file from %r.", configuration_path) return configuration configuration = Configuration() - logger.info("No configuration file provided - using default configuration.") + logger.debug("No configuration file provided - using default configuration.") return configuration def _get_serial_port(self, serial_port, configuration, use_dummy_serial_port): - """Get the serial port or a dummy serial port if specified. + """Get the serial port or a dummy serial port if specified. If a serial port instance is provided, return that + as the serial port to use. - :param str|serial.Serial serial_port: - :param data_gateway.configuration.Configuration configuration: - :param bool use_dummy_serial_port: - :return serial.Serial: + :param str|serial.Serial serial_port: the name of a serial port or a `serial.Serial` instance + :param data_gateway.configuration.Configuration configuration: the packet reader configuration + :param bool use_dummy_serial_port: if `True`, use a dummy serial port instead + :return serial.Serial|data_gateway.dummy_serial.DummySerial: """ if isinstance(serial_port, str): if not use_dummy_serial_port: @@ -196,17 +198,17 @@ def _get_serial_port(self, serial_port, configuration, use_dummy_serial_port): tx_size=configuration.serial_buffer_tx_size, ) else: - logger.warning("Serial port buffer size can only be set on Windows.") + logger.debug("Serial port buffer size can only be set on Windows.") return serial_port def _load_routine(self, routine_path): - """Load a sensor commands routine from the path if exists, otherwise return no routine. If in interactive mode, - the routine file is ignored. Note that "\n" has to be added to the end of each command sent to the serial port - for it to be executed - this is done automatically in this method. + """Load a sensor commands routine from the path if it exists, otherwise return no routine. If in interactive + mode, the routine file is ignored. Note that "\n" has to be added to the end of each command sent to the serial + port for it to be executed - this is done automatically in this method. - :param str routine_path: - :return data_gateway.routine.Routine|None: + :param str routine_path: the path to the JSON routine file + :return data_gateway.routine.Routine|None: a sensor routine instance """ if os.path.exists(routine_path): if self.interactive: @@ -220,10 +222,10 @@ def _load_routine(self, routine_path): packet_reader=self.packet_reader, ) - logger.info("Loaded routine file from %r.", routine_path) + logger.debug("Loaded routine file from %r.", routine_path) return routine - logger.info( + logger.debug( "No routine file found at %r - no commands will be sent to the sensors unless given in interactive mode.", routine_path, ) @@ -232,8 +234,8 @@ def _update_and_create_output_directory(self, output_directory_path): """Set the output directory to a path relative to the current directory if the path does not start with "/" and create it if it does not already exist. - :param str output_directory_path: - :return str: + :param str output_directory_path: the path to the directory to write output data to + :return str: the updated output directory path """ if not output_directory_path.startswith("/"): output_directory_path = os.path.join(".", output_directory_path) @@ -241,7 +243,7 @@ def _update_and_create_output_directory(self, output_directory_path): os.makedirs(output_directory_path, exist_ok=True) return output_directory_path - def _send_commands_to_sensors(self): + def _send_commands_from_stdin_to_sensors(self): """Send commands from `stdin` to the sensors until the "stop" command is received or the packet reader is otherwise stopped. A record is kept of the commands sent to the sensors as a text file in the session subdirectory. Available commands: [startBaros, startMics, startIMU, getBattery, stop]. @@ -268,6 +270,8 @@ def _send_commands_to_sensors(self): time.sleep(int(line.split(" ")[-1].strip())) elif line == "stop\n": self.packet_reader.stop = True + self.serial_port.write(line.encode("utf_8")) + break # Send the command to the node self.serial_port.write(line.encode("utf_8")) diff --git a/tests/test_cli.py b/tests/test_cli.py index 4bfc2ca5..bb1beb0a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -8,6 +8,7 @@ from click.testing import CliRunner from data_gateway.cli import CREATE_INSTALLATION_CLOUD_FUNCTION_URL, gateway_cli +from data_gateway.configuration import Configuration from data_gateway.dummy_serial import DummySerial from data_gateway.exceptions import DataMustBeSavedError from tests import LENGTH, PACKET_KEY, RANDOM_BYTES, TEST_BUCKET_NAME, TEST_PROJECT_NAME @@ -253,8 +254,10 @@ def test_save_locally(self): def test_start_with_config_file(self): """Ensure a configuration file can be provided via the CLI.""" - with self.assertLogs() as logging_context: - with tempfile.TemporaryDirectory() as temporary_directory: + with tempfile.TemporaryDirectory() as temporary_directory: + with mock.patch( + "data_gateway.data_gateway.Configuration.from_dict", return_value=Configuration() + ) as mock_configuration_from_dict: result = CliRunner().invoke( gateway_cli, [ @@ -271,7 +274,7 @@ def test_start_with_config_file(self): self.assertIsNone(result.exception) self.assertEqual(result.exit_code, 0) - self.assertIn("Loaded configuration file", logging_context.output[0]) + mock_configuration_from_dict.assert_called() class TestCreateInstallation(BaseTestCase): diff --git a/tests/test_cloud_functions/test_main.py b/tests/test_cloud_functions/test_main.py index 47041b62..76dabe04 100644 --- a/tests/test_cloud_functions/test_main.py +++ b/tests/test_cloud_functions/test_main.py @@ -1,3 +1,4 @@ +import copy import json import os import sys @@ -22,7 +23,7 @@ from cloud_functions.window_handler import ConfigurationAlreadyExists # noqa -class TestCleanAndUploadWindow(CredentialsEnvironmentVariableAsFile, BaseTestCase): +class TestCleanAndUploadWindow(BaseTestCase, CredentialsEnvironmentVariableAsFile): SOURCE_PROJECT_NAME = "source-project" SOURCE_BUCKET_NAME = TEST_BUCKET_NAME WINDOW = BaseTestCase().random_window(sensors=["Constat"], window_duration=1) @@ -59,7 +60,7 @@ def test_clean_and_upload_window(self): main.clean_and_upload_window(event=self.MOCK_EVENT, context=self._make_mock_context()) # Check configuration without user data was added. - expected_configuration = self.VALID_CONFIGURATION.copy() + expected_configuration = copy.deepcopy(self.VALID_CONFIGURATION) del expected_configuration["session_data"] self.assertIn("add_configuration", mock_dataset.mock_calls[1][0]) self.assertEqual(mock_dataset.mock_calls[1].args[0], expected_configuration) diff --git a/tests/test_data_gateway.py b/tests/test_data_gateway.py index c1b636e8..f1d92b90 100644 --- a/tests/test_data_gateway.py +++ b/tests/test_data_gateway.py @@ -115,7 +115,7 @@ def test_update_handles_fails_if_start_and_end_handles_are_incorrect(self): with self.assertLogs() as logging_context: data_gateway.start(stop_when_no_more_data=True) - self.assertIn("Handle error", logging_context.output[3]) + self.assertIn("Handle error", logging_context.output[1]) def test_update_handles(self): """Test that the handles can be updated.""" @@ -143,7 +143,7 @@ def test_update_handles(self): with self.assertLogs() as logging_context: data_gateway.start(stop_when_no_more_data=True) - self.assertIn("Successfully updated handles", logging_context.output[2]) + self.assertIn("Successfully updated handles", logging_context.output[1]) def test_data_gateway_with_baros_p_sensor(self): """Test that the packet reader works with the Baro_P sensor.""" From 680ee55888f6939c9ab4b9b088636acfa7295754 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 7 Feb 2022 18:04:44 +0000 Subject: [PATCH 25/84] TST: Fix CredentialsEnvironmentVariableAsFile usage --- tests/test_cloud_functions/base.py | 21 ++-- tests/test_cloud_functions/test_big_query.py | 115 ++++++++++--------- tests/test_cloud_functions/test_main.py | 39 ++++--- 3 files changed, 91 insertions(+), 84 deletions(-) diff --git a/tests/test_cloud_functions/base.py b/tests/test_cloud_functions/base.py index 3673509e..ef5ecf12 100644 --- a/tests/test_cloud_functions/base.py +++ b/tests/test_cloud_functions/base.py @@ -10,31 +10,30 @@ class CredentialsEnvironmentVariableAsFile: tests that require credentials to be present as a file are run. """ - credentials_path = "temporary_file.json" - current_google_application_credentials_variable_value = None + def __init__(self): + self.credentials_path = "temporary_file.json" + self.current_google_application_credentials_variable_value = None - @classmethod - def setUpClass(cls): + def __enter__(self): """Temporarily write the credentials to a file so that the tests can run on GitHub where the credentials are only provided as JSON in an environment variable. Set the credentials environment variable to point to this file instead of the credentials JSON. :return None: """ - cls.current_google_application_credentials_variable_value = os.environ["GOOGLE_APPLICATION_CREDENTIALS"] + self.current_google_application_credentials_variable_value = os.environ["GOOGLE_APPLICATION_CREDENTIALS"] credentials = GCPCredentialsManager().get_credentials(as_dict=True) - with open(cls.credentials_path, "w") as f: + with open(self.credentials_path, "w") as f: json.dump(credentials, f) - os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = cls.credentials_path + os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = self.credentials_path - @classmethod - def tearDownClass(cls): + def __exit__(self, exc_type, exc_val, exc_tb): """Remove the temporary credentials file and restore the credentials environment variable to its original value. :return None: """ - os.remove(cls.credentials_path) - os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = cls.current_google_application_credentials_variable_value + os.remove(self.credentials_path) + os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = self.current_google_application_credentials_variable_value diff --git a/tests/test_cloud_functions/test_big_query.py b/tests/test_cloud_functions/test_big_query.py index 5ffb92fe..9d1b7038 100644 --- a/tests/test_cloud_functions/test_big_query.py +++ b/tests/test_cloud_functions/test_big_query.py @@ -18,7 +18,7 @@ ) -class TestBigQueryDataset(CredentialsEnvironmentVariableAsFile, BaseTestCase): +class TestBigQueryDataset(BaseTestCase): def test_insert_sensor_data(self): """Test that sensor data can be sent to BigQuery for insertion.""" data = { @@ -32,15 +32,16 @@ def test_insert_sensor_data(self): "Constat": [[1636559720.639327, 36, 37, 38, 39]], } - with patch("big_query.bigquery.Client.get_table"): - with patch("big_query.bigquery.Client.insert_rows", return_value=None) as mock_insert_rows: + with CredentialsEnvironmentVariableAsFile(): + with patch("big_query.bigquery.Client.get_table"): + with patch("big_query.bigquery.Client.insert_rows", return_value=None) as mock_insert_rows: - BigQueryDataset(project_name="my-project", dataset_name="my-dataset").add_sensor_data( - data=data, - configuration_id="dbfed555-1b70-4191-96cb-c22071464b90", - installation_reference="turbine-1", - label="my-test", - ) + BigQueryDataset(project_name="my-project", dataset_name="my-dataset").add_sensor_data( + data=data, + configuration_id="dbfed555-1b70-4191-96cb-c22071464b90", + installation_reference="turbine-1", + label="my-test", + ) new_rows = mock_insert_rows.call_args.kwargs["rows"] self.assertEqual(len(new_rows), 8) @@ -116,12 +117,13 @@ def test_insert_sensor_data(self): def test_add_new_sensor_type(self): """Test that new sensor types can be added and that their references are their names slugified.""" - with patch("big_query.bigquery.Client.get_table"): - with patch("big_query.bigquery.Client.insert_rows", return_value=None) as mock_insert_rows: + with CredentialsEnvironmentVariableAsFile(): + with patch("big_query.bigquery.Client.get_table"): + with patch("big_query.bigquery.Client.insert_rows", return_value=None) as mock_insert_rows: - BigQueryDataset(project_name="my-project", dataset_name="my-dataset").add_sensor_type( - name="My sensor_Name" - ) + BigQueryDataset(project_name="my-project", dataset_name="my-dataset").add_sensor_type( + name="My sensor_Name" + ) self.assertEqual( mock_insert_rows.call_args.kwargs["rows"][0], @@ -136,17 +138,18 @@ def test_add_new_sensor_type(self): def test_add_installation(self): """Test that installations can be added.""" - with patch("big_query.bigquery.Client.get_table"): - with patch("big_query.bigquery.Client.insert_rows", return_value=None) as mock_insert_rows: - with patch("big_query.bigquery.Client.query", return_value=Mock(result=lambda: [])): - - BigQueryDataset(project_name="my-project", dataset_name="my-dataset").add_installation( - reference="my-installation", - turbine_id="my-turbine", - blade_id="my-blade", - hardware_version="1.0.0", - sensor_coordinates={"my-sensor": [[0, 1, 2], [3, 8, 7]]}, - ) + with CredentialsEnvironmentVariableAsFile(): + with patch("big_query.bigquery.Client.get_table"): + with patch("big_query.bigquery.Client.insert_rows", return_value=None) as mock_insert_rows: + with patch("big_query.bigquery.Client.query", return_value=Mock(result=lambda: [])): + + BigQueryDataset(project_name="my-project", dataset_name="my-dataset").add_installation( + reference="my-installation", + turbine_id="my-turbine", + blade_id="my-blade", + hardware_version="1.0.0", + sensor_coordinates={"my-sensor": [[0, 1, 2], [3, 8, 7]]}, + ) self.assertEqual( mock_insert_rows.call_args.kwargs["rows"][0], @@ -162,27 +165,29 @@ def test_add_installation(self): def test_add_installation_raises_error_if_installation_already_exists(self): """Test that an error is raised if attempting to add an installation that already exists.""" - dataset = BigQueryDataset(project_name="my-project", dataset_name="my-dataset") - - with patch("big_query.bigquery.Client.query", return_value=Mock(result=lambda: [1])): - with self.assertRaises(InstallationWithSameNameAlreadyExists): - dataset.add_installation( - reference="my-installation", - turbine_id="my-turbine", - blade_id="my-blade", - hardware_version="1.0.0", - sensor_coordinates={"my-sensor": [[0, 1, 2], [3, 8, 7]]}, - ) + with CredentialsEnvironmentVariableAsFile(): + dataset = BigQueryDataset(project_name="my-project", dataset_name="my-dataset") + + with patch("big_query.bigquery.Client.query", return_value=Mock(result=lambda: [1])): + with self.assertRaises(InstallationWithSameNameAlreadyExists): + dataset.add_installation( + reference="my-installation", + turbine_id="my-turbine", + blade_id="my-blade", + hardware_version="1.0.0", + sensor_coordinates={"my-sensor": [[0, 1, 2], [3, 8, 7]]}, + ) def test_add_configuration(self): """Test that a configuration can be added.""" - with patch("big_query.bigquery.Client.get_table"): - with patch("big_query.bigquery.Client.insert_rows", return_value=None) as mock_insert_rows: - with patch("big_query.bigquery.Client.query", return_value=Mock(result=lambda: [])): + with CredentialsEnvironmentVariableAsFile(): + with patch("big_query.bigquery.Client.get_table"): + with patch("big_query.bigquery.Client.insert_rows", return_value=None) as mock_insert_rows: + with patch("big_query.bigquery.Client.query", return_value=Mock(result=lambda: [])): - BigQueryDataset(project_name="my-project", dataset_name="my-dataset").add_configuration( - configuration={"blah": "blah", "installation_data": {"stuff": "data"}} - ) + BigQueryDataset(project_name="my-project", dataset_name="my-dataset").add_configuration( + configuration={"blah": "blah", "installation_data": {"stuff": "data"}} + ) del mock_insert_rows.call_args.kwargs["rows"][0]["id"] @@ -201,15 +206,17 @@ def test_add_configuration_raises_error_if_installation_already_exists(self): existing configuration is returned. """ existing_configuration_id = "0846401a-89fb-424e-89e6-039063e0ee6d" - dataset = BigQueryDataset(project_name="my-project", dataset_name="my-dataset") - - with patch( - "big_query.bigquery.Client.query", - return_value=Mock(result=lambda: [Mock(id=existing_configuration_id)]), - ): - with self.assertRaises(ConfigurationAlreadyExists): - configuration_id = dataset.add_configuration( - configuration={"blah": "blah", "installation_data": {"stuff": "data"}} - ) - - self.assertEqual(configuration_id, existing_configuration_id) + + with CredentialsEnvironmentVariableAsFile(): + dataset = BigQueryDataset(project_name="my-project", dataset_name="my-dataset") + + with patch( + "big_query.bigquery.Client.query", + return_value=Mock(result=lambda: [Mock(id=existing_configuration_id)]), + ): + with self.assertRaises(ConfigurationAlreadyExists): + configuration_id = dataset.add_configuration( + configuration={"blah": "blah", "installation_data": {"stuff": "data"}} + ) + + self.assertEqual(configuration_id, existing_configuration_id) diff --git a/tests/test_cloud_functions/test_main.py b/tests/test_cloud_functions/test_main.py index 76dabe04..7ab54a0b 100644 --- a/tests/test_cloud_functions/test_main.py +++ b/tests/test_cloud_functions/test_main.py @@ -23,7 +23,7 @@ from cloud_functions.window_handler import ConfigurationAlreadyExists # noqa -class TestCleanAndUploadWindow(BaseTestCase, CredentialsEnvironmentVariableAsFile): +class TestCleanAndUploadWindow(BaseTestCase): SOURCE_PROJECT_NAME = "source-project" SOURCE_BUCKET_NAME = TEST_BUCKET_NAME WINDOW = BaseTestCase().random_window(sensors=["Constat"], window_duration=1) @@ -40,24 +40,25 @@ def test_clean_and_upload_window(self): """Test that a window file is cleaned and uploaded to its destination bucket following the relevant Google Cloud storage trigger. """ - GoogleCloudStorageClient(self.SOURCE_PROJECT_NAME).upload_from_string( - string=json.dumps(self.WINDOW, cls=OctueJSONEncoder), - bucket_name=self.SOURCE_BUCKET_NAME, - path_in_bucket="window-0.json", - metadata={"data_gateway__configuration": self.VALID_CONFIGURATION}, - ) - - with patch.dict( - os.environ, - { - "SOURCE_PROJECT_NAME": self.SOURCE_PROJECT_NAME, - "DESTINATION_PROJECT_NAME": "destination-project", - "DESTINATION_BUCKET_NAME": "destination-bucket", - "BIG_QUERY_DATASET_NAME": "blah", - }, - ): - with patch("window_handler.BigQueryDataset") as mock_dataset: - main.clean_and_upload_window(event=self.MOCK_EVENT, context=self._make_mock_context()) + with CredentialsEnvironmentVariableAsFile(): + GoogleCloudStorageClient(self.SOURCE_PROJECT_NAME).upload_from_string( + string=json.dumps(self.WINDOW, cls=OctueJSONEncoder), + bucket_name=self.SOURCE_BUCKET_NAME, + path_in_bucket="window-0.json", + metadata={"data_gateway__configuration": self.VALID_CONFIGURATION}, + ) + + with patch.dict( + os.environ, + { + "SOURCE_PROJECT_NAME": self.SOURCE_PROJECT_NAME, + "DESTINATION_PROJECT_NAME": "destination-project", + "DESTINATION_BUCKET_NAME": "destination-bucket", + "BIG_QUERY_DATASET_NAME": "blah", + }, + ): + with patch("window_handler.BigQueryDataset") as mock_dataset: + main.clean_and_upload_window(event=self.MOCK_EVENT, context=self._make_mock_context()) # Check configuration without user data was added. expected_configuration = copy.deepcopy(self.VALID_CONFIGURATION) From b1897fa0cda707e4cffe37878ed899b9075d1165 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 7 Feb 2022 18:18:00 +0000 Subject: [PATCH 26/84] TST: Mock BiqQuery client in tests --- tests/test_cloud_functions/test_main.py | 38 ++++++++++++------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/test_cloud_functions/test_main.py b/tests/test_cloud_functions/test_main.py index 7ab54a0b..92b639d8 100644 --- a/tests/test_cloud_functions/test_main.py +++ b/tests/test_cloud_functions/test_main.py @@ -11,7 +11,6 @@ from tests import TEST_BUCKET_NAME # noqa from tests.base import BaseTestCase # noqa from tests.test_cloud_functions import REPOSITORY_ROOT -from tests.test_cloud_functions.base import CredentialsEnvironmentVariableAsFile # Manually add the cloud_functions package to the path (its imports have to be done in a certain way for Google Cloud @@ -40,23 +39,23 @@ def test_clean_and_upload_window(self): """Test that a window file is cleaned and uploaded to its destination bucket following the relevant Google Cloud storage trigger. """ - with CredentialsEnvironmentVariableAsFile(): - GoogleCloudStorageClient(self.SOURCE_PROJECT_NAME).upload_from_string( - string=json.dumps(self.WINDOW, cls=OctueJSONEncoder), - bucket_name=self.SOURCE_BUCKET_NAME, - path_in_bucket="window-0.json", - metadata={"data_gateway__configuration": self.VALID_CONFIGURATION}, - ) - - with patch.dict( - os.environ, - { - "SOURCE_PROJECT_NAME": self.SOURCE_PROJECT_NAME, - "DESTINATION_PROJECT_NAME": "destination-project", - "DESTINATION_BUCKET_NAME": "destination-bucket", - "BIG_QUERY_DATASET_NAME": "blah", - }, - ): + GoogleCloudStorageClient(self.SOURCE_PROJECT_NAME).upload_from_string( + string=json.dumps(self.WINDOW, cls=OctueJSONEncoder), + bucket_name=self.SOURCE_BUCKET_NAME, + path_in_bucket="window-0.json", + metadata={"data_gateway__configuration": self.VALID_CONFIGURATION}, + ) + + with patch.dict( + os.environ, + { + "SOURCE_PROJECT_NAME": self.SOURCE_PROJECT_NAME, + "DESTINATION_PROJECT_NAME": "destination-project", + "DESTINATION_BUCKET_NAME": "destination-bucket", + "BIG_QUERY_DATASET_NAME": "blah", + }, + ): + with patch("big_query.bigquery.Client"): with patch("window_handler.BigQueryDataset") as mock_dataset: main.clean_and_upload_window(event=self.MOCK_EVENT, context=self._make_mock_context()) @@ -95,7 +94,8 @@ def test_clean_and_upload_window_for_existing_configuration(self): side_effect=ConfigurationAlreadyExists("blah", "8b9337d8-40b1-4872-b2f5-b1bfe82b241e"), ): with patch("window_handler.BigQueryDataset.add_sensor_data", return_value=None): - main.clean_and_upload_window(event=self.MOCK_EVENT, context=self._make_mock_context()) + with patch("big_query.bigquery.Client"): + main.clean_and_upload_window(event=self.MOCK_EVENT, context=self._make_mock_context()) @staticmethod def _make_mock_context(): From 2934726c99baf058be58f49528106bc921515e1f Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 8 Feb 2022 11:40:48 +0000 Subject: [PATCH 27/84] ENH: Continue running if unknown packet is read from serial port --- data_gateway/packet_reader.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/data_gateway/packet_reader.py b/data_gateway/packet_reader.py index 783ad301..957a1a0a 100644 --- a/data_gateway/packet_reader.py +++ b/data_gateway/packet_reader.py @@ -159,10 +159,7 @@ def parse_packets(self, packet_queue, error_queue): if packet_type not in self.handles: logger.error("Received packet with unknown type: %s", packet_type) - - raise exceptions.UnknownPacketTypeError( - "Received packet with unknown type: {}".format(packet_type) - ) + continue if len(payload) == 244: # If the full data payload is received, proceed parsing it timestamp = int.from_bytes(payload[240:244], self.config.endian, signed=False) / (2 ** 16) From 46fabce97de699c329b5c81fd4fe44d1c814ce4d Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 8 Feb 2022 11:45:22 +0000 Subject: [PATCH 28/84] TST: Only save locally in CLI tests --- tests/test_cli.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index bb1beb0a..2c2087ee 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -11,7 +11,7 @@ from data_gateway.configuration import Configuration from data_gateway.dummy_serial import DummySerial from data_gateway.exceptions import DataMustBeSavedError -from tests import LENGTH, PACKET_KEY, RANDOM_BYTES, TEST_BUCKET_NAME, TEST_PROJECT_NAME +from tests import LENGTH, PACKET_KEY, RANDOM_BYTES from tests.base import BaseTestCase @@ -74,8 +74,8 @@ def test_start(self): [ "start", "--interactive", - f"--gcp-project-name={TEST_PROJECT_NAME}", - f"--gcp-bucket-name={TEST_BUCKET_NAME}", + "--save-locally", + "--no-upload-to-cloud", "--use-dummy-serial-port", f"--output-dir={temporary_directory}", ], @@ -98,8 +98,8 @@ def test_start_with_default_output_directory(self): [ "start", "--interactive", - f"--gcp-project-name={TEST_PROJECT_NAME}", - f"--gcp-bucket-name={TEST_BUCKET_NAME}", + "--save-locally", + "--no-upload-to-cloud", "--use-dummy-serial-port", ], input="sleep 2\nstop\n", @@ -263,8 +263,8 @@ def test_start_with_config_file(self): [ "start", "--interactive", - f"--gcp-project-name={TEST_PROJECT_NAME}", - f"--gcp-bucket-name={TEST_BUCKET_NAME}", + "--save-locally", + "--no-upload-to-cloud", "--use-dummy-serial-port", f"--config-file={CONFIGURATION_PATH}", f"--output-dir={temporary_directory}", From 9bd2db9f42282c957d8325d77554e1e945aef429 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 8 Feb 2022 11:49:41 +0000 Subject: [PATCH 29/84] TST: Update unknown packet error test --- tests/test_data_gateway.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_data_gateway.py b/tests/test_data_gateway.py index f1d92b90..7f0a258a 100644 --- a/tests/test_data_gateway.py +++ b/tests/test_data_gateway.py @@ -6,7 +6,6 @@ from octue.cloud import storage from octue.cloud.storage.client import GoogleCloudStorageClient -from data_gateway import exceptions from data_gateway.configuration import Configuration from data_gateway.data_gateway import DataGateway from data_gateway.dummy_serial import DummySerial @@ -29,8 +28,8 @@ def setUpClass(cls): cls.WINDOW_SIZE = 10 cls.storage_client = GoogleCloudStorageClient(project_name=TEST_PROJECT_NAME) - def test_error_is_raised_if_unknown_sensor_type_packet_is_received(self): - """Test that an `UnknownPacketTypeException` is raised if an unknown sensor type packet is received.""" + def test_error_is_logged_if_unknown_sensor_type_packet_is_received(self): + """Test that an error is logged if an unknown sensor type packet is received.""" serial_port = DummySerial(port="test") packet_type = bytes([0]) @@ -46,8 +45,10 @@ def test_error_is_raised_if_unknown_sensor_type_packet_is_received(self): project_name=TEST_PROJECT_NAME, bucket_name=TEST_BUCKET_NAME, ) - with self.assertRaises(exceptions.UnknownPacketTypeError): - data_gateway.start() + with self.assertLogs() as logging_context: + data_gateway.start(stop_when_no_more_data=True) + + self.assertIn("Received packet with unknown type: 0", logging_context.output[1]) def test_configuration_file_is_persisted(self): """Test that the configuration file is persisted.""" From 9be30c2e7e88943009e7ce828a31a6c84e23f0a3 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 8 Feb 2022 12:09:16 +0000 Subject: [PATCH 30/84] TST: Mock the BigQuery client for testing --- tests/test_cloud_functions/base.py | 39 ------- tests/test_cloud_functions/mocks.py | 42 +++++++ tests/test_cloud_functions/test_big_query.py | 113 +++++++++---------- 3 files changed, 93 insertions(+), 101 deletions(-) delete mode 100644 tests/test_cloud_functions/base.py create mode 100644 tests/test_cloud_functions/mocks.py diff --git a/tests/test_cloud_functions/base.py b/tests/test_cloud_functions/base.py deleted file mode 100644 index ef5ecf12..00000000 --- a/tests/test_cloud_functions/base.py +++ /dev/null @@ -1,39 +0,0 @@ -import json -import os - -from octue.cloud.credentials import GCPCredentialsManager - - -class CredentialsEnvironmentVariableAsFile: - """Temporarily store JSON credentials from the `GOOGLE_APPLICATION_CREDENTIALS` environment variable in a file for - use during the test class's test run. This is useful on GitHub where a file cannot be created for a secret but - tests that require credentials to be present as a file are run. - """ - - def __init__(self): - self.credentials_path = "temporary_file.json" - self.current_google_application_credentials_variable_value = None - - def __enter__(self): - """Temporarily write the credentials to a file so that the tests can run on GitHub where the credentials are - only provided as JSON in an environment variable. Set the credentials environment variable to point to this - file instead of the credentials JSON. - - :return None: - """ - self.current_google_application_credentials_variable_value = os.environ["GOOGLE_APPLICATION_CREDENTIALS"] - - credentials = GCPCredentialsManager().get_credentials(as_dict=True) - - with open(self.credentials_path, "w") as f: - json.dump(credentials, f) - - os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = self.credentials_path - - def __exit__(self, exc_type, exc_val, exc_tb): - """Remove the temporary credentials file and restore the credentials environment variable to its original value. - - :return None: - """ - os.remove(self.credentials_path) - os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = self.current_google_application_credentials_variable_value diff --git a/tests/test_cloud_functions/mocks.py b/tests/test_cloud_functions/mocks.py new file mode 100644 index 00000000..ab4a9d4e --- /dev/null +++ b/tests/test_cloud_functions/mocks.py @@ -0,0 +1,42 @@ +class MockBigQueryClient: + def __init__(self, expected_query_result=None): + self.expected_query_result = expected_query_result + self.rows = None + + def get_table(self, name): + """Do nothing. + + :param str name: + :return None: + """ + pass + + def insert_rows(self, table, rows): + """Store the given rows in the `self.rows` attribute. + + :param str table: + :param list(dict) rows: + :return None: + """ + self.rows = rows + + def query(self, query): + """Return the `self.expected_query_result` attribute in a `MockQueryResult` instance. + + :param str query: + :return MockQueryResult: + """ + self.query = query + return MockQueryResult(result=self.expected_query_result) + + +class MockQueryResult: + def __init__(self, result): + self._result = result + + def result(self): + """Return the `self._result` attribute. + + :return any: + """ + return self._result diff --git a/tests/test_cloud_functions/test_big_query.py b/tests/test_cloud_functions/test_big_query.py index 9d1b7038..1116e5f1 100644 --- a/tests/test_cloud_functions/test_big_query.py +++ b/tests/test_cloud_functions/test_big_query.py @@ -5,7 +5,7 @@ from tests.base import BaseTestCase from tests.test_cloud_functions import REPOSITORY_ROOT -from tests.test_cloud_functions.base import CredentialsEnvironmentVariableAsFile +from tests.test_cloud_functions.mocks import MockBigQueryClient # Manually add the cloud_functions package to the path (its imports have to be done in a certain way for Google Cloud @@ -32,19 +32,17 @@ def test_insert_sensor_data(self): "Constat": [[1636559720.639327, 36, 37, 38, 39]], } - with CredentialsEnvironmentVariableAsFile(): - with patch("big_query.bigquery.Client.get_table"): - with patch("big_query.bigquery.Client.insert_rows", return_value=None) as mock_insert_rows: + mock_big_query_client = MockBigQueryClient() - BigQueryDataset(project_name="my-project", dataset_name="my-dataset").add_sensor_data( - data=data, - configuration_id="dbfed555-1b70-4191-96cb-c22071464b90", - installation_reference="turbine-1", - label="my-test", - ) + with patch("big_query.bigquery.Client", return_value=mock_big_query_client): + BigQueryDataset(project_name="my-project", dataset_name="my-dataset").add_sensor_data( + data=data, + configuration_id="dbfed555-1b70-4191-96cb-c22071464b90", + installation_reference="turbine-1", + label="my-test", + ) - new_rows = mock_insert_rows.call_args.kwargs["rows"] - self.assertEqual(len(new_rows), 8) + self.assertEqual(len(mock_big_query_client.rows), 8) expected_rows = [ { @@ -113,20 +111,17 @@ def test_insert_sensor_data(self): }, ] - self.assertEqual(new_rows, expected_rows) + self.assertEqual(mock_big_query_client.rows, expected_rows) def test_add_new_sensor_type(self): """Test that new sensor types can be added and that their references are their names slugified.""" - with CredentialsEnvironmentVariableAsFile(): - with patch("big_query.bigquery.Client.get_table"): - with patch("big_query.bigquery.Client.insert_rows", return_value=None) as mock_insert_rows: + mock_big_query_client = MockBigQueryClient() - BigQueryDataset(project_name="my-project", dataset_name="my-dataset").add_sensor_type( - name="My sensor_Name" - ) + with patch("big_query.bigquery.Client", return_value=mock_big_query_client): + BigQueryDataset(project_name="my-project", dataset_name="my-dataset").add_sensor_type(name="My sensor_Name") self.assertEqual( - mock_insert_rows.call_args.kwargs["rows"][0], + mock_big_query_client.rows[0], { "reference": "my-sensor-name", "name": "My sensor_Name", @@ -138,21 +133,19 @@ def test_add_new_sensor_type(self): def test_add_installation(self): """Test that installations can be added.""" - with CredentialsEnvironmentVariableAsFile(): - with patch("big_query.bigquery.Client.get_table"): - with patch("big_query.bigquery.Client.insert_rows", return_value=None) as mock_insert_rows: - with patch("big_query.bigquery.Client.query", return_value=Mock(result=lambda: [])): - - BigQueryDataset(project_name="my-project", dataset_name="my-dataset").add_installation( - reference="my-installation", - turbine_id="my-turbine", - blade_id="my-blade", - hardware_version="1.0.0", - sensor_coordinates={"my-sensor": [[0, 1, 2], [3, 8, 7]]}, - ) + mock_big_query_client = MockBigQueryClient(expected_query_result=[]) + + with patch("big_query.bigquery.Client", return_value=mock_big_query_client): + BigQueryDataset(project_name="my-project", dataset_name="my-dataset").add_installation( + reference="my-installation", + turbine_id="my-turbine", + blade_id="my-blade", + hardware_version="1.0.0", + sensor_coordinates={"my-sensor": [[0, 1, 2], [3, 8, 7]]}, + ) self.assertEqual( - mock_insert_rows.call_args.kwargs["rows"][0], + mock_big_query_client.rows[0], { "reference": "my-installation", "turbine_id": "my-turbine", @@ -165,34 +158,33 @@ def test_add_installation(self): def test_add_installation_raises_error_if_installation_already_exists(self): """Test that an error is raised if attempting to add an installation that already exists.""" - with CredentialsEnvironmentVariableAsFile(): + mock_big_query_client = MockBigQueryClient(expected_query_result=[1]) + + with patch("big_query.bigquery.Client", return_value=mock_big_query_client): dataset = BigQueryDataset(project_name="my-project", dataset_name="my-dataset") - with patch("big_query.bigquery.Client.query", return_value=Mock(result=lambda: [1])): - with self.assertRaises(InstallationWithSameNameAlreadyExists): - dataset.add_installation( - reference="my-installation", - turbine_id="my-turbine", - blade_id="my-blade", - hardware_version="1.0.0", - sensor_coordinates={"my-sensor": [[0, 1, 2], [3, 8, 7]]}, - ) + with self.assertRaises(InstallationWithSameNameAlreadyExists): + dataset.add_installation( + reference="my-installation", + turbine_id="my-turbine", + blade_id="my-blade", + hardware_version="1.0.0", + sensor_coordinates={"my-sensor": [[0, 1, 2], [3, 8, 7]]}, + ) def test_add_configuration(self): """Test that a configuration can be added.""" - with CredentialsEnvironmentVariableAsFile(): - with patch("big_query.bigquery.Client.get_table"): - with patch("big_query.bigquery.Client.insert_rows", return_value=None) as mock_insert_rows: - with patch("big_query.bigquery.Client.query", return_value=Mock(result=lambda: [])): + mock_big_query_client = MockBigQueryClient(expected_query_result=[]) - BigQueryDataset(project_name="my-project", dataset_name="my-dataset").add_configuration( - configuration={"blah": "blah", "installation_data": {"stuff": "data"}} - ) + with patch("big_query.bigquery.Client", return_value=mock_big_query_client): + BigQueryDataset(project_name="my-project", dataset_name="my-dataset").add_configuration( + configuration={"blah": "blah", "installation_data": {"stuff": "data"}} + ) - del mock_insert_rows.call_args.kwargs["rows"][0]["id"] + del mock_big_query_client.rows[0]["id"] self.assertEqual( - mock_insert_rows.call_args.kwargs["rows"][0], + mock_big_query_client.rows[0], { "software_configuration": '{"blah": "blah"}', "software_configuration_hash": "a9a553b17102e3f08a1ca32486086cdb8699f8f50c358b0fed8071b1d4c11bb2", @@ -206,17 +198,14 @@ def test_add_configuration_raises_error_if_installation_already_exists(self): existing configuration is returned. """ existing_configuration_id = "0846401a-89fb-424e-89e6-039063e0ee6d" + mock_big_query_client = MockBigQueryClient(expected_query_result=[Mock(id=existing_configuration_id)]) - with CredentialsEnvironmentVariableAsFile(): + with patch("big_query.bigquery.Client", return_value=mock_big_query_client): dataset = BigQueryDataset(project_name="my-project", dataset_name="my-dataset") - with patch( - "big_query.bigquery.Client.query", - return_value=Mock(result=lambda: [Mock(id=existing_configuration_id)]), - ): - with self.assertRaises(ConfigurationAlreadyExists): - configuration_id = dataset.add_configuration( - configuration={"blah": "blah", "installation_data": {"stuff": "data"}} - ) + with self.assertRaises(ConfigurationAlreadyExists): + configuration_id = dataset.add_configuration( + configuration={"blah": "blah", "installation_data": {"stuff": "data"}} + ) - self.assertEqual(configuration_id, existing_configuration_id) + self.assertEqual(configuration_id, existing_configuration_id) From 7e723efbc498df7aec8395aaa74c5dcdfe19e147 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 9 Feb 2022 14:47:33 +0000 Subject: [PATCH 31/84] ENH: Use thread pool to dispatch single reader and parser threads --- data_gateway/data_gateway.py | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/data_gateway/data_gateway.py b/data_gateway/data_gateway.py index 3105d2c6..309dbeb7 100644 --- a/data_gateway/data_gateway.py +++ b/data_gateway/data_gateway.py @@ -111,28 +111,24 @@ def start(self, stop_when_no_more_data=False): self.packet_reader.persist_configuration() - reader_thread_pool = ThreadPoolExecutor(thread_name_prefix="ReaderThread") + thread_pool = ThreadPoolExecutor(thread_name_prefix="DataGatewayThread") packet_queue = queue.Queue() error_queue = queue.Queue() try: - for _ in range(reader_thread_pool._max_workers): - reader_thread_pool.submit( - self.packet_reader.read_packets, - self.serial_port, - packet_queue, - error_queue, - stop_when_no_more_data, - ) - - parser_thread = threading.Thread( - name="ParserThread", - target=self.packet_reader.parse_packets, - kwargs={"packet_queue": packet_queue, "error_queue": error_queue}, - daemon=True, + thread_pool.submit( + self.packet_reader.read_packets, + serial_port=self.serial_port, + packet_queue=packet_queue, + error_queue=error_queue, + stop_when_no_more_data=stop_when_no_more_data, ) - parser_thread.start() + thread_pool.submit( + self.packet_reader.parse_packets, + packet_queue=packet_queue, + error_queue=error_queue, + ) if self.interactive: interactive_commands_thread = threading.Thread( @@ -155,7 +151,7 @@ def start(self, stop_when_no_more_data=False): finally: logger.info("Stopping gateway.") self.packet_reader.stop = True - reader_thread_pool.shutdown(wait=False) + thread_pool.shutdown(wait=False) self.packet_reader.writer.force_persist() self.packet_reader.uploader.force_persist() From 0ab623b0dcc8f7d992add98df1ed4132d3d3a0a2 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 11 Feb 2022 11:38:00 +0000 Subject: [PATCH 32/84] FEA: Run gateway on multiple processors --- data_gateway/cli.py | 3 +- data_gateway/data_gateway.py | 68 ++++++++++++++------------ data_gateway/packet_reader.py | 91 ++++++++++++++++++++++------------- data_gateway/persistence.py | 3 +- tests/__init__.py | 2 +- tests/test_data_gateway.py | 41 ++++++++-------- 6 files changed, 119 insertions(+), 89 deletions(-) diff --git a/data_gateway/cli.py b/data_gateway/cli.py index 42d32952..76cb2b15 100644 --- a/data_gateway/cli.py +++ b/data_gateway/cli.py @@ -41,7 +41,7 @@ def gateway_cli(logger_uri, log_level): from octue.log_handlers import apply_log_handler, get_remote_handler # Apply log handler locally. - apply_log_handler(log_level=log_level.upper(), include_thread_name=True) + apply_log_handler(log_level=log_level.upper(), include_thread_name=True, include_process_name=True) # Stream logs to remote handler if required. if logger_uri is not None: @@ -49,6 +49,7 @@ def gateway_cli(logger_uri, log_level): handler=get_remote_handler(logger_uri=logger_uri), log_level=log_level.upper(), include_thread_name=True, + include_process_name=True, ) diff --git a/data_gateway/data_gateway.py b/data_gateway/data_gateway.py index 309dbeb7..5d3f9d30 100644 --- a/data_gateway/data_gateway.py +++ b/data_gateway/data_gateway.py @@ -1,13 +1,12 @@ import json -import logging +import multiprocessing import os -import queue import sys import threading import time -from concurrent.futures import ThreadPoolExecutor import serial +from octue.log_handlers import apply_log_handler from data_gateway.configuration import Configuration from data_gateway.dummy_serial import DummySerial @@ -16,7 +15,8 @@ from data_gateway.routine import Routine -logger = logging.getLogger(__name__) +logger = multiprocessing.get_logger() +apply_log_handler(logger=logger) class DataGateway: @@ -78,7 +78,7 @@ def __init__( self.packet_reader = PacketReader( save_locally=save_locally, upload_to_cloud=upload_to_cloud, - output_directory=self._update_and_create_output_directory(output_directory_path=output_directory), + output_directory=self._update_output_directory(output_directory_path=output_directory), window_size=window_size, project_name=project_name, bucket_name=bucket_name, @@ -109,27 +109,40 @@ def start(self, stop_when_no_more_data=False): self.packet_reader.window_size, ) - self.packet_reader.persist_configuration() + # self.packet_reader.persist_configuration() - thread_pool = ThreadPoolExecutor(thread_name_prefix="DataGatewayThread") - packet_queue = queue.Queue() - error_queue = queue.Queue() + packet_queue = multiprocessing.Queue() + error_queue = multiprocessing.Queue() + stop_signal = multiprocessing.Value("i", 0) try: - thread_pool.submit( - self.packet_reader.read_packets, - serial_port=self.serial_port, - packet_queue=packet_queue, - error_queue=error_queue, - stop_when_no_more_data=stop_when_no_more_data, + reader_process = multiprocessing.Process( + name="ReaderProcess", + target=self.packet_reader.read_packets, + kwargs={ + "serial_port": self.serial_port, + "packet_queue": packet_queue, + "error_queue": error_queue, + "stop_signal": stop_signal, + "stop_when_no_more_data": stop_when_no_more_data, + }, + daemon=True, ) - thread_pool.submit( - self.packet_reader.parse_packets, - packet_queue=packet_queue, - error_queue=error_queue, + parser_process = multiprocessing.Process( + name="ParserProcess", + target=self.packet_reader.parse_packets, + kwargs={ + "packet_queue": packet_queue, + "error_queue": error_queue, + "stop_signal": stop_signal, + }, + daemon=True, ) + reader_process.start() + parser_process.start() + if self.interactive: interactive_commands_thread = threading.Thread( name="InteractiveCommandsThread", @@ -144,16 +157,13 @@ def start(self, stop_when_no_more_data=False): routine_thread.start() # Raise any errors from the reader threads and parser thread. - while not self.packet_reader.stop: + while stop_signal.value == 0: if not error_queue.empty(): - raise error_queue.get() + raise error_queue.get(timeout=10) finally: - logger.info("Stopping gateway.") - self.packet_reader.stop = True - thread_pool.shutdown(wait=False) - self.packet_reader.writer.force_persist() - self.packet_reader.uploader.force_persist() + logger.info("Sending stop signal.") + stop_signal.value = 1 def _load_configuration(self, configuration_path): """Load a configuration from the path if it exists, otherwise load the default configuration. @@ -226,9 +236,8 @@ def _load_routine(self, routine_path): routine_path, ) - def _update_and_create_output_directory(self, output_directory_path): - """Set the output directory to a path relative to the current directory if the path does not start with "/" and - create it if it does not already exist. + def _update_output_directory(self, output_directory_path): + """Set the output directory to a path relative to the current directory if the path does not start with "/". :param str output_directory_path: the path to the directory to write output data to :return str: the updated output directory path @@ -236,7 +245,6 @@ def _update_and_create_output_directory(self, output_directory_path): if not output_directory_path.startswith("/"): output_directory_path = os.path.join(".", output_directory_path) - os.makedirs(output_directory_path, exist_ok=True) return output_directory_path def _send_commands_from_stdin_to_sensors(self): diff --git a/data_gateway/packet_reader.py b/data_gateway/packet_reader.py index 957a1a0a..f3a6b004 100644 --- a/data_gateway/packet_reader.py +++ b/data_gateway/packet_reader.py @@ -1,17 +1,20 @@ import datetime import json -import logging +import multiprocessing import os +import queue import struct from octue.cloud import storage +from octue.log_handlers import apply_log_handler from data_gateway import MICROPHONE_SENSOR_NAME, exceptions from data_gateway.configuration import Configuration from data_gateway.persistence import BatchingFileWriter, BatchingUploader, NoOperationContextManager -logger = logging.getLogger(__name__) +logger = multiprocessing.get_logger() +apply_log_handler(logger=logger) class PacketReader: @@ -43,40 +46,23 @@ def __init__( self.upload_to_cloud = upload_to_cloud self.output_directory = output_directory self.window_size = window_size + self.project_name = project_name + self.bucket_name = bucket_name self.config = configuration or Configuration() + self.save_csv_files = save_csv_files + + self.uploader = None + self.writer = None self.handles = self.config.default_handles self.sleep = False self.stop = False self.sensor_time_offset = None self.session_subdirectory = str(hash(datetime.datetime.now()))[1:7] + os.makedirs(os.path.join(output_directory, self.session_subdirectory), exist_ok=True) logger.warning("Timestamp synchronisation unavailable with current hardware; defaulting to using system clock.") - if upload_to_cloud: - self.uploader = BatchingUploader( - sensor_names=self.config.sensor_names, - project_name=project_name, - bucket_name=bucket_name, - window_size=self.window_size, - session_subdirectory=self.session_subdirectory, - output_directory=output_directory, - metadata={"data_gateway__configuration": self.config.to_dict()}, - ) - else: - self.uploader = NoOperationContextManager() - - if save_locally: - self.writer = BatchingFileWriter( - sensor_names=self.config.sensor_names, - window_size=self.window_size, - session_subdirectory=self.session_subdirectory, - output_directory=output_directory, - save_csv_files=save_csv_files, - ) - else: - self.writer = NoOperationContextManager() - - def read_packets(self, serial_port, packet_queue, error_queue, stop_when_no_more_data=False): + def read_packets(self, serial_port, packet_queue, error_queue, stop_signal, stop_when_no_more_data=False): """Read packets from a serial port and send them to the parser thread for processing and persistence. :param serial.Serial serial_port: name of serial port to read from @@ -86,7 +72,7 @@ def read_packets(self, serial_port, packet_queue, error_queue, stop_when_no_more :return None: """ try: - logger.debug("Beginning reading packets from serial port.") + logger.info("Beginning reading packets from serial port.") previous_timestamp = {} data = {} @@ -98,12 +84,13 @@ def read_packets(self, serial_port, packet_queue, error_queue, stop_when_no_more for _ in range(self.config.number_of_sensors[sensor_name]) ] - while not self.stop: + while stop_signal.value == 0: serial_data = serial_port.read() if len(serial_data) == 0: if stop_when_no_more_data: - self.stop = True + logger.info("Sending stop signal.") + stop_signal.value = 1 break continue @@ -113,7 +100,7 @@ def read_packets(self, serial_port, packet_queue, error_queue, stop_when_no_more packet_type = str(int.from_bytes(serial_port.read(), self.config.endian)) length = int.from_bytes(serial_port.read(), self.config.endian) payload = serial_port.read(length) - logger.debug("Packet received.") + logger.info("Read packet from serial port.") if packet_type == str(self.config.type_handle_def): self.update_handles(payload) @@ -140,8 +127,10 @@ def read_packets(self, serial_port, packet_queue, error_queue, stop_when_no_more except Exception as e: error_queue.put(e) + logger.info("Sending stop signal.") + stop_signal.value = 1 - def parse_packets(self, packet_queue, error_queue): + def parse_packets(self, packet_queue, error_queue, stop_signal): """Get packets from a thread-safe packet queue, check if a full payload has been received (i.e. correct length) with the correct packet type handle, then parse the payload. After parsing/processing, upload them to Google Cloud storage and/or write them to disk. If any errors are raised, put them on the error queue for the main @@ -151,11 +140,38 @@ def parse_packets(self, packet_queue, error_queue): :param queue.Queue error_queue: a thread-safe queue to put any exceptions on to for the main thread to handle :return None: """ + logger.info("Beginning parsing packets from serial port.") + + if self.upload_to_cloud: + self.uploader = BatchingUploader( + sensor_names=self.config.sensor_names, + project_name=self.project_name, + bucket_name=self.bucket_name, + window_size=self.window_size, + session_subdirectory=self.session_subdirectory, + output_directory=self.output_directory, + metadata={"data_gateway__configuration": self.config.to_dict()}, + ) + else: + self.uploader = NoOperationContextManager() + + if self.save_locally: + self.writer = BatchingFileWriter( + sensor_names=self.config.sensor_names, + window_size=self.window_size, + session_subdirectory=self.session_subdirectory, + output_directory=self.output_directory, + save_csv_files=self.save_csv_files, + ) + else: + self.writer = NoOperationContextManager() + try: with self.uploader: with self.writer: - while not self.stop: - packet_type, payload, data, previous_timestamp = packet_queue.get().values() + while stop_signal.value == 0: + packet_type, payload, data, previous_timestamp = packet_queue.get(timeout=1).values() + logger.info("Received packet for parsing.") if packet_type not in self.handles: logger.error("Received packet with unknown type: %s", packet_type) @@ -182,9 +198,16 @@ def parse_packets(self, packet_queue, error_queue): ]: self._parse_info_packet(self.handles[packet_type], payload) + except queue.Empty: + pass + except Exception as e: error_queue.put(e) + finally: + logger.info("Sending stop signal.") + stop_signal.value = 1 + def update_handles(self, payload): """Update the Bluetooth handles object. Handles are updated every time a new Bluetooth connection is established. diff --git a/data_gateway/persistence.py b/data_gateway/persistence.py index fad3ccc2..0df4a2f5 100644 --- a/data_gateway/persistence.py +++ b/data_gateway/persistence.py @@ -48,6 +48,8 @@ class TimeBatcher: :return None: """ + _file_prefix = "window" + def __init__( self, sensor_names, @@ -62,7 +64,6 @@ def __init__( self._session_subdirectory = session_subdirectory self._start_time = time.perf_counter() self._window_number = 0 - self._file_prefix = "window" def __enter__(self): return self diff --git a/tests/__init__.py b/tests/__init__.py index b6c30085..280da59e 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -3,7 +3,7 @@ from data_gateway.configuration import Configuration -apply_log_handler(include_thread_name=True) +apply_log_handler(include_thread_name=True, include_process_name=True) TEST_PROJECT_NAME = "a-project-name" diff --git a/tests/test_data_gateway.py b/tests/test_data_gateway.py index 7f0a258a..eee20493 100644 --- a/tests/test_data_gateway.py +++ b/tests/test_data_gateway.py @@ -9,6 +9,7 @@ from data_gateway.configuration import Configuration from data_gateway.data_gateway import DataGateway from data_gateway.dummy_serial import DummySerial +from data_gateway.persistence import TimeBatcher from tests import LENGTH, PACKET_KEY, RANDOM_BYTES, TEST_BUCKET_NAME, TEST_PROJECT_NAME from tests.base import BaseTestCase @@ -164,13 +165,11 @@ def test_data_gateway_with_baros_p_sensor(self): bucket_name=TEST_BUCKET_NAME, ) - data_gateway.start(stop_when_no_more_data=True) - self._check_data_is_written_to_files( - data_gateway.packet_reader, temporary_directory, sensor_names=["Baros_P"] - ) + data_gateway.start(stop_when_no_more_data=False) + self._check_data_is_written_to_files(temporary_directory, sensor_names=["Baros_P"]) self._check_windows_are_uploaded_to_cloud( - data_gateway.packet_reader, sensor_names=["Baros_P"], number_of_windows_to_check=1 + temporary_directory, sensor_names=["Baros_P"], number_of_windows_to_check=1 ) def test_data_gateway_with_baros_t_sensor(self): @@ -462,31 +461,29 @@ def test_data_gateway_with_info_packets(self): ]: self.assertIn(message, log_messages_combined) - def _check_windows_are_uploaded_to_cloud(self, packet_reader, sensor_names, number_of_windows_to_check=5): + def _check_windows_are_uploaded_to_cloud(self, output_directory, sensor_names, number_of_windows_to_check=5): """Check that non-trivial windows from a packet reader for a particular sensor are uploaded to cloud storage.""" - number_of_windows = packet_reader.uploader._window_number - self.assertTrue(number_of_windows > 0) - - for i in range(number_of_windows_to_check): - data = json.loads( - self.storage_client.download_as_string( - bucket_name=TEST_BUCKET_NAME, - path_in_bucket=storage.path.join( - packet_reader.uploader.output_directory, - packet_reader.uploader._session_subdirectory, - f"window-{i}.json", - ), - ) + window_paths = [ + blob.name + for blob in self.storage_client.scandir( + cloud_path=storage.path.generate_gs_path(TEST_BUCKET_NAME, *output_directory.split(os.path.pathsep)) ) + ] + + self.assertTrue(len(window_paths) >= number_of_windows_to_check) + + for path in window_paths: + data = json.loads(self.storage_client.download_as_string(bucket_name=TEST_BUCKET_NAME, path_in_bucket=path)) for name in sensor_names: lines = data["sensor_data"][name] self.assertTrue(len(lines[0]) > 1) - def _check_data_is_written_to_files(self, packet_reader, temporary_directory, sensor_names): + def _check_data_is_written_to_files(self, output_directory, sensor_names): """Check that non-trivial data is written to the given file.""" - window_directory = os.path.join(temporary_directory, packet_reader.writer._session_subdirectory) - windows = [file for file in os.listdir(window_directory) if file.startswith(packet_reader.writer._file_prefix)] + session_subdirectory = os.listdir(output_directory)[0] + window_directory = os.path.join(output_directory, session_subdirectory) + windows = [file for file in os.listdir(window_directory) if file.startswith(TimeBatcher._file_prefix)] self.assertTrue(len(windows) > 0) for window in windows: From a1869bd60b3e5dd626a1fe68d5a91ff91c1bec47 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 11 Feb 2022 11:46:58 +0000 Subject: [PATCH 33/84] FIX: Share stop signal with MainProcess's threads --- data_gateway/data_gateway.py | 15 +++++++++++---- data_gateway/packet_reader.py | 1 - data_gateway/routine.py | 10 ++++++---- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/data_gateway/data_gateway.py b/data_gateway/data_gateway.py index 5d3f9d30..ee8782ff 100644 --- a/data_gateway/data_gateway.py +++ b/data_gateway/data_gateway.py @@ -147,13 +147,19 @@ def start(self, stop_when_no_more_data=False): interactive_commands_thread = threading.Thread( name="InteractiveCommandsThread", target=self._send_commands_from_stdin_to_sensors, + kwargs={"stop_signal": stop_signal}, daemon=True, ) interactive_commands_thread.start() elif self.routine is not None: - routine_thread = threading.Thread(name="RoutineCommandsThread", target=self.routine.run, daemon=True) + routine_thread = threading.Thread( + name="RoutineCommandsThread", + target=self.routine.run, + kwargs={"stop_signal": stop_signal}, + daemon=True, + ) routine_thread.start() # Raise any errors from the reader threads and parser thread. @@ -247,7 +253,7 @@ def _update_output_directory(self, output_directory_path): return output_directory_path - def _send_commands_from_stdin_to_sensors(self): + def _send_commands_from_stdin_to_sensors(self, stop_signal): """Send commands from `stdin` to the sensors until the "stop" command is received or the packet reader is otherwise stopped. A record is kept of the commands sent to the sensors as a text file in the session subdirectory. Available commands: [startBaros, startMics, startIMU, getBattery, stop]. @@ -265,7 +271,7 @@ def _send_commands_from_stdin_to_sensors(self): exist_ok=True, ) - while not self.packet_reader.stop: + while stop_signal.value == 0: for line in sys.stdin: with open(commands_record_file, "a") as f: f.write(line) @@ -273,8 +279,9 @@ def _send_commands_from_stdin_to_sensors(self): if line.startswith("sleep") and line.endswith("\n"): time.sleep(int(line.split(" ")[-1].strip())) elif line == "stop\n": - self.packet_reader.stop = True self.serial_port.write(line.encode("utf_8")) + logger.info("Sending stop signal.") + stop_signal.value = 1 break # Send the command to the node diff --git a/data_gateway/packet_reader.py b/data_gateway/packet_reader.py index f3a6b004..8daafcf7 100644 --- a/data_gateway/packet_reader.py +++ b/data_gateway/packet_reader.py @@ -55,7 +55,6 @@ def __init__( self.writer = None self.handles = self.config.default_handles self.sleep = False - self.stop = False self.sensor_time_offset = None self.session_subdirectory = str(hash(datetime.datetime.now()))[1:7] diff --git a/data_gateway/routine.py b/data_gateway/routine.py index 9c0d9014..3368091a 100644 --- a/data_gateway/routine.py +++ b/data_gateway/routine.py @@ -39,7 +39,7 @@ def __init__(self, commands, action, packet_reader, period=None, stop_after=None if self.stop_after: logger.warning("The `stop_after` parameter is ignored unless `period` is also given.") - def run(self): + def run(self, stop_signal): """Send the commands to the action after the given delays, repeating if a period was given. :return None: @@ -47,7 +47,7 @@ def run(self): scheduler = sched.scheduler(time.perf_counter) start_time = time.perf_counter() - while not self.packet_reader.stop: + while not stop_signal: cycle_start_time = time.perf_counter() for command, delay in self.commands: @@ -56,7 +56,8 @@ def run(self): scheduler.run(blocking=True) if self.period is None: - self.packet_reader.stop = True + logger.info("Sending stop signal.") + stop_signal.value = 1 return elapsed_time = time.perf_counter() - cycle_start_time @@ -64,7 +65,8 @@ def run(self): if self.stop_after: if time.perf_counter() - start_time >= self.stop_after: - self.packet_reader.stop = True + logger.info("Sending stop signal.") + stop_signal.value = 1 return def _wrap_action_with_logger(self, action): From 977a66a01c188beecd460c0bfdc27a026ac0a712 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 11 Feb 2022 12:03:37 +0000 Subject: [PATCH 34/84] ENH: Log full output directory at start --- data_gateway/cli.py | 8 +++----- data_gateway/data_gateway.py | 16 +--------------- data_gateway/packet_reader.py | 2 -- data_gateway/persistence.py | 16 +++++++++++++--- 4 files changed, 17 insertions(+), 25 deletions(-) diff --git a/data_gateway/cli.py b/data_gateway/cli.py index 76cb2b15..2a87ba1b 100644 --- a/data_gateway/cli.py +++ b/data_gateway/cli.py @@ -1,5 +1,5 @@ import json -import logging +import multiprocessing import os import click @@ -15,7 +15,7 @@ SUPERVISORD_PROGRAM_NAME = "AerosenseGateway" CREATE_INSTALLATION_CLOUD_FUNCTION_URL = "https://europe-west6-aerosense-twined.cloudfunctions.net/create-installation" -logger = logging.getLogger(__name__) +logger = multiprocessing.get_logger() @click.group(context_settings={"help_option_names": ["-h", "--help"]}) @@ -40,12 +40,10 @@ def gateway_cli(logger_uri, log_level): """ from octue.log_handlers import apply_log_handler, get_remote_handler - # Apply log handler locally. - apply_log_handler(log_level=log_level.upper(), include_thread_name=True, include_process_name=True) - # Stream logs to remote handler if required. if logger_uri is not None: apply_log_handler( + logger=logger, handler=get_remote_handler(logger_uri=logger_uri), log_level=log_level.upper(), include_thread_name=True, diff --git a/data_gateway/data_gateway.py b/data_gateway/data_gateway.py index ee8782ff..accd8b81 100644 --- a/data_gateway/data_gateway.py +++ b/data_gateway/data_gateway.py @@ -6,7 +6,6 @@ import time import serial -from octue.log_handlers import apply_log_handler from data_gateway.configuration import Configuration from data_gateway.dummy_serial import DummySerial @@ -16,7 +15,6 @@ logger = multiprocessing.get_logger() -apply_log_handler(logger=logger) class DataGateway: @@ -95,19 +93,7 @@ def start(self, stop_when_no_more_data=False): :return None: """ - logger.info("Starting packet reader.") - - if self.packet_reader.upload_to_cloud: - logger.debug( - "Files will be uploaded to cloud storage at intervals of %s seconds.", - self.packet_reader.window_size, - ) - - if self.packet_reader.save_locally: - logger.debug( - "Files will be saved locally to disk at intervals of %s seconds.", - self.packet_reader.window_size, - ) + logger.info("Starting data gateway.") # self.packet_reader.persist_configuration() diff --git a/data_gateway/packet_reader.py b/data_gateway/packet_reader.py index 8daafcf7..e29f5789 100644 --- a/data_gateway/packet_reader.py +++ b/data_gateway/packet_reader.py @@ -6,7 +6,6 @@ import struct from octue.cloud import storage -from octue.log_handlers import apply_log_handler from data_gateway import MICROPHONE_SENSOR_NAME, exceptions from data_gateway.configuration import Configuration @@ -14,7 +13,6 @@ logger = multiprocessing.get_logger() -apply_log_handler(logger=logger) class PacketReader: diff --git a/data_gateway/persistence.py b/data_gateway/persistence.py index 0df4a2f5..8d70f71b 100644 --- a/data_gateway/persistence.py +++ b/data_gateway/persistence.py @@ -2,17 +2,18 @@ import copy import csv import json -import logging +import multiprocessing import os import time from octue.cloud import storage from octue.cloud.storage.client import GoogleCloudStorageClient +from octue.log_handlers import apply_log_handler from octue.utils.persistence import calculate_disk_usage, get_oldest_file_in_directory -logger = logging.getLogger(__name__) - +logger = multiprocessing.get_logger() +apply_log_handler(logger=logger) DEFAULT_OUTPUT_DIRECTORY = "data_gateway" @@ -62,6 +63,7 @@ def __init__( self.output_directory = output_directory self.ready_window = {"sensor_time_offset": None, "sensor_data": {}} self._session_subdirectory = session_subdirectory + self._full_output_path = None self._start_time = time.perf_counter() self._window_number = 0 @@ -157,7 +159,9 @@ def __init__( self._save_csv_files = save_csv_files self.storage_limit = storage_limit super().__init__(sensor_names, window_size, session_subdirectory, output_directory) + self._full_output_path = os.path.join(self.output_directory, self._session_subdirectory) os.makedirs(os.path.join(self.output_directory, self._session_subdirectory), exist_ok=True) + logger.info(f"Windows will be saved to {self._full_output_path!r} at intervals of {self.window_size} seconds.") def _persist_window(self, window=None): """Write a window of serialised data to disk, deleting the oldest window first if the storage limit has been @@ -253,11 +257,17 @@ def __init__( self.upload_timeout = upload_timeout self.upload_backup_files = upload_backup_files super().__init__(sensor_names, window_size, session_subdirectory, output_directory) + self._full_output_path = storage.path.join(self.output_directory, self._session_subdirectory) + self._backup_directory = os.path.join(self.output_directory, ".backup") self._backup_writer = BatchingFileWriter( sensor_names, window_size, session_subdirectory, output_directory=self._backup_directory ) + logger.info( + f"Windows will be uploaded to {self._full_output_path!r} at intervals of {self.window_size} seconds." + ) + def _persist_window(self): """Upload a window to Google Cloud storage. If the window fails to upload, it is instead written to disk. From 6d689dd587317338ec4fd82c31a5222a0ba096a2 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 11 Feb 2022 13:36:59 +0000 Subject: [PATCH 35/84] DEP: Use latest octue version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 62f45ec2..97eca16e 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ "click>=7.1.2", "pyserial==3.5", "python-slugify==5.0.2", - "octue==0.10.0", + "octue @ https://github.com/octue/octue-sdk-python/archive/enhancement/allow-apply-log-handler-to-work-on-logger-instance.zip", ], url="https://gitlab.com/windenergie-hsr/aerosense/digital-twin/data-gateway", license="MIT", From 75749b9570ead32d1c63eb0a4e8c0cd36cea5264 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 11 Feb 2022 13:37:35 +0000 Subject: [PATCH 36/84] ENH: Demote packet receipt log messages --- data_gateway/packet_reader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data_gateway/packet_reader.py b/data_gateway/packet_reader.py index e29f5789..ddb1a7ba 100644 --- a/data_gateway/packet_reader.py +++ b/data_gateway/packet_reader.py @@ -97,7 +97,7 @@ def read_packets(self, serial_port, packet_queue, error_queue, stop_signal, stop packet_type = str(int.from_bytes(serial_port.read(), self.config.endian)) length = int.from_bytes(serial_port.read(), self.config.endian) payload = serial_port.read(length) - logger.info("Read packet from serial port.") + logger.debug("Read packet from serial port.") if packet_type == str(self.config.type_handle_def): self.update_handles(payload) @@ -168,7 +168,7 @@ def parse_packets(self, packet_queue, error_queue, stop_signal): with self.writer: while stop_signal.value == 0: packet_type, payload, data, previous_timestamp = packet_queue.get(timeout=1).values() - logger.info("Received packet for parsing.") + logger.debug("Received packet for parsing.") if packet_type not in self.handles: logger.error("Received packet with unknown type: %s", packet_type) From 18cb4dcb9a48d19e00f48f9a8c6e41aac4fcd01f Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 11 Feb 2022 13:39:11 +0000 Subject: [PATCH 37/84] ENH: Set parser process wait timeout to 1 hour --- data_gateway/packet_reader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data_gateway/packet_reader.py b/data_gateway/packet_reader.py index ddb1a7ba..ee0a6484 100644 --- a/data_gateway/packet_reader.py +++ b/data_gateway/packet_reader.py @@ -127,7 +127,7 @@ def read_packets(self, serial_port, packet_queue, error_queue, stop_signal, stop logger.info("Sending stop signal.") stop_signal.value = 1 - def parse_packets(self, packet_queue, error_queue, stop_signal): + def parse_packets(self, packet_queue, error_queue, stop_signal, timeout=3600): """Get packets from a thread-safe packet queue, check if a full payload has been received (i.e. correct length) with the correct packet type handle, then parse the payload. After parsing/processing, upload them to Google Cloud storage and/or write them to disk. If any errors are raised, put them on the error queue for the main @@ -167,7 +167,7 @@ def parse_packets(self, packet_queue, error_queue, stop_signal): with self.uploader: with self.writer: while stop_signal.value == 0: - packet_type, payload, data, previous_timestamp = packet_queue.get(timeout=1).values() + packet_type, payload, data, previous_timestamp = packet_queue.get(timeout=timeout).values() logger.debug("Received packet for parsing.") if packet_type not in self.handles: From f0cc89cc51ebac6eeea0e9f5208d3315e1833011 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 11 Feb 2022 13:45:52 +0000 Subject: [PATCH 38/84] FIX: Move local variables from reader process to parser process skipci --- data_gateway/packet_reader.py | 53 +++++++++++++++++------------------ 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/data_gateway/packet_reader.py b/data_gateway/packet_reader.py index ee0a6484..9717e394 100644 --- a/data_gateway/packet_reader.py +++ b/data_gateway/packet_reader.py @@ -71,16 +71,6 @@ def read_packets(self, serial_port, packet_queue, error_queue, stop_signal, stop try: logger.info("Beginning reading packets from serial port.") - previous_timestamp = {} - data = {} - - for sensor_name in self.config.sensor_names: - previous_timestamp[sensor_name] = -1 - data[sensor_name] = [ - ([0] * self.config.samples_per_packet[sensor_name]) - for _ in range(self.config.number_of_sensors[sensor_name]) - ] - while stop_signal.value == 0: serial_data = serial_port.read() @@ -96,11 +86,11 @@ def read_packets(self, serial_port, packet_queue, error_queue, stop_signal, stop packet_type = str(int.from_bytes(serial_port.read(), self.config.endian)) length = int.from_bytes(serial_port.read(), self.config.endian) - payload = serial_port.read(length) + packet = serial_port.read(length) logger.debug("Read packet from serial port.") if packet_type == str(self.config.type_handle_def): - self.update_handles(payload) + self.update_handles(packet) continue # Check for bytes in serial input buffer. A full buffer results in overflow. @@ -113,14 +103,7 @@ def read_packets(self, serial_port, packet_queue, error_queue, stop_signal, stop serial_port.open() continue - packet_queue.put( - { - "packet_type": packet_type, - "payload": payload, - "data": data, - "previous_timestamp": previous_timestamp, - } - ) + packet_queue.put({"packet_type": packet_type, "packet": packet}) except Exception as e: error_queue.put(e) @@ -163,37 +146,53 @@ def parse_packets(self, packet_queue, error_queue, stop_signal, timeout=3600): else: self.writer = NoOperationContextManager() + previous_timestamp = {} + data = {} + + for sensor_name in self.config.sensor_names: + previous_timestamp[sensor_name] = -1 + data[sensor_name] = [ + ([0] * self.config.samples_per_packet[sensor_name]) + for _ in range(self.config.number_of_sensors[sensor_name]) + ] + try: with self.uploader: with self.writer: while stop_signal.value == 0: - packet_type, payload, data, previous_timestamp = packet_queue.get(timeout=timeout).values() + packet_type, packet = packet_queue.get(timeout=timeout).values() logger.debug("Received packet for parsing.") if packet_type not in self.handles: logger.error("Received packet with unknown type: %s", packet_type) continue - if len(payload) == 244: # If the full data payload is received, proceed parsing it - timestamp = int.from_bytes(payload[240:244], self.config.endian, signed=False) / (2 ** 16) + if len(packet) == 244: # If the full data payload is received, proceed parsing it + timestamp = int.from_bytes(packet[240:244], self.config.endian, signed=False) / (2 ** 16) data, sensor_names = self._parse_sensor_packet_data( - self.handles[packet_type], payload, data + packet_type=self.handles[packet_type], + payload=packet, + data=data, ) for sensor_name in sensor_names: self._check_for_packet_loss(sensor_name, timestamp, previous_timestamp) + self._timestamp_and_persist_data( - data, sensor_name, timestamp, self.config.period[sensor_name] + data=data, + sensor_name=sensor_name, + timestamp=timestamp, + period=self.config.period[sensor_name], ) - elif len(payload) >= 1 and self.handles[packet_type] in [ + elif len(packet) >= 1 and self.handles[packet_type] in [ "Mic 1", "Cmd Decline", "Sleep State", "Info Message", ]: - self._parse_info_packet(self.handles[packet_type], payload) + self._parse_info_packet(self.handles[packet_type], packet) except queue.Empty: pass From 5b60978eb4ed94133a2af52dd085dca983198b85 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 11 Feb 2022 13:51:11 +0000 Subject: [PATCH 39/84] ENH: Apply log handler to top level multiprocessed logger skipci --- data_gateway/data_gateway.py | 2 ++ data_gateway/persistence.py | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/data_gateway/data_gateway.py b/data_gateway/data_gateway.py index accd8b81..f3ca760b 100644 --- a/data_gateway/data_gateway.py +++ b/data_gateway/data_gateway.py @@ -6,6 +6,7 @@ import time import serial +from octue.log_handlers import apply_log_handler from data_gateway.configuration import Configuration from data_gateway.dummy_serial import DummySerial @@ -15,6 +16,7 @@ logger = multiprocessing.get_logger() +apply_log_handler(logger=logger, include_process_name=True, include_thread_name=True) class DataGateway: diff --git a/data_gateway/persistence.py b/data_gateway/persistence.py index 8d70f71b..4339b3d8 100644 --- a/data_gateway/persistence.py +++ b/data_gateway/persistence.py @@ -8,12 +8,11 @@ from octue.cloud import storage from octue.cloud.storage.client import GoogleCloudStorageClient -from octue.log_handlers import apply_log_handler from octue.utils.persistence import calculate_disk_usage, get_oldest_file_in_directory logger = multiprocessing.get_logger() -apply_log_handler(logger=logger) +# apply_log_handler(logger=logger) DEFAULT_OUTPUT_DIRECTORY = "data_gateway" From de318c7205954f2994a55a6386a76d3640fbd7fc Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 11 Feb 2022 14:27:12 +0000 Subject: [PATCH 40/84] REF: Remove unnecessary error queue --- data_gateway/data_gateway.py | 91 ++++++++++++++++------------------- data_gateway/packet_reader.py | 12 ++--- 2 files changed, 45 insertions(+), 58 deletions(-) diff --git a/data_gateway/data_gateway.py b/data_gateway/data_gateway.py index f3ca760b..8870561a 100644 --- a/data_gateway/data_gateway.py +++ b/data_gateway/data_gateway.py @@ -100,64 +100,55 @@ def start(self, stop_when_no_more_data=False): # self.packet_reader.persist_configuration() packet_queue = multiprocessing.Queue() - error_queue = multiprocessing.Queue() stop_signal = multiprocessing.Value("i", 0) - try: - reader_process = multiprocessing.Process( - name="ReaderProcess", - target=self.packet_reader.read_packets, - kwargs={ - "serial_port": self.serial_port, - "packet_queue": packet_queue, - "error_queue": error_queue, - "stop_signal": stop_signal, - "stop_when_no_more_data": stop_when_no_more_data, - }, - daemon=True, - ) + reader_process = multiprocessing.Process( + name="ReaderProcess", + target=self.packet_reader.read_packets, + kwargs={ + "serial_port": self.serial_port, + "packet_queue": packet_queue, + "stop_signal": stop_signal, + "stop_when_no_more_data": stop_when_no_more_data, + }, + daemon=True, + ) - parser_process = multiprocessing.Process( - name="ParserProcess", - target=self.packet_reader.parse_packets, - kwargs={ - "packet_queue": packet_queue, - "error_queue": error_queue, - "stop_signal": stop_signal, - }, - daemon=True, - ) + parser_process = multiprocessing.Process( + name="ParserProcess", + target=self.packet_reader.parse_packets, + kwargs={ + "packet_queue": packet_queue, + "stop_signal": stop_signal, + }, + daemon=True, + ) - reader_process.start() - parser_process.start() + reader_process.start() + parser_process.start() - if self.interactive: - interactive_commands_thread = threading.Thread( - name="InteractiveCommandsThread", - target=self._send_commands_from_stdin_to_sensors, - kwargs={"stop_signal": stop_signal}, - daemon=True, - ) - - interactive_commands_thread.start() + if self.interactive: + interactive_commands_thread = threading.Thread( + name="InteractiveCommandsThread", + target=self._send_commands_from_stdin_to_sensors, + kwargs={"stop_signal": stop_signal}, + daemon=True, + ) - elif self.routine is not None: - routine_thread = threading.Thread( - name="RoutineCommandsThread", - target=self.routine.run, - kwargs={"stop_signal": stop_signal}, - daemon=True, - ) - routine_thread.start() + interactive_commands_thread.start() - # Raise any errors from the reader threads and parser thread. - while stop_signal.value == 0: - if not error_queue.empty(): - raise error_queue.get(timeout=10) + elif self.routine is not None: + routine_thread = threading.Thread( + name="RoutineCommandsThread", + target=self.routine.run, + kwargs={"stop_signal": stop_signal}, + daemon=True, + ) + routine_thread.start() - finally: - logger.info("Sending stop signal.") - stop_signal.value = 1 + # Wait for the stop signal before exiting. + while stop_signal.value == 0: + time.sleep(5) def _load_configuration(self, configuration_path): """Load a configuration from the path if it exists, otherwise load the default configuration. diff --git a/data_gateway/packet_reader.py b/data_gateway/packet_reader.py index 9717e394..a96faebe 100644 --- a/data_gateway/packet_reader.py +++ b/data_gateway/packet_reader.py @@ -59,12 +59,11 @@ def __init__( os.makedirs(os.path.join(output_directory, self.session_subdirectory), exist_ok=True) logger.warning("Timestamp synchronisation unavailable with current hardware; defaulting to using system clock.") - def read_packets(self, serial_port, packet_queue, error_queue, stop_signal, stop_when_no_more_data=False): + def read_packets(self, serial_port, packet_queue, stop_signal, stop_when_no_more_data=False): """Read packets from a serial port and send them to the parser thread for processing and persistence. :param serial.Serial serial_port: name of serial port to read from :param queue.Queue packet_queue: a thread-safe queue to put packets on to for the parser thread to pick up - :param queue.Queue error_queue: a thread-safe queue to put any exceptions on to for the main thread to handle :param bool stop_when_no_more_data: if `True`, stop reading when no more data is received from the port (for testing) :return None: """ @@ -106,18 +105,17 @@ def read_packets(self, serial_port, packet_queue, error_queue, stop_signal, stop packet_queue.put({"packet_type": packet_type, "packet": packet}) except Exception as e: - error_queue.put(e) logger.info("Sending stop signal.") stop_signal.value = 1 + raise e - def parse_packets(self, packet_queue, error_queue, stop_signal, timeout=3600): + def parse_packets(self, packet_queue, stop_signal, timeout=3600): """Get packets from a thread-safe packet queue, check if a full payload has been received (i.e. correct length) with the correct packet type handle, then parse the payload. After parsing/processing, upload them to Google Cloud storage and/or write them to disk. If any errors are raised, put them on the error queue for the main thread to handle. :param queue.Queue packet_queue: a thread-safe queue of packets provided by a reader thread - :param queue.Queue error_queue: a thread-safe queue to put any exceptions on to for the main thread to handle :return None: """ logger.info("Beginning parsing packets from serial port.") @@ -198,11 +196,9 @@ def parse_packets(self, packet_queue, error_queue, stop_signal, timeout=3600): pass except Exception as e: - error_queue.put(e) - - finally: logger.info("Sending stop signal.") stop_signal.value = 1 + raise e def update_handles(self, payload): """Update the Bluetooth handles object. Handles are updated every time a new Bluetooth connection is From 922754ec2c90e9b54b04dbe67b57535506d01729 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 11 Feb 2022 14:43:17 +0000 Subject: [PATCH 41/84] ENH: More regularly check for stop signal in ParserProcess --- data_gateway/data_gateway.py | 2 +- data_gateway/packet_reader.py | 24 +++++++++++------------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/data_gateway/data_gateway.py b/data_gateway/data_gateway.py index 8870561a..f1cafe2e 100644 --- a/data_gateway/data_gateway.py +++ b/data_gateway/data_gateway.py @@ -109,7 +109,6 @@ def start(self, stop_when_no_more_data=False): "serial_port": self.serial_port, "packet_queue": packet_queue, "stop_signal": stop_signal, - "stop_when_no_more_data": stop_when_no_more_data, }, daemon=True, ) @@ -120,6 +119,7 @@ def start(self, stop_when_no_more_data=False): kwargs={ "packet_queue": packet_queue, "stop_signal": stop_signal, + "stop_when_no_more_data": stop_when_no_more_data, }, daemon=True, ) diff --git a/data_gateway/packet_reader.py b/data_gateway/packet_reader.py index a96faebe..b3634d40 100644 --- a/data_gateway/packet_reader.py +++ b/data_gateway/packet_reader.py @@ -59,12 +59,11 @@ def __init__( os.makedirs(os.path.join(output_directory, self.session_subdirectory), exist_ok=True) logger.warning("Timestamp synchronisation unavailable with current hardware; defaulting to using system clock.") - def read_packets(self, serial_port, packet_queue, stop_signal, stop_when_no_more_data=False): + def read_packets(self, serial_port, packet_queue, stop_signal): """Read packets from a serial port and send them to the parser thread for processing and persistence. :param serial.Serial serial_port: name of serial port to read from :param queue.Queue packet_queue: a thread-safe queue to put packets on to for the parser thread to pick up - :param bool stop_when_no_more_data: if `True`, stop reading when no more data is received from the port (for testing) :return None: """ try: @@ -74,10 +73,6 @@ def read_packets(self, serial_port, packet_queue, stop_signal, stop_when_no_more serial_data = serial_port.read() if len(serial_data) == 0: - if stop_when_no_more_data: - logger.info("Sending stop signal.") - stop_signal.value = 1 - break continue if serial_data[0] != self.config.packet_key: @@ -109,13 +104,14 @@ def read_packets(self, serial_port, packet_queue, stop_signal, stop_when_no_more stop_signal.value = 1 raise e - def parse_packets(self, packet_queue, stop_signal, timeout=3600): + def parse_packets(self, packet_queue, stop_signal, stop_when_no_more_data=False): """Get packets from a thread-safe packet queue, check if a full payload has been received (i.e. correct length) with the correct packet type handle, then parse the payload. After parsing/processing, upload them to Google Cloud storage and/or write them to disk. If any errors are raised, put them on the error queue for the main thread to handle. :param queue.Queue packet_queue: a thread-safe queue of packets provided by a reader thread + :param bool stop_when_no_more_data: if `True`, stop reading when no more data is received (for testing) :return None: """ logger.info("Beginning parsing packets from serial port.") @@ -158,7 +154,13 @@ def parse_packets(self, packet_queue, stop_signal, timeout=3600): with self.uploader: with self.writer: while stop_signal.value == 0: - packet_type, packet = packet_queue.get(timeout=timeout).values() + try: + packet_type, packet = packet_queue.get(timeout=5).values() + except queue.Empty: + if stop_when_no_more_data: + break + continue + logger.debug("Received packet for parsing.") if packet_type not in self.handles: @@ -192,13 +194,9 @@ def parse_packets(self, packet_queue, stop_signal, timeout=3600): ]: self._parse_info_packet(self.handles[packet_type], packet) - except queue.Empty: - pass - - except Exception as e: + finally: logger.info("Sending stop signal.") stop_signal.value = 1 - raise e def update_handles(self, payload): """Update the Bluetooth handles object. Handles are updated every time a new Bluetooth connection is From f5599ab8c54143189fdf7300087dc011c354eb9b Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 11 Feb 2022 14:45:09 +0000 Subject: [PATCH 42/84] ENH: Gracefully handle KeyboardInterrupt in other processes --- data_gateway/packet_reader.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/data_gateway/packet_reader.py b/data_gateway/packet_reader.py index b3634d40..6dbb6b2f 100644 --- a/data_gateway/packet_reader.py +++ b/data_gateway/packet_reader.py @@ -99,10 +99,12 @@ def read_packets(self, serial_port, packet_queue, stop_signal): packet_queue.put({"packet_type": packet_type, "packet": packet}) - except Exception as e: + except KeyboardInterrupt: + pass + + finally: logger.info("Sending stop signal.") stop_signal.value = 1 - raise e def parse_packets(self, packet_queue, stop_signal, stop_when_no_more_data=False): """Get packets from a thread-safe packet queue, check if a full payload has been received (i.e. correct length) @@ -194,6 +196,9 @@ def parse_packets(self, packet_queue, stop_signal, stop_when_no_more_data=False) ]: self._parse_info_packet(self.handles[packet_type], packet) + except KeyboardInterrupt: + pass + finally: logger.info("Sending stop signal.") stop_signal.value = 1 From 7ca32bf5badb3dce93f83510817ad61771bddbb6 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 14 Feb 2022 11:33:55 +0000 Subject: [PATCH 43/84] FIX: Check stop signal properly in Routine --- data_gateway/routine.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data_gateway/routine.py b/data_gateway/routine.py index 3368091a..c73d0c36 100644 --- a/data_gateway/routine.py +++ b/data_gateway/routine.py @@ -1,9 +1,9 @@ -import logging +import multiprocessing import sched import time -logger = logging.getLogger(__name__) +logger = multiprocessing.get_logger() class Routine: @@ -47,7 +47,7 @@ def run(self, stop_signal): scheduler = sched.scheduler(time.perf_counter) start_time = time.perf_counter() - while not stop_signal: + while stop_signal.value == 0: cycle_start_time = time.perf_counter() for command, delay in self.commands: From a0ab181b311bb81ef7aad1a2c94af223b7f4592e Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 14 Feb 2022 11:42:22 +0000 Subject: [PATCH 44/84] TST: Remove redundant assertion in CLI test --- tests/test_cli.py | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 2c2087ee..4d822c8a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -197,26 +197,21 @@ def test_start_and_stop_in_interactive_mode(self): """Ensure the gateway can be started and stopped via the CLI in interactive mode.""" with tempfile.TemporaryDirectory() as temporary_directory: with EnvironmentVariableRemover("GOOGLE_APPLICATION_CREDENTIALS"): - with mock.patch("logging.StreamHandler.emit") as mock_local_logger_emit: - result = CliRunner().invoke( - gateway_cli, - [ - "start", - "--interactive", - "--save-locally", - "--no-upload-to-cloud", - "--use-dummy-serial-port", - f"--output-dir={temporary_directory}", - ], - input="stop\n", - ) - - self.assertIsNone(result.exception) - self.assertEqual(result.exit_code, 0) + result = CliRunner().invoke( + gateway_cli, + [ + "start", + "--interactive", + "--save-locally", + "--no-upload-to-cloud", + "--use-dummy-serial-port", + f"--output-dir={temporary_directory}", + ], + input="stop\n", + ) - self.assertTrue( - any(call_arg[0][0].msg == "Stopping gateway." for call_arg in mock_local_logger_emit.call_args_list) - ) + self.assertIsNone(result.exception) + self.assertEqual(result.exit_code, 0) def test_save_locally(self): """Ensure `--save-locally` mode writes data to disk.""" From 9a514363fff32bc8bc5a6f630c1f4b22a864d6a8 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 14 Feb 2022 12:03:57 +0000 Subject: [PATCH 45/84] ENH: Allow specific time after which to stop gateway if no data received --- data_gateway/data_gateway.py | 5 +++-- data_gateway/packet_reader.py | 13 +++++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/data_gateway/data_gateway.py b/data_gateway/data_gateway.py index f1cafe2e..e30248a2 100644 --- a/data_gateway/data_gateway.py +++ b/data_gateway/data_gateway.py @@ -88,11 +88,12 @@ def __init__( self.routine = self._load_routine(routine_path=routine_path) - def start(self, stop_when_no_more_data=False): + def start(self, stop_when_no_more_data_after=False): """Begin reading and persisting data from the serial port for the sensors at the installation defined in the configuration. In interactive mode, commands can be sent to the nodes/sensors via the serial port by typing them into `stdin` and pressing enter. These commands are: [startBaros, startMics, startIMU, getBattery, stop]. + :param float|bool stop_when_no_more_data_after: the number of seconds after receiving no data to stop the gateway (mainly for testing); if `False`, no limit is applied :return None: """ logger.info("Starting data gateway.") @@ -119,7 +120,7 @@ def start(self, stop_when_no_more_data=False): kwargs={ "packet_queue": packet_queue, "stop_signal": stop_signal, - "stop_when_no_more_data": stop_when_no_more_data, + "stop_when_no_more_data_after": stop_when_no_more_data_after, }, daemon=True, ) diff --git a/data_gateway/packet_reader.py b/data_gateway/packet_reader.py index 6dbb6b2f..191cd629 100644 --- a/data_gateway/packet_reader.py +++ b/data_gateway/packet_reader.py @@ -106,14 +106,14 @@ def read_packets(self, serial_port, packet_queue, stop_signal): logger.info("Sending stop signal.") stop_signal.value = 1 - def parse_packets(self, packet_queue, stop_signal, stop_when_no_more_data=False): + def parse_packets(self, packet_queue, stop_signal, stop_when_no_more_data_after=False): """Get packets from a thread-safe packet queue, check if a full payload has been received (i.e. correct length) with the correct packet type handle, then parse the payload. After parsing/processing, upload them to Google Cloud storage and/or write them to disk. If any errors are raised, put them on the error queue for the main thread to handle. :param queue.Queue packet_queue: a thread-safe queue of packets provided by a reader thread - :param bool stop_when_no_more_data: if `True`, stop reading when no more data is received (for testing) + :param float|bool stop_when_no_more_data_after: the number of seconds after receiving no data to stop the gateway (mainly for testing); if `False`, no limit is applied :return None: """ logger.info("Beginning parsing packets from serial port.") @@ -152,14 +152,19 @@ def parse_packets(self, packet_queue, stop_signal, stop_when_no_more_data=False) for _ in range(self.config.number_of_sensors[sensor_name]) ] + if stop_when_no_more_data_after is False: + timeout = 5 + else: + timeout = stop_when_no_more_data_after + try: with self.uploader: with self.writer: while stop_signal.value == 0: try: - packet_type, packet = packet_queue.get(timeout=5).values() + packet_type, packet = packet_queue.get(timeout=timeout).values() except queue.Empty: - if stop_when_no_more_data: + if stop_when_no_more_data_after is not False: break continue From eda11c854b639adea4818f0ac37cd6fd98e57793 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 14 Feb 2022 12:05:59 +0000 Subject: [PATCH 46/84] TST: Update tests to work with multiprocessing --- tests/test_data_gateway.py | 90 ++++++++++++++++++++------------------ 1 file changed, 47 insertions(+), 43 deletions(-) diff --git a/tests/test_data_gateway.py b/tests/test_data_gateway.py index eee20493..15e361ba 100644 --- a/tests/test_data_gateway.py +++ b/tests/test_data_gateway.py @@ -47,7 +47,7 @@ def test_error_is_logged_if_unknown_sensor_type_packet_is_received(self): bucket_name=TEST_BUCKET_NAME, ) with self.assertLogs() as logging_context: - data_gateway.start(stop_when_no_more_data=True) + data_gateway.start(stop_when_no_more_data_after=0.1) self.assertIn("Received packet with unknown type: 0", logging_context.output[1]) @@ -69,7 +69,7 @@ def test_configuration_file_is_persisted(self): bucket_name=TEST_BUCKET_NAME, ) - data_gateway.start(stop_when_no_more_data=True) + data_gateway.start(stop_when_no_more_data_after=0.1) configuration_path = os.path.join( temporary_directory, data_gateway.packet_reader.session_subdirectory, "configuration.json" @@ -116,7 +116,7 @@ def test_update_handles_fails_if_start_and_end_handles_are_incorrect(self): ) with self.assertLogs() as logging_context: - data_gateway.start(stop_when_no_more_data=True) + data_gateway.start(stop_when_no_more_data_after=0.1) self.assertIn("Handle error", logging_context.output[1]) def test_update_handles(self): @@ -144,7 +144,7 @@ def test_update_handles(self): ) with self.assertLogs() as logging_context: - data_gateway.start(stop_when_no_more_data=True) + data_gateway.start(stop_when_no_more_data_after=0.1) self.assertIn("Successfully updated handles", logging_context.output[1]) def test_data_gateway_with_baros_p_sensor(self): @@ -165,11 +165,13 @@ def test_data_gateway_with_baros_p_sensor(self): bucket_name=TEST_BUCKET_NAME, ) - data_gateway.start(stop_when_no_more_data=False) + data_gateway.start(stop_when_no_more_data_after=0.1) self._check_data_is_written_to_files(temporary_directory, sensor_names=["Baros_P"]) self._check_windows_are_uploaded_to_cloud( - temporary_directory, sensor_names=["Baros_P"], number_of_windows_to_check=1 + temporary_directory, + sensor_names=["Baros_P"], + number_of_windows_to_check=1, ) def test_data_gateway_with_baros_t_sensor(self): @@ -189,13 +191,13 @@ def test_data_gateway_with_baros_t_sensor(self): project_name=TEST_PROJECT_NAME, bucket_name=TEST_BUCKET_NAME, ) - data_gateway.start(stop_when_no_more_data=True) - self._check_data_is_written_to_files( - data_gateway.packet_reader, temporary_directory, sensor_names=["Baros_T"] - ) + data_gateway.start(stop_when_no_more_data_after=0.1) + self._check_data_is_written_to_files(temporary_directory, sensor_names=["Baros_T"]) self._check_windows_are_uploaded_to_cloud( - data_gateway.packet_reader, sensor_names=["Baros_T"], number_of_windows_to_check=1 + temporary_directory, + sensor_names=["Baros_T"], + number_of_windows_to_check=1, ) def test_data_gateway_with_diff_baros_sensor(self): @@ -215,13 +217,11 @@ def test_data_gateway_with_diff_baros_sensor(self): project_name=TEST_PROJECT_NAME, bucket_name=TEST_BUCKET_NAME, ) - data_gateway.start(stop_when_no_more_data=True) - self._check_data_is_written_to_files( - data_gateway.packet_reader, temporary_directory, sensor_names=["Diff_Baros"] - ) + data_gateway.start(stop_when_no_more_data_after=0.1) + self._check_data_is_written_to_files(temporary_directory, sensor_names=["Diff_Baros"]) self._check_windows_are_uploaded_to_cloud( - data_gateway.packet_reader, + temporary_directory, sensor_names=["Diff_Baros"], number_of_windows_to_check=1, ) @@ -243,11 +243,13 @@ def test_data_gateway_with_mic_sensor(self): project_name=TEST_PROJECT_NAME, bucket_name=TEST_BUCKET_NAME, ) - data_gateway.start(stop_when_no_more_data=True) - self._check_data_is_written_to_files(data_gateway.packet_reader, temporary_directory, sensor_names=["Mics"]) + data_gateway.start(stop_when_no_more_data_after=0.1) + self._check_data_is_written_to_files(temporary_directory, sensor_names=["Mics"]) self._check_windows_are_uploaded_to_cloud( - data_gateway.packet_reader, sensor_names=["Mics"], number_of_windows_to_check=1 + temporary_directory, + sensor_names=["Mics"], + number_of_windows_to_check=1, ) def test_data_gateway_with_acc_sensor(self): @@ -267,11 +269,13 @@ def test_data_gateway_with_acc_sensor(self): project_name=TEST_PROJECT_NAME, bucket_name=TEST_BUCKET_NAME, ) - data_gateway.start(stop_when_no_more_data=True) - self._check_data_is_written_to_files(data_gateway.packet_reader, temporary_directory, sensor_names=["Acc"]) + data_gateway.start(stop_when_no_more_data_after=0.1) + self._check_data_is_written_to_files(temporary_directory, sensor_names=["Acc"]) self._check_windows_are_uploaded_to_cloud( - data_gateway.packet_reader, sensor_names=["Acc"], number_of_windows_to_check=1 + temporary_directory, + sensor_names=["Acc"], + number_of_windows_to_check=1, ) def test_data_gateway_with_gyro_sensor(self): @@ -291,11 +295,13 @@ def test_data_gateway_with_gyro_sensor(self): project_name=TEST_PROJECT_NAME, bucket_name=TEST_BUCKET_NAME, ) - data_gateway.start(stop_when_no_more_data=True) - self._check_data_is_written_to_files(data_gateway.packet_reader, temporary_directory, sensor_names=["Gyro"]) + data_gateway.start(stop_when_no_more_data_after=0.1) + self._check_data_is_written_to_files(temporary_directory, sensor_names=["Gyro"]) self._check_windows_are_uploaded_to_cloud( - data_gateway.packet_reader, sensor_names=["Gyro"], number_of_windows_to_check=1 + temporary_directory, + sensor_names=["Gyro"], + number_of_windows_to_check=1, ) def test_data_gateway_with_mag_sensor(self): @@ -315,11 +321,13 @@ def test_data_gateway_with_mag_sensor(self): project_name=TEST_PROJECT_NAME, bucket_name=TEST_BUCKET_NAME, ) - data_gateway.start(stop_when_no_more_data=True) - self._check_data_is_written_to_files(data_gateway.packet_reader, temporary_directory, sensor_names=["Mag"]) + data_gateway.start(stop_when_no_more_data_after=0.1) + self._check_data_is_written_to_files(temporary_directory, sensor_names=["Mag"]) self._check_windows_are_uploaded_to_cloud( - data_gateway.packet_reader, sensor_names=["Mag"], number_of_windows_to_check=1 + temporary_directory, + sensor_names=["Mag"], + number_of_windows_to_check=1, ) def test_data_gateway_with_connections_statistics(self): @@ -339,14 +347,14 @@ def test_data_gateway_with_connections_statistics(self): project_name=TEST_PROJECT_NAME, bucket_name=TEST_BUCKET_NAME, ) - data_gateway.start(stop_when_no_more_data=True) + data_gateway.start(stop_when_no_more_data_after=0.1) - self._check_data_is_written_to_files( - data_gateway.packet_reader, temporary_directory, sensor_names=["Constat"] - ) + self._check_data_is_written_to_files(temporary_directory, sensor_names=["Constat"]) self._check_windows_are_uploaded_to_cloud( - data_gateway.packet_reader, sensor_names=["Constat"], number_of_windows_to_check=1 + temporary_directory, + sensor_names=["Constat"], + number_of_windows_to_check=1, ) def test_data_gateway_with_connections_statistics_in_sleep_mode(self): @@ -375,11 +383,9 @@ def test_data_gateway_with_connections_statistics_in_sleep_mode(self): ) with patch("data_gateway.packet_reader.logger") as mock_logger: - data_gateway.start(stop_when_no_more_data=True) + data_gateway.start(stop_when_no_more_data_after=0.1) - self._check_data_is_written_to_files( - data_gateway.packet_reader, temporary_directory, sensor_names=["Constat"] - ) + self._check_data_is_written_to_files(temporary_directory, sensor_names=["Constat"]) self.assertEqual(0, mock_logger.warning.call_count) def test_all_sensors_together(self): @@ -401,14 +407,12 @@ def test_all_sensors_together(self): project_name=TEST_PROJECT_NAME, bucket_name=TEST_BUCKET_NAME, ) - data_gateway.start(stop_when_no_more_data=True) + data_gateway.start(stop_when_no_more_data_after=0.1) - self._check_data_is_written_to_files( - data_gateway.packet_reader, temporary_directory, sensor_names=sensor_names - ) + self._check_data_is_written_to_files(temporary_directory, sensor_names=sensor_names) self._check_windows_are_uploaded_to_cloud( - data_gateway.packet_reader, + temporary_directory, sensor_names=sensor_names, number_of_windows_to_check=1, ) @@ -442,7 +446,7 @@ def test_data_gateway_with_info_packets(self): ) with self.assertLogs() as logging_context: - data_gateway.start(stop_when_no_more_data=True) + data_gateway.start(stop_when_no_more_data_after=0.1) log_messages_combined = "\n".join(logging_context.output) From a1f639a2076724386681f14641fd60e39a00a341 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 14 Feb 2022 14:19:47 +0000 Subject: [PATCH 47/84] FIX: Ensure PacketReader has correct default output directory --- data_gateway/packet_reader.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/data_gateway/packet_reader.py b/data_gateway/packet_reader.py index 191cd629..d1655d69 100644 --- a/data_gateway/packet_reader.py +++ b/data_gateway/packet_reader.py @@ -9,7 +9,12 @@ from data_gateway import MICROPHONE_SENSOR_NAME, exceptions from data_gateway.configuration import Configuration -from data_gateway.persistence import BatchingFileWriter, BatchingUploader, NoOperationContextManager +from data_gateway.persistence import ( + DEFAULT_OUTPUT_DIRECTORY, + BatchingFileWriter, + BatchingUploader, + NoOperationContextManager, +) logger = multiprocessing.get_logger() @@ -33,7 +38,7 @@ def __init__( self, save_locally, upload_to_cloud, - output_directory=None, + output_directory=DEFAULT_OUTPUT_DIRECTORY, window_size=600, project_name=None, bucket_name=None, From 6ca0c3e7c57a06e7c0f6693bdde641f5eadc6f77 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 14 Feb 2022 14:20:35 +0000 Subject: [PATCH 48/84] TST: Factor out PacketReader tests relying on log messages --- tests/test_data_gateway.py | 77 ------------------------------------- tests/test_packet_reader.py | 49 +++++++++++++++++++++++ 2 files changed, 49 insertions(+), 77 deletions(-) create mode 100644 tests/test_packet_reader.py diff --git a/tests/test_data_gateway.py b/tests/test_data_gateway.py index 15e361ba..20d2fdcb 100644 --- a/tests/test_data_gateway.py +++ b/tests/test_data_gateway.py @@ -29,28 +29,6 @@ def setUpClass(cls): cls.WINDOW_SIZE = 10 cls.storage_client = GoogleCloudStorageClient(project_name=TEST_PROJECT_NAME) - def test_error_is_logged_if_unknown_sensor_type_packet_is_received(self): - """Test that an error is logged if an unknown sensor type packet is received.""" - serial_port = DummySerial(port="test") - packet_type = bytes([0]) - - serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[0]))) - serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[1]))) - - with tempfile.TemporaryDirectory() as temporary_directory: - data_gateway = DataGateway( - serial_port=serial_port, - save_locally=True, - output_directory=temporary_directory, - window_size=self.WINDOW_SIZE, - project_name=TEST_PROJECT_NAME, - bucket_name=TEST_BUCKET_NAME, - ) - with self.assertLogs() as logging_context: - data_gateway.start(stop_when_no_more_data_after=0.1) - - self.assertIn("Received packet with unknown type: 0", logging_context.output[1]) - def test_configuration_file_is_persisted(self): """Test that the configuration file is persisted.""" serial_port = DummySerial(port="test") @@ -92,61 +70,6 @@ def test_configuration_file_is_persisted(self): # Test configuration is valid. Configuration.from_dict(json.loads(configuration)) - def test_update_handles_fails_if_start_and_end_handles_are_incorrect(self): - """Test that an error is raised if the start and end handles are incorrect when trying to update handles.""" - serial_port = DummySerial(port="test") - - # Set packet type to handles update packet. - packet_type = bytes([255]) - - # Set first two bytes of payload to incorrect range for updating handles. - payload = bytearray(RANDOM_BYTES[0]) - payload[0:1] = int(0).to_bytes(1, "little") - payload[2:3] = int(255).to_bytes(1, "little") - serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, payload))) - - with tempfile.TemporaryDirectory() as temporary_directory: - data_gateway = DataGateway( - serial_port, - save_locally=True, - output_directory=temporary_directory, - window_size=self.WINDOW_SIZE, - project_name=TEST_PROJECT_NAME, - bucket_name=TEST_BUCKET_NAME, - ) - - with self.assertLogs() as logging_context: - data_gateway.start(stop_when_no_more_data_after=0.1) - self.assertIn("Handle error", logging_context.output[1]) - - def test_update_handles(self): - """Test that the handles can be updated.""" - serial_port = DummySerial(port="test") - - # Set packet type to handles update packet. - packet_type = bytes([255]) - - # Set first two bytes of payload to correct range for updating handles. - payload = bytearray(RANDOM_BYTES[0]) - payload[0:1] = int(0).to_bytes(1, "little") - payload[2:3] = int(26).to_bytes(1, "little") - serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, payload))) - - with tempfile.TemporaryDirectory() as temporary_directory: - data_gateway = DataGateway( - serial_port, - save_locally=True, - upload_to_cloud=False, - output_directory=temporary_directory, - window_size=self.WINDOW_SIZE, - project_name=TEST_PROJECT_NAME, - bucket_name=TEST_BUCKET_NAME, - ) - - with self.assertLogs() as logging_context: - data_gateway.start(stop_when_no_more_data_after=0.1) - self.assertIn("Successfully updated handles", logging_context.output[1]) - def test_data_gateway_with_baros_p_sensor(self): """Test that the packet reader works with the Baro_P sensor.""" serial_port = DummySerial(port="test") diff --git a/tests/test_packet_reader.py b/tests/test_packet_reader.py new file mode 100644 index 00000000..a0157c15 --- /dev/null +++ b/tests/test_packet_reader.py @@ -0,0 +1,49 @@ +import multiprocessing +from unittest.mock import patch + +from data_gateway.packet_reader import PacketReader +from tests import LENGTH, PACKET_KEY, RANDOM_BYTES +from tests.base import BaseTestCase + + +class TestPacketReader(BaseTestCase): + def test_error_is_logged_if_unknown_sensor_type_packet_is_received(self): + """Test that an error is logged if an unknown sensor type packet is received.""" + queue = multiprocessing.Queue() + queue.put({"packet_type": bytes([0]), "packet": b"".join((PACKET_KEY, bytes([0]), LENGTH, RANDOM_BYTES[0]))}) + + packet_reader = PacketReader(save_locally=False, upload_to_cloud=False) + + with patch("data_gateway.packet_reader.logger") as mock_logger: + packet_reader.parse_packets( + packet_queue=queue, + stop_signal=multiprocessing.Value("i", 0), + stop_when_no_more_data_after=0.1, + ) + + self.assertIn("Received packet with unknown type: ", mock_logger.method_calls[2].args[0]) + + def test_update_handles_fails_if_start_and_end_handles_are_incorrect(self): + """Test that an error is raised if the start and end handles are incorrect when trying to update handles.""" + packet = bytearray(RANDOM_BYTES[0]) + packet[0:1] = int(0).to_bytes(1, "little") + packet[2:3] = int(255).to_bytes(1, "little") + + packet_reader = PacketReader(save_locally=False, upload_to_cloud=False) + + with patch("data_gateway.packet_reader.logger") as mock_logger: + packet_reader.update_handles(packet) + + self.assertIn("Handle error", mock_logger.method_calls[0].args[0]) + + def test_update_handles(self): + """Test that the handles can be updated.""" + packet = bytearray(RANDOM_BYTES[0]) + packet[0:1] = int(0).to_bytes(1, "little") + packet[2:3] = int(26).to_bytes(1, "little") + packet_reader = PacketReader(save_locally=False, upload_to_cloud=False) + + with patch("data_gateway.packet_reader.logger") as mock_logger: + packet_reader.update_handles(packet) + + self.assertIn("Successfully updated handles", mock_logger.method_calls[0].args[0]) From e1e63044c4e73c7bd6f69a15218717bba3385331 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 14 Feb 2022 14:54:51 +0000 Subject: [PATCH 49/84] REF: Simplify output directory in persistence module --- data_gateway/packet_reader.py | 7 ++-- data_gateway/persistence.py | 54 +++++++----------------- tests/test_persistence.py | 77 ++++++++++++----------------------- 3 files changed, 45 insertions(+), 93 deletions(-) diff --git a/data_gateway/packet_reader.py b/data_gateway/packet_reader.py index d1655d69..4814e909 100644 --- a/data_gateway/packet_reader.py +++ b/data_gateway/packet_reader.py @@ -60,8 +60,9 @@ def __init__( self.sleep = False self.sensor_time_offset = None self.session_subdirectory = str(hash(datetime.datetime.now()))[1:7] + self.full_output_directory = os.path.join(self.output_directory, self.session_subdirectory) - os.makedirs(os.path.join(output_directory, self.session_subdirectory), exist_ok=True) + os.makedirs(self.session_subdirectory, exist_ok=True) logger.warning("Timestamp synchronisation unavailable with current hardware; defaulting to using system clock.") def read_packets(self, serial_port, packet_queue, stop_signal): @@ -130,7 +131,7 @@ def parse_packets(self, packet_queue, stop_signal, stop_when_no_more_data_after= bucket_name=self.bucket_name, window_size=self.window_size, session_subdirectory=self.session_subdirectory, - output_directory=self.output_directory, + output_directory=self.full_output_directory, metadata={"data_gateway__configuration": self.config.to_dict()}, ) else: @@ -141,7 +142,7 @@ def parse_packets(self, packet_queue, stop_signal, stop_when_no_more_data_after= sensor_names=self.config.sensor_names, window_size=self.window_size, session_subdirectory=self.session_subdirectory, - output_directory=self.output_directory, + output_directory=self.full_output_directory, save_csv_files=self.save_csv_files, ) else: diff --git a/data_gateway/persistence.py b/data_gateway/persistence.py index 4339b3d8..f9991bbc 100644 --- a/data_gateway/persistence.py +++ b/data_gateway/persistence.py @@ -43,26 +43,17 @@ class TimeBatcher: :param iter(str) sensor_names: names of sensors to group data for :param float window_size: length of time window in seconds - :param str session_subdirectory: directory within output directory to persist into :param str output_directory: directory to write windows to :return None: """ _file_prefix = "window" - def __init__( - self, - sensor_names, - window_size, - session_subdirectory, - output_directory=DEFAULT_OUTPUT_DIRECTORY, - ): + def __init__(self, sensor_names, window_size, output_directory=DEFAULT_OUTPUT_DIRECTORY): self.current_window = {"sensor_time_offset": None, "sensor_data": {name: [] for name in sensor_names}} self.window_size = window_size self.output_directory = output_directory self.ready_window = {"sensor_time_offset": None, "sensor_data": {}} - self._session_subdirectory = session_subdirectory - self._full_output_path = None self._start_time = time.perf_counter() self._window_number = 0 @@ -140,7 +131,6 @@ class BatchingFileWriter(TimeBatcher): :param iter(str) sensor_names: names of sensors to make windows for :param float window_size: length of time window in seconds - :param str session_subdirectory: directory within output directory to persist into :param str output_directory: directory to write windows to :param int storage_limit: storage limit in bytes (default is 1 GB) :return None: @@ -150,17 +140,15 @@ def __init__( self, sensor_names, window_size, - session_subdirectory, save_csv_files=False, output_directory=DEFAULT_OUTPUT_DIRECTORY, storage_limit=1024 ** 3, ): self._save_csv_files = save_csv_files self.storage_limit = storage_limit - super().__init__(sensor_names, window_size, session_subdirectory, output_directory) - self._full_output_path = os.path.join(self.output_directory, self._session_subdirectory) - os.makedirs(os.path.join(self.output_directory, self._session_subdirectory), exist_ok=True) - logger.info(f"Windows will be saved to {self._full_output_path!r} at intervals of {self.window_size} seconds.") + super().__init__(sensor_names, window_size, output_directory) + os.makedirs(self.output_directory, exist_ok=True) + logger.info(f"Windows will be saved to {self.output_directory!r} at intervals of {self.window_size} seconds.") def _persist_window(self, window=None): """Write a window of serialised data to disk, deleting the oldest window first if the storage limit has been @@ -193,13 +181,11 @@ def _manage_storage(self): :return None: """ - session_directory = os.path.join(self.output_directory, self._session_subdirectory) - filter = lambda path: os.path.split(path)[-1].startswith("window") # noqa storage_limit_in_mb = self.storage_limit / 1024 ** 2 - if calculate_disk_usage(session_directory, filter) >= self.storage_limit: - oldest_window = get_oldest_file_in_directory(session_directory, filter) + if calculate_disk_usage(self.output_directory, filter) >= self.storage_limit: + oldest_window = get_oldest_file_in_directory(self.output_directory, filter) logger.warning( "Storage limit reached (%s MB) - deleting oldest window (%r).", @@ -209,7 +195,7 @@ def _manage_storage(self): os.remove(oldest_window) - elif calculate_disk_usage(session_directory, filter) >= 0.9 * self.storage_limit: + elif calculate_disk_usage(self.output_directory, filter) >= 0.9 * self.storage_limit: logger.warning("90% of storage limit reached - %s MB remaining.", 0.1 * storage_limit_in_mb) def _generate_window_path(self): @@ -218,7 +204,7 @@ def _generate_window_path(self): :return str: """ filename = f"{self._file_prefix}-{self._window_number}.json" - return os.path.join(self.output_directory, self._session_subdirectory, filename) + return os.path.join(self.output_directory, filename) class BatchingUploader(TimeBatcher): @@ -230,7 +216,6 @@ class BatchingUploader(TimeBatcher): :param str project_name: name of Google Cloud project to upload to :param str bucket_name: name of Google Cloud bucket to upload to :param float window_size: length of time window in seconds - :param str session_subdirectory: directory within output directory to persist into :param str output_directory: directory to write windows to :param float upload_timeout: time after which to give up trying to upload to the cloud :param bool upload_backup_files: attempt to upload backed-up windows on next window upload @@ -243,7 +228,6 @@ def __init__( project_name, bucket_name, window_size, - session_subdirectory, output_directory=DEFAULT_OUTPUT_DIRECTORY, metadata=None, upload_timeout=60, @@ -255,16 +239,13 @@ def __init__( self.metadata = metadata or {} self.upload_timeout = upload_timeout self.upload_backup_files = upload_backup_files - super().__init__(sensor_names, window_size, session_subdirectory, output_directory) - self._full_output_path = storage.path.join(self.output_directory, self._session_subdirectory) + super().__init__(sensor_names, window_size, output_directory) self._backup_directory = os.path.join(self.output_directory, ".backup") - self._backup_writer = BatchingFileWriter( - sensor_names, window_size, session_subdirectory, output_directory=self._backup_directory - ) + self._backup_writer = BatchingFileWriter(sensor_names, window_size, output_directory=self._backup_directory) logger.info( - f"Windows will be uploaded to {self._full_output_path!r} at intervals of {self.window_size} seconds." + f"Windows will be uploaded to {self.output_directory!r} at intervals of {self.window_size} seconds." ) def _persist_window(self): @@ -303,25 +284,20 @@ def _generate_window_path(self): :return str: """ filename = f"{self._file_prefix}-{self._window_number}.json" - return storage.path.join(self.output_directory, self._session_subdirectory, filename) + return storage.path.join(self.output_directory, filename) def _attempt_to_upload_backup_files(self): """Check for backup files and attempt to upload them to cloud storage again. :return None: """ - backup_filenames = os.listdir(os.path.join(self._backup_directory, self._session_subdirectory)) - - if not backup_filenames: - return - - for filename in backup_filenames: + for filename in os.listdir(self._backup_directory): if not filename.startswith(self._file_prefix): continue - local_path = os.path.join(self._backup_directory, self._session_subdirectory, filename) - path_in_bucket = storage.path.join(self.output_directory, self._session_subdirectory, filename) + local_path = os.path.join(self._backup_directory, filename) + path_in_bucket = storage.path.join(self.output_directory, filename) try: self.client.upload_file( diff --git a/tests/test_persistence.py b/tests/test_persistence.py index cddd1efd..057cfc0c 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -25,8 +25,7 @@ def test_data_is_batched(self): with tempfile.TemporaryDirectory() as temporary_directory: writer = BatchingFileWriter( sensor_names=["test"], - session_subdirectory="this-session", - output_directory=temporary_directory, + output_directory=os.path.join(temporary_directory, "this-session"), window_size=600, ) @@ -38,8 +37,7 @@ def test_data_is_written_to_disk_in_windows(self): with tempfile.TemporaryDirectory() as temporary_directory: writer = BatchingFileWriter( sensor_names=["test"], - session_subdirectory="this-session", - output_directory=temporary_directory, + output_directory=os.path.join(temporary_directory, "this-session"), window_size=0.01, ) @@ -55,19 +53,19 @@ def test_data_is_written_to_disk_in_windows(self): self.assertEqual(len(writer.current_window["sensor_data"]["test"]), 0) - with open(os.path.join(temporary_directory, writer._session_subdirectory, "window-0.json")) as f: + with open(os.path.join(writer.output_directory, "window-0.json")) as f: self.assertEqual(json.load(f)["sensor_data"], {"test": ["ping", "pong"]}) - with open(os.path.join(temporary_directory, writer._session_subdirectory, "window-1.json")) as f: + with open(os.path.join(writer.output_directory, "window-1.json")) as f: self.assertEqual(json.load(f)["sensor_data"], {"test": ["ding", "dong"]}) def test_oldest_window_is_deleted_when_storage_limit_reached(self): """Check that (only) the oldest window is deleted when the storage limit is reached.""" with tempfile.TemporaryDirectory() as temporary_directory: + writer = BatchingFileWriter( sensor_names=["test"], - session_subdirectory="this-session", - output_directory=temporary_directory, + output_directory=os.path.join(temporary_directory, "this-session"), window_size=0.01, storage_limit=1, ) @@ -75,7 +73,7 @@ def test_oldest_window_is_deleted_when_storage_limit_reached(self): with writer: writer.add_to_current_window(sensor_name="test", data="ping,") - first_window_path = os.path.join(temporary_directory, writer._session_subdirectory, "window-0.json") + first_window_path = os.path.join(writer.output_directory, "window-0.json") # Check first file is written to disk. self.assertTrue(os.path.exists(first_window_path)) @@ -87,17 +85,15 @@ def test_oldest_window_is_deleted_when_storage_limit_reached(self): self.assertFalse(os.path.exists(first_window_path)) # Check the second file has not been deleted. - self.assertTrue( - os.path.exists(os.path.join(temporary_directory, writer._session_subdirectory, "window-1.json")) - ) + self.assertTrue(os.path.exists(os.path.join(writer.output_directory, "window-1.json"))) def test_that_csv_files_are_written(self): """Test that data is written to disk as CSV-files if the `save_csv_files` option is `True`.""" with tempfile.TemporaryDirectory() as temporary_directory: + writer = BatchingFileWriter( sensor_names=["sensor1", "sensor2"], - session_subdirectory="this-session", - output_directory=temporary_directory, + output_directory=os.path.join(temporary_directory, "this-session"), save_csv_files=True, window_size=0.01, ) @@ -108,11 +104,11 @@ def test_that_csv_files_are_written(self): writer.add_to_current_window(sensor_name="sensor1", data=[4, 5, 6]) writer.add_to_current_window(sensor_name="sensor2", data=[4, 5, 6]) - with open(os.path.join(temporary_directory, writer._session_subdirectory, "sensor1.csv")) as f: + with open(os.path.join(writer.output_directory, "sensor1.csv")) as f: reader = csv.reader(f) self.assertEqual([row for row in reader], [["1", "2", "3"], ["4", "5", "6"]]) - with open(os.path.join(temporary_directory, writer._session_subdirectory, "sensor2.csv")) as f: + with open(os.path.join(writer.output_directory, "sensor2.csv")) as f: reader = csv.reader(f) self.assertEqual([row for row in reader], [["1", "2", "3"], ["4", "5", "6"]]) @@ -133,8 +129,7 @@ def test_data_is_batched(self): project_name=TEST_PROJECT_NAME, bucket_name=TEST_BUCKET_NAME, window_size=600, - session_subdirectory="this-session", - output_directory=tempfile.TemporaryDirectory().name, + output_directory=storage.path.join(tempfile.TemporaryDirectory().name, "this-session"), ) uploader.add_to_current_window(sensor_name="test", data="blah,") @@ -147,8 +142,7 @@ def test_data_is_uploaded_in_windows_and_can_be_retrieved_from_cloud_storage(sel project_name=TEST_PROJECT_NAME, bucket_name=TEST_BUCKET_NAME, window_size=0.01, - session_subdirectory="this-session", - output_directory=tempfile.TemporaryDirectory().name, + output_directory=storage.path.join(tempfile.TemporaryDirectory().name, "this-session"), ) with uploader: @@ -170,9 +164,7 @@ def test_data_is_uploaded_in_windows_and_can_be_retrieved_from_cloud_storage(sel json.loads( self.storage_client.download_as_string( bucket_name=TEST_BUCKET_NAME, - path_in_bucket=storage.path.join( - uploader.output_directory, uploader._session_subdirectory, "window-0.json" - ), + path_in_bucket=storage.path.join(uploader.output_directory, "window-0.json"), ) )["sensor_data"], {"test": ["ping", "pong"]}, @@ -182,9 +174,7 @@ def test_data_is_uploaded_in_windows_and_can_be_retrieved_from_cloud_storage(sel json.loads( self.storage_client.download_as_string( bucket_name=TEST_BUCKET_NAME, - path_in_bucket=storage.path.join( - uploader.output_directory, uploader._session_subdirectory, "window-1.json" - ), + path_in_bucket=storage.path.join(uploader.output_directory, "window-1.json"), ) )["sensor_data"], {"test": ["ding", "dong"]}, @@ -204,8 +194,7 @@ def test_window_is_written_to_disk_if_upload_fails(self): project_name=TEST_PROJECT_NAME, bucket_name=TEST_BUCKET_NAME, window_size=0.01, - session_subdirectory="this-session", - output_directory=temporary_directory, + output_directory=storage.path.join(temporary_directory, "this-session"), upload_backup_files=False, ) @@ -217,15 +206,11 @@ def test_window_is_written_to_disk_if_upload_fails(self): with self.assertRaises(google.api_core.exceptions.NotFound): self.storage_client.download_as_string( bucket_name=TEST_BUCKET_NAME, - path_in_bucket=storage.path.join( - uploader.output_directory, uploader._session_subdirectory, "window-0.json" - ), + path_in_bucket=storage.path.join(uploader.output_directory, "window-0.json"), ) # Check that a backup file has been written. - with open( - os.path.join(temporary_directory, ".backup", uploader._session_subdirectory, "window-0.json") - ) as f: + with open(os.path.join(uploader.output_directory, ".backup", "window-0.json")) as f: self.assertEqual(json.load(f)["sensor_data"], {"test": ["ping", "pong"]}) def test_backup_files_are_uploaded_on_next_upload_attempt(self): @@ -242,8 +227,7 @@ def test_backup_files_are_uploaded_on_next_upload_attempt(self): project_name=TEST_PROJECT_NAME, bucket_name=TEST_BUCKET_NAME, window_size=10, - session_subdirectory="this-session", - output_directory=temporary_directory, + output_directory=storage.path.join(temporary_directory, "this-session"), upload_backup_files=True, ) @@ -255,12 +239,10 @@ def test_backup_files_are_uploaded_on_next_upload_attempt(self): with self.assertRaises(google.api_core.exceptions.NotFound): self.storage_client.download_as_string( bucket_name=TEST_BUCKET_NAME, - path_in_bucket=storage.path.join( - uploader.output_directory, uploader._session_subdirectory, "window-0.json" - ), + path_in_bucket=storage.path.join(uploader.output_directory, "window-0.json"), ) - backup_path = os.path.join(temporary_directory, ".backup", uploader._session_subdirectory, "window-0.json") + backup_path = os.path.join(uploader._backup_directory, "window-0.json") # Check that a backup file has been written. with open(backup_path) as f: @@ -274,9 +256,7 @@ def test_backup_files_are_uploaded_on_next_upload_attempt(self): json.loads( self.storage_client.download_as_string( bucket_name=TEST_BUCKET_NAME, - path_in_bucket=storage.path.join( - uploader.output_directory, uploader._session_subdirectory, "window-0.json" - ), + path_in_bucket=storage.path.join(uploader.output_directory, "window-0.json"), ) )["sensor_data"], {"test": ["ping", "pong"]}, @@ -286,9 +266,7 @@ def test_backup_files_are_uploaded_on_next_upload_attempt(self): json.loads( self.storage_client.download_as_string( bucket_name=TEST_BUCKET_NAME, - path_in_bucket=storage.path.join( - uploader.output_directory, uploader._session_subdirectory, "window-1.json" - ), + path_in_bucket=storage.path.join(uploader.output_directory, "window-1.json"), ) )["sensor_data"], {"test": [["ding", "dong"]]}, @@ -304,8 +282,7 @@ def test_metadata_is_added_to_uploaded_files(self): project_name=TEST_PROJECT_NAME, bucket_name=TEST_BUCKET_NAME, window_size=0.01, - session_subdirectory="this-session", - output_directory=tempfile.TemporaryDirectory().name, + output_directory=storage.path.join(tempfile.TemporaryDirectory().name, "this-session"), metadata={"big": "rock"}, ) @@ -314,9 +291,7 @@ def test_metadata_is_added_to_uploaded_files(self): metadata = self.storage_client.get_metadata( bucket_name=TEST_BUCKET_NAME, - path_in_bucket=storage.path.join( - uploader.output_directory, uploader._session_subdirectory, "window-0.json" - ), + path_in_bucket=storage.path.join(uploader.output_directory, "window-0.json"), ) self.assertEqual(metadata["custom_metadata"], {"big": "rock"}) From d7e882eb011ea7d59cdf5586c0f26180d04099e4 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 14 Feb 2022 15:03:16 +0000 Subject: [PATCH 50/84] REF: Simplify output directory in PacketReader --- data_gateway/packet_reader.py | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/data_gateway/packet_reader.py b/data_gateway/packet_reader.py index 4814e909..0ac80d6b 100644 --- a/data_gateway/packet_reader.py +++ b/data_gateway/packet_reader.py @@ -47,7 +47,8 @@ def __init__( ): self.save_locally = save_locally self.upload_to_cloud = upload_to_cloud - self.output_directory = output_directory + self.session_subdirectory = str(hash(datetime.datetime.now()))[1:7] + self.output_directory = os.path.join(output_directory, self.session_subdirectory) self.window_size = window_size self.project_name = project_name self.bucket_name = bucket_name @@ -59,8 +60,6 @@ def __init__( self.handles = self.config.default_handles self.sleep = False self.sensor_time_offset = None - self.session_subdirectory = str(hash(datetime.datetime.now()))[1:7] - self.full_output_directory = os.path.join(self.output_directory, self.session_subdirectory) os.makedirs(self.session_subdirectory, exist_ok=True) logger.warning("Timestamp synchronisation unavailable with current hardware; defaulting to using system clock.") @@ -130,8 +129,7 @@ def parse_packets(self, packet_queue, stop_signal, stop_when_no_more_data_after= project_name=self.project_name, bucket_name=self.bucket_name, window_size=self.window_size, - session_subdirectory=self.session_subdirectory, - output_directory=self.full_output_directory, + output_directory=self.output_directory, metadata={"data_gateway__configuration": self.config.to_dict()}, ) else: @@ -141,8 +139,7 @@ def parse_packets(self, packet_queue, stop_signal, stop_when_no_more_data_after= self.writer = BatchingFileWriter( sensor_names=self.config.sensor_names, window_size=self.window_size, - session_subdirectory=self.session_subdirectory, - output_directory=self.full_output_directory, + output_directory=self.output_directory, save_csv_files=self.save_csv_files, ) else: @@ -254,19 +251,14 @@ def persist_configuration(self): configuration_dictionary = self.config.to_dict() if self.save_locally: - with open( - os.path.abspath(os.path.join(self.output_directory, self.session_subdirectory, "configuration.json")), - "w", - ) as f: + with open(os.path.abspath(os.path.join(self.output_directory, "configuration.json")), "w") as f: json.dump(configuration_dictionary, f) if self.upload_to_cloud: self.uploader.client.upload_from_string( string=json.dumps(configuration_dictionary), bucket_name=self.uploader.bucket_name, - path_in_bucket=storage.path.join( - self.output_directory, self.session_subdirectory, "configuration.json" - ), + path_in_bucket=storage.path.join(self.output_directory, "configuration.json"), ) def _parse_sensor_packet_data(self, packet_type, payload, data): From 13d69a6db651a100abe43277fab8516315de6db3 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 14 Feb 2022 15:16:55 +0000 Subject: [PATCH 51/84] FIX: Ensure configuration file is persisted --- data_gateway/data_gateway.py | 3 --- data_gateway/packet_reader.py | 6 ++++-- tests/test_data_gateway.py | 16 ++++------------ 3 files changed, 8 insertions(+), 17 deletions(-) diff --git a/data_gateway/data_gateway.py b/data_gateway/data_gateway.py index e30248a2..d3eee09f 100644 --- a/data_gateway/data_gateway.py +++ b/data_gateway/data_gateway.py @@ -97,9 +97,6 @@ def start(self, stop_when_no_more_data_after=False): :return None: """ logger.info("Starting data gateway.") - - # self.packet_reader.persist_configuration() - packet_queue = multiprocessing.Queue() stop_signal = multiprocessing.Value("i", 0) diff --git a/data_gateway/packet_reader.py b/data_gateway/packet_reader.py index 0ac80d6b..0fde4a0f 100644 --- a/data_gateway/packet_reader.py +++ b/data_gateway/packet_reader.py @@ -61,7 +61,7 @@ def __init__( self.sleep = False self.sensor_time_offset = None - os.makedirs(self.session_subdirectory, exist_ok=True) + os.makedirs(self.output_directory, exist_ok=True) logger.warning("Timestamp synchronisation unavailable with current hardware; defaulting to using system clock.") def read_packets(self, serial_port, packet_queue, stop_signal): @@ -145,6 +145,8 @@ def parse_packets(self, packet_queue, stop_signal, stop_when_no_more_data_after= else: self.writer = NoOperationContextManager() + self._persist_configuration() + previous_timestamp = {} data = {} @@ -243,7 +245,7 @@ def update_handles(self, payload): logger.error("Handle error: %s %s", start_handle, end_handle) - def persist_configuration(self): + def _persist_configuration(self): """Persist the configuration to disk and/or cloud storage. :return None: diff --git a/tests/test_data_gateway.py b/tests/test_data_gateway.py index 20d2fdcb..44732ccd 100644 --- a/tests/test_data_gateway.py +++ b/tests/test_data_gateway.py @@ -49,25 +49,16 @@ def test_configuration_file_is_persisted(self): data_gateway.start(stop_when_no_more_data_after=0.1) - configuration_path = os.path.join( - temporary_directory, data_gateway.packet_reader.session_subdirectory, "configuration.json" - ) - # Check configuration file is present and valid locally. - with open(configuration_path) as f: + with open(os.path.join(data_gateway.packet_reader.output_directory, "configuration.json")) as f: Configuration.from_dict(json.load(f)) # Check configuration file is present and valid on the cloud. configuration = self.storage_client.download_as_string( bucket_name=TEST_BUCKET_NAME, - path_in_bucket=storage.path.join( - data_gateway.packet_reader.uploader.output_directory, - data_gateway.packet_reader.session_subdirectory, - "configuration.json", - ), + path_in_bucket=storage.path.join(data_gateway.packet_reader.output_directory, "configuration.json"), ) - # Test configuration is valid. Configuration.from_dict(json.loads(configuration)) def test_data_gateway_with_baros_p_sensor(self): @@ -393,8 +384,9 @@ def _check_windows_are_uploaded_to_cloud(self, output_directory, sensor_names, n window_paths = [ blob.name for blob in self.storage_client.scandir( - cloud_path=storage.path.generate_gs_path(TEST_BUCKET_NAME, *output_directory.split(os.path.pathsep)) + cloud_path=storage.path.generate_gs_path(TEST_BUCKET_NAME, *os.path.split(output_directory)) ) + if not blob.name.endswith("configuration.json") ] self.assertTrue(len(window_paths) >= number_of_windows_to_check) From 728528c2ed98e731f841d9e42b80fdbf69388c54 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 14 Feb 2022 15:36:16 +0000 Subject: [PATCH 52/84] TST: Fix info packets test --- tests/test_data_gateway.py | 48 ------------------------------------ tests/test_packet_reader.py | 49 +++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 48 deletions(-) diff --git a/tests/test_data_gateway.py b/tests/test_data_gateway.py index 44732ccd..508e3b4d 100644 --- a/tests/test_data_gateway.py +++ b/tests/test_data_gateway.py @@ -331,54 +331,6 @@ def test_all_sensors_together(self): number_of_windows_to_check=1, ) - def test_data_gateway_with_info_packets(self): - """Test that the packet reader works with info packets.""" - serial_port = DummySerial(port="test") - - packet_types = [bytes([40]), bytes([54]), bytes([56]), bytes([58])] - - payloads = [ - [bytes([1]), bytes([2]), bytes([3])], - [bytes([0]), bytes([1]), bytes([2]), bytes([3])], - [bytes([0]), bytes([1])], - [bytes([0])], - ] - - for index, packet_type in enumerate(packet_types): - for payload in payloads[index]: - serial_port.write(data=b"".join((PACKET_KEY, packet_type, bytes([1]), payload))) - - with tempfile.TemporaryDirectory() as temporary_directory: - data_gateway = DataGateway( - serial_port, - save_locally=True, - upload_to_cloud=False, - output_directory=temporary_directory, - window_size=self.WINDOW_SIZE, - project_name=TEST_PROJECT_NAME, - bucket_name=TEST_BUCKET_NAME, - ) - - with self.assertLogs() as logging_context: - data_gateway.start(stop_when_no_more_data_after=0.1) - - log_messages_combined = "\n".join(logging_context.output) - - for message in [ - "Microphone data reading done", - "Microphone data erasing done", - "Microphones started ", - "Command declined, Bad block detection ongoing", - "Command declined, Task already registered, cannot register again", - "Command declined, Task is not registered, cannot de-register", - "Command declined, Connection Parameter update unfinished", - "\nExiting sleep\n", - "\nEntering sleep\n", - "Battery info", - "Voltage : 0.000000V\n Cycle count: 0.000000\nState of charge: 0.000000%", - ]: - self.assertIn(message, log_messages_combined) - def _check_windows_are_uploaded_to_cloud(self, output_directory, sensor_names, number_of_windows_to_check=5): """Check that non-trivial windows from a packet reader for a particular sensor are uploaded to cloud storage.""" window_paths = [ diff --git a/tests/test_packet_reader.py b/tests/test_packet_reader.py index a0157c15..15a8b6d9 100644 --- a/tests/test_packet_reader.py +++ b/tests/test_packet_reader.py @@ -1,4 +1,5 @@ import multiprocessing +import tempfile from unittest.mock import patch from data_gateway.packet_reader import PacketReader @@ -47,3 +48,51 @@ def test_update_handles(self): packet_reader.update_handles(packet) self.assertIn("Successfully updated handles", mock_logger.method_calls[0].args[0]) + + def test_packet_reader_with_info_packets(self): + """Test that the packet reader works with info packets.""" + packet_types = [bytes([40]), bytes([54]), bytes([56]), bytes([58])] + + packets = [ + [bytes([1]), bytes([2]), bytes([3])], + [bytes([0]), bytes([1]), bytes([2]), bytes([3])], + [bytes([0]), bytes([1])], + [bytes([0])], + ] + + queue = multiprocessing.Queue() + + for index, packet_type in enumerate(packet_types): + for packet in packets[index]: + queue.put({"packet_type": str(int.from_bytes(packet_type, "little")), "packet": packet}) + + with tempfile.TemporaryDirectory() as temporary_directory: + packet_reader = PacketReader( + save_locally=True, + upload_to_cloud=False, + output_directory=temporary_directory, + ) + + with patch("data_gateway.packet_reader.logger") as mock_logger: + packet_reader.parse_packets( + packet_queue=queue, + stop_signal=multiprocessing.Value("i", 0), + stop_when_no_more_data_after=0.1, + ) + + log_messages = [call_arg.args for call_arg in mock_logger.info.call_args_list] + + for message in [ + ("Microphone data reading done",), + ("Microphone data erasing done",), + ("Microphones started ",), + ("Command declined, %s", "Bad block detection ongoing"), + ("Command declined, %s", "Task already registered, cannot register again"), + ("Command declined, %s", "Task is not registered, cannot de-register"), + ("Command declined, %s", "Connection Parameter update unfinished"), + ("\n%s\n", "Exiting sleep"), + ("\n%s\n", "Entering sleep"), + ("Battery info",), + ("Voltage : %fV\n Cycle count: %f\nState of charge: %f%%", 0.0, 0.0, 0.0), + ]: + self.assertIn(message, log_messages) From e825bbab36f11fa5c151e49f0d03ddd75093ab75 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 14 Feb 2022 15:42:07 +0000 Subject: [PATCH 53/84] TST: Fix Routine tests --- tests/test_routine.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/test_routine.py b/tests/test_routine.py index 4123329c..5d7a66f9 100644 --- a/tests/test_routine.py +++ b/tests/test_routine.py @@ -1,5 +1,7 @@ +import multiprocessing import time from unittest import TestCase +from unittest.mock import patch from data_gateway.packet_reader import PacketReader from data_gateway.routine import Routine @@ -20,7 +22,7 @@ def record_commands(command): ) start_time = time.perf_counter() - routine.run() + routine.run(stop_signal=multiprocessing.Value("i", 0)) self.assertEqual(recorded_commands[0][0], "first-command") self.assertAlmostEqual(recorded_commands[0][1], start_time + 0.1, delta=0.2) @@ -51,7 +53,7 @@ def test_error_raised_if_stop_after_time_is_less_than_period(self): def test_warning_raised_if_stop_after_time_provided_without_a_period(self): """Test that a warning is raised if the `stop_after` time is provided without a period.""" - with self.assertLogs() as logging_context: + with patch("data_gateway.routine.logger") as mock_logger: Routine( commands=[("first-command", 10), ("second-command", 0.3)], action=None, @@ -60,8 +62,8 @@ def test_warning_raised_if_stop_after_time_provided_without_a_period(self): ) self.assertEqual( - logging_context.output[1], - "WARNING:data_gateway.routine:The `stop_after` parameter is ignored unless `period` is also given.", + mock_logger.warning.call_args_list[0].args[0], + "The `stop_after` parameter is ignored unless `period` is also given.", ) def test_routine_with_period(self): @@ -80,7 +82,7 @@ def record_commands(command): ) start_time = time.perf_counter() - routine.run() + routine.run(stop_signal=multiprocessing.Value("i", 0)) self.assertEqual(recorded_commands[0][0], "first-command") self.assertAlmostEqual(recorded_commands[0][1], start_time + 0.1, delta=0.2) From 51467c9998afa75b5b2b83d849a87b69a8d2d271 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 14 Feb 2022 15:56:05 +0000 Subject: [PATCH 54/84] FIX: Write commands file to correct path --- data_gateway/data_gateway.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/data_gateway/data_gateway.py b/data_gateway/data_gateway.py index d3eee09f..118ebcd7 100644 --- a/data_gateway/data_gateway.py +++ b/data_gateway/data_gateway.py @@ -237,16 +237,8 @@ def _send_commands_from_stdin_to_sensors(self, stop_signal): :return None: """ - commands_record_file = os.path.join( - self.packet_reader.output_directory, - self.packet_reader.session_subdirectory, - "commands.txt", - ) - - os.makedirs( - os.path.join(self.packet_reader.output_directory, self.packet_reader.session_subdirectory), - exist_ok=True, - ) + commands_record_file = os.path.join(self.packet_reader.output_directory, "commands.txt") + os.makedirs(os.path.join(self.packet_reader.output_directory), exist_ok=True) while stop_signal.value == 0: for line in sys.stdin: From 7ad959cf482fcc9dc6f7dffb1915770f9636f24c Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 14 Feb 2022 15:56:33 +0000 Subject: [PATCH 55/84] TST: Wait for process in test --- tests/test_cli.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index 4d822c8a..b55e147a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,7 @@ import json import os import tempfile +import time from unittest import mock from unittest.mock import call @@ -237,6 +238,8 @@ def test_save_locally(self): session_subdirectory = [item for item in os.scandir(temporary_directory) if item.is_dir()][0].name + # Wait for the parser process to receive stop signal and persist the window it has open. + time.sleep(2) with open(os.path.join(temporary_directory, session_subdirectory, "window-0.json")) as f: data = json.loads(f.read()) From 9a07226a1554877148595cb9fda71bdf613706dd Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 14 Feb 2022 17:02:01 +0000 Subject: [PATCH 56/84] TST: Fix output directory in data gateway tests --- tests/test_data_gateway.py | 52 ++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/tests/test_data_gateway.py b/tests/test_data_gateway.py index 508e3b4d..f60799da 100644 --- a/tests/test_data_gateway.py +++ b/tests/test_data_gateway.py @@ -62,7 +62,7 @@ def test_configuration_file_is_persisted(self): Configuration.from_dict(json.loads(configuration)) def test_data_gateway_with_baros_p_sensor(self): - """Test that the packet reader works with the Baro_P sensor.""" + """Test that the packet reader works with the "Baros_P" sensor.""" serial_port = DummySerial(port="test") packet_type = bytes([34]) @@ -80,10 +80,10 @@ def test_data_gateway_with_baros_p_sensor(self): ) data_gateway.start(stop_when_no_more_data_after=0.1) - self._check_data_is_written_to_files(temporary_directory, sensor_names=["Baros_P"]) + self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=["Baros_P"]) self._check_windows_are_uploaded_to_cloud( - temporary_directory, + data_gateway.packet_reader.output_directory, sensor_names=["Baros_P"], number_of_windows_to_check=1, ) @@ -106,10 +106,10 @@ def test_data_gateway_with_baros_t_sensor(self): bucket_name=TEST_BUCKET_NAME, ) data_gateway.start(stop_when_no_more_data_after=0.1) - self._check_data_is_written_to_files(temporary_directory, sensor_names=["Baros_T"]) + self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=["Baros_T"]) self._check_windows_are_uploaded_to_cloud( - temporary_directory, + data_gateway.packet_reader.output_directory, sensor_names=["Baros_T"], number_of_windows_to_check=1, ) @@ -132,10 +132,14 @@ def test_data_gateway_with_diff_baros_sensor(self): bucket_name=TEST_BUCKET_NAME, ) data_gateway.start(stop_when_no_more_data_after=0.1) - self._check_data_is_written_to_files(temporary_directory, sensor_names=["Diff_Baros"]) + + self._check_data_is_written_to_files( + data_gateway.packet_reader.output_directory, + sensor_names=["Diff_Baros"], + ) self._check_windows_are_uploaded_to_cloud( - temporary_directory, + data_gateway.packet_reader.output_directory, sensor_names=["Diff_Baros"], number_of_windows_to_check=1, ) @@ -158,10 +162,10 @@ def test_data_gateway_with_mic_sensor(self): bucket_name=TEST_BUCKET_NAME, ) data_gateway.start(stop_when_no_more_data_after=0.1) - self._check_data_is_written_to_files(temporary_directory, sensor_names=["Mics"]) + self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=["Mics"]) self._check_windows_are_uploaded_to_cloud( - temporary_directory, + data_gateway.packet_reader.output_directory, sensor_names=["Mics"], number_of_windows_to_check=1, ) @@ -184,10 +188,10 @@ def test_data_gateway_with_acc_sensor(self): bucket_name=TEST_BUCKET_NAME, ) data_gateway.start(stop_when_no_more_data_after=0.1) - self._check_data_is_written_to_files(temporary_directory, sensor_names=["Acc"]) + self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=["Acc"]) self._check_windows_are_uploaded_to_cloud( - temporary_directory, + data_gateway.packet_reader.output_directory, sensor_names=["Acc"], number_of_windows_to_check=1, ) @@ -210,10 +214,10 @@ def test_data_gateway_with_gyro_sensor(self): bucket_name=TEST_BUCKET_NAME, ) data_gateway.start(stop_when_no_more_data_after=0.1) - self._check_data_is_written_to_files(temporary_directory, sensor_names=["Gyro"]) + self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=["Gyro"]) self._check_windows_are_uploaded_to_cloud( - temporary_directory, + data_gateway.packet_reader.output_directory, sensor_names=["Gyro"], number_of_windows_to_check=1, ) @@ -236,10 +240,10 @@ def test_data_gateway_with_mag_sensor(self): bucket_name=TEST_BUCKET_NAME, ) data_gateway.start(stop_when_no_more_data_after=0.1) - self._check_data_is_written_to_files(temporary_directory, sensor_names=["Mag"]) + self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=["Mag"]) self._check_windows_are_uploaded_to_cloud( - temporary_directory, + data_gateway.packet_reader.output_directory, sensor_names=["Mag"], number_of_windows_to_check=1, ) @@ -263,10 +267,10 @@ def test_data_gateway_with_connections_statistics(self): ) data_gateway.start(stop_when_no_more_data_after=0.1) - self._check_data_is_written_to_files(temporary_directory, sensor_names=["Constat"]) + self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=["Constat"]) self._check_windows_are_uploaded_to_cloud( - temporary_directory, + data_gateway.packet_reader.output_directory, sensor_names=["Constat"], number_of_windows_to_check=1, ) @@ -299,7 +303,7 @@ def test_data_gateway_with_connections_statistics_in_sleep_mode(self): with patch("data_gateway.packet_reader.logger") as mock_logger: data_gateway.start(stop_when_no_more_data_after=0.1) - self._check_data_is_written_to_files(temporary_directory, sensor_names=["Constat"]) + self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=["Constat"]) self.assertEqual(0, mock_logger.warning.call_count) def test_all_sensors_together(self): @@ -323,10 +327,10 @@ def test_all_sensors_together(self): ) data_gateway.start(stop_when_no_more_data_after=0.1) - self._check_data_is_written_to_files(temporary_directory, sensor_names=sensor_names) + self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=sensor_names) self._check_windows_are_uploaded_to_cloud( - temporary_directory, + data_gateway.packet_reader.output_directory, sensor_names=sensor_names, number_of_windows_to_check=1, ) @@ -336,7 +340,7 @@ def _check_windows_are_uploaded_to_cloud(self, output_directory, sensor_names, n window_paths = [ blob.name for blob in self.storage_client.scandir( - cloud_path=storage.path.generate_gs_path(TEST_BUCKET_NAME, *os.path.split(output_directory)) + cloud_path=storage.path.generate_gs_path(TEST_BUCKET_NAME, output_directory) ) if not blob.name.endswith("configuration.json") ] @@ -352,13 +356,11 @@ def _check_windows_are_uploaded_to_cloud(self, output_directory, sensor_names, n def _check_data_is_written_to_files(self, output_directory, sensor_names): """Check that non-trivial data is written to the given file.""" - session_subdirectory = os.listdir(output_directory)[0] - window_directory = os.path.join(output_directory, session_subdirectory) - windows = [file for file in os.listdir(window_directory) if file.startswith(TimeBatcher._file_prefix)] + windows = [file for file in os.listdir(output_directory) if file.startswith(TimeBatcher._file_prefix)] self.assertTrue(len(windows) > 0) for window in windows: - with open(os.path.join(window_directory, window)) as f: + with open(os.path.join(output_directory, window)) as f: data = json.load(f) for name in sensor_names: From dbc32e2fb429a6bb9fdcdedb05fa8d1e3137e410 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 14 Feb 2022 17:12:42 +0000 Subject: [PATCH 57/84] FIX: Leave output directory unaltered in DataGateway --- data_gateway/data_gateway.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/data_gateway/data_gateway.py b/data_gateway/data_gateway.py index 118ebcd7..97c3232f 100644 --- a/data_gateway/data_gateway.py +++ b/data_gateway/data_gateway.py @@ -78,7 +78,7 @@ def __init__( self.packet_reader = PacketReader( save_locally=save_locally, upload_to_cloud=upload_to_cloud, - output_directory=self._update_output_directory(output_directory_path=output_directory), + output_directory=output_directory, window_size=window_size, project_name=project_name, bucket_name=bucket_name, @@ -219,17 +219,6 @@ def _load_routine(self, routine_path): routine_path, ) - def _update_output_directory(self, output_directory_path): - """Set the output directory to a path relative to the current directory if the path does not start with "/". - - :param str output_directory_path: the path to the directory to write output data to - :return str: the updated output directory path - """ - if not output_directory_path.startswith("/"): - output_directory_path = os.path.join(".", output_directory_path) - - return output_directory_path - def _send_commands_from_stdin_to_sensors(self, stop_signal): """Send commands from `stdin` to the sensors until the "stop" command is received or the packet reader is otherwise stopped. A record is kept of the commands sent to the sensors as a text file in the session From 40d6679ceec908f20896940f62f9761e54a005ae Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 14 Feb 2022 17:23:27 +0000 Subject: [PATCH 58/84] TST: Stop PacketReader tests producing empty folders --- tests/test_packet_reader.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tests/test_packet_reader.py b/tests/test_packet_reader.py index 15a8b6d9..626f181c 100644 --- a/tests/test_packet_reader.py +++ b/tests/test_packet_reader.py @@ -13,7 +13,11 @@ def test_error_is_logged_if_unknown_sensor_type_packet_is_received(self): queue = multiprocessing.Queue() queue.put({"packet_type": bytes([0]), "packet": b"".join((PACKET_KEY, bytes([0]), LENGTH, RANDOM_BYTES[0]))}) - packet_reader = PacketReader(save_locally=False, upload_to_cloud=False) + packet_reader = PacketReader( + save_locally=False, + upload_to_cloud=False, + output_directory=tempfile.TemporaryDirectory().name, + ) with patch("data_gateway.packet_reader.logger") as mock_logger: packet_reader.parse_packets( @@ -30,7 +34,11 @@ def test_update_handles_fails_if_start_and_end_handles_are_incorrect(self): packet[0:1] = int(0).to_bytes(1, "little") packet[2:3] = int(255).to_bytes(1, "little") - packet_reader = PacketReader(save_locally=False, upload_to_cloud=False) + packet_reader = PacketReader( + save_locally=False, + upload_to_cloud=False, + output_directory=tempfile.TemporaryDirectory().name, + ) with patch("data_gateway.packet_reader.logger") as mock_logger: packet_reader.update_handles(packet) @@ -42,7 +50,11 @@ def test_update_handles(self): packet = bytearray(RANDOM_BYTES[0]) packet[0:1] = int(0).to_bytes(1, "little") packet[2:3] = int(26).to_bytes(1, "little") - packet_reader = PacketReader(save_locally=False, upload_to_cloud=False) + packet_reader = PacketReader( + save_locally=False, + upload_to_cloud=False, + output_directory=tempfile.TemporaryDirectory().name, + ) with patch("data_gateway.packet_reader.logger") as mock_logger: packet_reader.update_handles(packet) From 3f4e8f4376ac57a0716e44a3740c97d10e8d5e58 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 14 Feb 2022 17:27:06 +0000 Subject: [PATCH 59/84] REF: Remove unnecessary directory creation --- data_gateway/data_gateway.py | 1 - 1 file changed, 1 deletion(-) diff --git a/data_gateway/data_gateway.py b/data_gateway/data_gateway.py index 97c3232f..b81c8bb0 100644 --- a/data_gateway/data_gateway.py +++ b/data_gateway/data_gateway.py @@ -227,7 +227,6 @@ def _send_commands_from_stdin_to_sensors(self, stop_signal): :return None: """ commands_record_file = os.path.join(self.packet_reader.output_directory, "commands.txt") - os.makedirs(os.path.join(self.packet_reader.output_directory), exist_ok=True) while stop_signal.value == 0: for line in sys.stdin: From 0f1a605b59a352cf0e044d81b7c17b936a684479 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 14 Feb 2022 17:28:38 +0000 Subject: [PATCH 60/84] TST: Move data gateway tests into their own test subpackage --- tests/test_data_gateway/__init__.py | 0 tests/{ => test_data_gateway}/test_cli.py | 2 +- tests/{ => test_data_gateway}/test_configuration.py | 2 +- tests/{ => test_data_gateway}/test_data_gateway.py | 0 tests/{ => test_data_gateway}/test_dummy_serial.py | 0 tests/{ => test_data_gateway}/test_packet_reader.py | 0 tests/{ => test_data_gateway}/test_persistence.py | 0 tests/{ => test_data_gateway}/test_routine.py | 0 8 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 tests/test_data_gateway/__init__.py rename tests/{ => test_data_gateway}/test_cli.py (99%) rename tests/{ => test_data_gateway}/test_configuration.py (91%) rename tests/{ => test_data_gateway}/test_data_gateway.py (100%) rename tests/{ => test_data_gateway}/test_dummy_serial.py (100%) rename tests/{ => test_data_gateway}/test_packet_reader.py (100%) rename tests/{ => test_data_gateway}/test_persistence.py (100%) rename tests/{ => test_data_gateway}/test_routine.py (100%) diff --git a/tests/test_data_gateway/__init__.py b/tests/test_data_gateway/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_cli.py b/tests/test_data_gateway/test_cli.py similarity index 99% rename from tests/test_cli.py rename to tests/test_data_gateway/test_cli.py index b55e147a..1c022783 100644 --- a/tests/test_cli.py +++ b/tests/test_data_gateway/test_cli.py @@ -16,7 +16,7 @@ from tests.base import BaseTestCase -CONFIGURATION_PATH = os.path.join(os.path.dirname(__file__), "valid_configuration.json") +CONFIGURATION_PATH = os.path.join(os.path.dirname(os.path.dirname(__file__)), "valid_configuration.json") class EnvironmentVariableRemover: diff --git a/tests/test_configuration.py b/tests/test_data_gateway/test_configuration.py similarity index 91% rename from tests/test_configuration.py rename to tests/test_data_gateway/test_configuration.py index 4283cdfd..b9a6f397 100644 --- a/tests/test_configuration.py +++ b/tests/test_data_gateway/test_configuration.py @@ -7,7 +7,7 @@ class TestConfiguration(BaseTestCase): - configuration_path = os.path.join(os.path.dirname(__file__), "valid_configuration.json") + configuration_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "valid_configuration.json") with open(configuration_path) as f: VALID_CONFIGURATION = json.load(f) diff --git a/tests/test_data_gateway.py b/tests/test_data_gateway/test_data_gateway.py similarity index 100% rename from tests/test_data_gateway.py rename to tests/test_data_gateway/test_data_gateway.py diff --git a/tests/test_dummy_serial.py b/tests/test_data_gateway/test_dummy_serial.py similarity index 100% rename from tests/test_dummy_serial.py rename to tests/test_data_gateway/test_dummy_serial.py diff --git a/tests/test_packet_reader.py b/tests/test_data_gateway/test_packet_reader.py similarity index 100% rename from tests/test_packet_reader.py rename to tests/test_data_gateway/test_packet_reader.py diff --git a/tests/test_persistence.py b/tests/test_data_gateway/test_persistence.py similarity index 100% rename from tests/test_persistence.py rename to tests/test_data_gateway/test_persistence.py diff --git a/tests/test_routine.py b/tests/test_data_gateway/test_routine.py similarity index 100% rename from tests/test_routine.py rename to tests/test_data_gateway/test_routine.py From d373ea531a803615aecedcd7d1ce283177b180cb Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 14 Feb 2022 18:07:39 +0000 Subject: [PATCH 61/84] TST: Avoid colons in cloud storage blob names --- tests/test_data_gateway.py | 268 +++++++++++++++++++------------------ 1 file changed, 139 insertions(+), 129 deletions(-) diff --git a/tests/test_data_gateway.py b/tests/test_data_gateway.py index f60799da..fb890899 100644 --- a/tests/test_data_gateway.py +++ b/tests/test_data_gateway.py @@ -1,3 +1,6 @@ +import shutil + +import coolname import json import os import tempfile @@ -29,6 +32,17 @@ def setUpClass(cls): cls.WINDOW_SIZE = 10 cls.storage_client = GoogleCloudStorageClient(project_name=TEST_PROJECT_NAME) + def setUp(self): + """Create a uniquely-named output directory.""" + self.output_directory = coolname.generate_slug(2) + + def tearDown(self): + """Delete the output directory created in `setUp`.""" + try: + shutil.rmtree(self.output_directory) + except FileNotFoundError: + pass + def test_configuration_file_is_persisted(self): """Test that the configuration file is persisted.""" serial_port = DummySerial(port="test") @@ -37,21 +51,20 @@ def test_configuration_file_is_persisted(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[0]))) serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[1]))) - with tempfile.TemporaryDirectory() as temporary_directory: - data_gateway = DataGateway( - serial_port=serial_port, - save_locally=True, - output_directory=temporary_directory, - window_size=self.WINDOW_SIZE, - project_name=TEST_PROJECT_NAME, - bucket_name=TEST_BUCKET_NAME, - ) + data_gateway = DataGateway( + serial_port=serial_port, + save_locally=True, + output_directory=self.output_directory, + window_size=self.WINDOW_SIZE, + project_name=TEST_PROJECT_NAME, + bucket_name=TEST_BUCKET_NAME, + ) - data_gateway.start(stop_when_no_more_data_after=0.1) + data_gateway.start(stop_when_no_more_data_after=0.1) - # Check configuration file is present and valid locally. - with open(os.path.join(data_gateway.packet_reader.output_directory, "configuration.json")) as f: - Configuration.from_dict(json.load(f)) + # Check configuration file is present and valid locally. + with open(os.path.join(data_gateway.packet_reader.output_directory, "configuration.json")) as f: + Configuration.from_dict(json.load(f)) # Check configuration file is present and valid on the cloud. configuration = self.storage_client.download_as_string( @@ -69,18 +82,17 @@ def test_data_gateway_with_baros_p_sensor(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[0]))) serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[1]))) - with tempfile.TemporaryDirectory() as temporary_directory: - data_gateway = DataGateway( - serial_port, - save_locally=True, - output_directory=temporary_directory, - window_size=self.WINDOW_SIZE, - project_name=TEST_PROJECT_NAME, - bucket_name=TEST_BUCKET_NAME, - ) + data_gateway = DataGateway( + serial_port, + save_locally=True, + output_directory=coolname.generate_slug(2), + window_size=self.WINDOW_SIZE, + project_name=TEST_PROJECT_NAME, + bucket_name=TEST_BUCKET_NAME, + ) - data_gateway.start(stop_when_no_more_data_after=0.1) - self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=["Baros_P"]) + data_gateway.start(stop_when_no_more_data_after=0.1) + self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=["Baros_P"]) self._check_windows_are_uploaded_to_cloud( data_gateway.packet_reader.output_directory, @@ -96,17 +108,17 @@ def test_data_gateway_with_baros_t_sensor(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[0]))) serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[1]))) - with tempfile.TemporaryDirectory() as temporary_directory: - data_gateway = DataGateway( - serial_port, - save_locally=True, - output_directory=temporary_directory, - window_size=self.WINDOW_SIZE, - project_name=TEST_PROJECT_NAME, - bucket_name=TEST_BUCKET_NAME, - ) - data_gateway.start(stop_when_no_more_data_after=0.1) - self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=["Baros_T"]) + + data_gateway = DataGateway( + serial_port, + save_locally=True, + output_directory=self.output_directory, + window_size=self.WINDOW_SIZE, + project_name=TEST_PROJECT_NAME, + bucket_name=TEST_BUCKET_NAME, + ) + data_gateway.start(stop_when_no_more_data_after=0.1) + self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=["Baros_T"]) self._check_windows_are_uploaded_to_cloud( data_gateway.packet_reader.output_directory, @@ -122,21 +134,21 @@ def test_data_gateway_with_diff_baros_sensor(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[0]))) serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[1]))) - with tempfile.TemporaryDirectory() as temporary_directory: - data_gateway = DataGateway( - serial_port, - save_locally=True, - output_directory=temporary_directory, - window_size=self.WINDOW_SIZE, - project_name=TEST_PROJECT_NAME, - bucket_name=TEST_BUCKET_NAME, - ) - data_gateway.start(stop_when_no_more_data_after=0.1) - self._check_data_is_written_to_files( - data_gateway.packet_reader.output_directory, - sensor_names=["Diff_Baros"], - ) + data_gateway = DataGateway( + serial_port, + save_locally=True, + output_directory=self.output_directory, + window_size=self.WINDOW_SIZE, + project_name=TEST_PROJECT_NAME, + bucket_name=TEST_BUCKET_NAME, + ) + data_gateway.start(stop_when_no_more_data_after=0.1) + + self._check_data_is_written_to_files( + data_gateway.packet_reader.output_directory, + sensor_names=["Diff_Baros"], + ) self._check_windows_are_uploaded_to_cloud( data_gateway.packet_reader.output_directory, @@ -152,17 +164,17 @@ def test_data_gateway_with_mic_sensor(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[0]))) serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[1]))) - with tempfile.TemporaryDirectory() as temporary_directory: - data_gateway = DataGateway( - serial_port, - save_locally=True, - output_directory=temporary_directory, - window_size=self.WINDOW_SIZE, - project_name=TEST_PROJECT_NAME, - bucket_name=TEST_BUCKET_NAME, - ) - data_gateway.start(stop_when_no_more_data_after=0.1) - self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=["Mics"]) + + data_gateway = DataGateway( + serial_port, + save_locally=True, + output_directory=self.output_directory, + window_size=self.WINDOW_SIZE, + project_name=TEST_PROJECT_NAME, + bucket_name=TEST_BUCKET_NAME, + ) + data_gateway.start(stop_when_no_more_data_after=0.1) + self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=["Mics"]) self._check_windows_are_uploaded_to_cloud( data_gateway.packet_reader.output_directory, @@ -178,17 +190,17 @@ def test_data_gateway_with_acc_sensor(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[0]))) serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[1]))) - with tempfile.TemporaryDirectory() as temporary_directory: - data_gateway = DataGateway( - serial_port, - save_locally=True, - output_directory=temporary_directory, - window_size=self.WINDOW_SIZE, - project_name=TEST_PROJECT_NAME, - bucket_name=TEST_BUCKET_NAME, - ) - data_gateway.start(stop_when_no_more_data_after=0.1) - self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=["Acc"]) + + data_gateway = DataGateway( + serial_port, + save_locally=True, + output_directory=self.output_directory, + window_size=self.WINDOW_SIZE, + project_name=TEST_PROJECT_NAME, + bucket_name=TEST_BUCKET_NAME, + ) + data_gateway.start(stop_when_no_more_data_after=0.1) + self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=["Acc"]) self._check_windows_are_uploaded_to_cloud( data_gateway.packet_reader.output_directory, @@ -204,17 +216,17 @@ def test_data_gateway_with_gyro_sensor(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[0]))) serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[1]))) - with tempfile.TemporaryDirectory() as temporary_directory: - data_gateway = DataGateway( - serial_port, - save_locally=True, - output_directory=temporary_directory, - window_size=self.WINDOW_SIZE, - project_name=TEST_PROJECT_NAME, - bucket_name=TEST_BUCKET_NAME, - ) - data_gateway.start(stop_when_no_more_data_after=0.1) - self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=["Gyro"]) + + data_gateway = DataGateway( + serial_port, + save_locally=True, + output_directory=self.output_directory, + window_size=self.WINDOW_SIZE, + project_name=TEST_PROJECT_NAME, + bucket_name=TEST_BUCKET_NAME, + ) + data_gateway.start(stop_when_no_more_data_after=0.1) + self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=["Gyro"]) self._check_windows_are_uploaded_to_cloud( data_gateway.packet_reader.output_directory, @@ -230,17 +242,17 @@ def test_data_gateway_with_mag_sensor(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[0]))) serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[1]))) - with tempfile.TemporaryDirectory() as temporary_directory: - data_gateway = DataGateway( - serial_port, - save_locally=True, - output_directory=temporary_directory, - window_size=self.WINDOW_SIZE, - project_name=TEST_PROJECT_NAME, - bucket_name=TEST_BUCKET_NAME, - ) - data_gateway.start(stop_when_no_more_data_after=0.1) - self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=["Mag"]) + + data_gateway = DataGateway( + serial_port, + save_locally=True, + output_directory=self.output_directory, + window_size=self.WINDOW_SIZE, + project_name=TEST_PROJECT_NAME, + bucket_name=TEST_BUCKET_NAME, + ) + data_gateway.start(stop_when_no_more_data_after=0.1) + self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=["Mag"]) self._check_windows_are_uploaded_to_cloud( data_gateway.packet_reader.output_directory, @@ -256,18 +268,18 @@ def test_data_gateway_with_connections_statistics(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[0]))) serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[1]))) - with tempfile.TemporaryDirectory() as temporary_directory: - data_gateway = DataGateway( - serial_port, - save_locally=True, - output_directory=temporary_directory, - window_size=self.WINDOW_SIZE, - project_name=TEST_PROJECT_NAME, - bucket_name=TEST_BUCKET_NAME, - ) - data_gateway.start(stop_when_no_more_data_after=0.1) - self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=["Constat"]) + data_gateway = DataGateway( + serial_port, + save_locally=True, + output_directory=self.output_directory, + window_size=self.WINDOW_SIZE, + project_name=TEST_PROJECT_NAME, + bucket_name=TEST_BUCKET_NAME, + ) + data_gateway.start(stop_when_no_more_data_after=0.1) + + self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=["Constat"]) self._check_windows_are_uploaded_to_cloud( data_gateway.packet_reader.output_directory, @@ -289,22 +301,21 @@ def test_data_gateway_with_connections_statistics_in_sleep_mode(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[0]))) serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[1]))) - with tempfile.TemporaryDirectory() as temporary_directory: - data_gateway = DataGateway( - serial_port, - save_locally=True, - upload_to_cloud=False, - output_directory=temporary_directory, - window_size=self.WINDOW_SIZE, - project_name=TEST_PROJECT_NAME, - bucket_name=TEST_BUCKET_NAME, - ) + data_gateway = DataGateway( + serial_port, + save_locally=True, + upload_to_cloud=False, + output_directory=self.output_directory, + window_size=self.WINDOW_SIZE, + project_name=TEST_PROJECT_NAME, + bucket_name=TEST_BUCKET_NAME, + ) - with patch("data_gateway.packet_reader.logger") as mock_logger: - data_gateway.start(stop_when_no_more_data_after=0.1) + with patch("data_gateway.packet_reader.logger") as mock_logger: + data_gateway.start(stop_when_no_more_data_after=0.1) - self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=["Constat"]) - self.assertEqual(0, mock_logger.warning.call_count) + self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=["Constat"]) + self.assertEqual(0, mock_logger.warning.call_count) def test_all_sensors_together(self): """Test that the packet reader works with all sensors together.""" @@ -316,18 +327,17 @@ def test_all_sensors_together(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[0]))) serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[1]))) - with tempfile.TemporaryDirectory() as temporary_directory: - data_gateway = DataGateway( - serial_port, - save_locally=True, - output_directory=temporary_directory, - window_size=self.WINDOW_SIZE, - project_name=TEST_PROJECT_NAME, - bucket_name=TEST_BUCKET_NAME, - ) - data_gateway.start(stop_when_no_more_data_after=0.1) + data_gateway = DataGateway( + serial_port, + save_locally=True, + output_directory=self.output_directory, + window_size=self.WINDOW_SIZE, + project_name=TEST_PROJECT_NAME, + bucket_name=TEST_BUCKET_NAME, + ) + data_gateway.start(stop_when_no_more_data_after=0.1) - self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=sensor_names) + self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=sensor_names) self._check_windows_are_uploaded_to_cloud( data_gateway.packet_reader.output_directory, From 2d2e83eb5790b4724c27690b6595fb3a8bf296a1 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 14 Feb 2022 18:34:26 +0000 Subject: [PATCH 62/84] TST: Remove problematic (for Windows) test --- tests/test_data_gateway/test_cli.py | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/tests/test_data_gateway/test_cli.py b/tests/test_data_gateway/test_cli.py index 1c022783..3e23d7c3 100644 --- a/tests/test_data_gateway/test_cli.py +++ b/tests/test_data_gateway/test_cli.py @@ -86,32 +86,6 @@ def test_start(self): self.assertIsNone(result.exception) self.assertEqual(result.exit_code, 0) - def test_start_with_default_output_directory(self): - """Ensure the gateway can be started via the CLI with a default output directory.""" - initial_directory = os.getcwd() - - with tempfile.TemporaryDirectory() as temporary_directory: - try: - os.chdir(temporary_directory) - - result = CliRunner().invoke( - gateway_cli, - [ - "start", - "--interactive", - "--save-locally", - "--no-upload-to-cloud", - "--use-dummy-serial-port", - ], - input="sleep 2\nstop\n", - ) - - self.assertIsNone(result.exception) - self.assertEqual(result.exit_code, 0) - - finally: - os.chdir(initial_directory) - def test_commands_are_recorded_in_interactive_mode(self): """Ensure commands given in interactive mode are recorded.""" with EnvironmentVariableRemover("GOOGLE_APPLICATION_CREDENTIALS"): From 7f56e3f3e0b72116470ddac62257ef36c12a95fa Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 15 Feb 2022 10:56:04 +0000 Subject: [PATCH 63/84] ENH: Remove thread name from logging context skipci --- data_gateway/cli.py | 1 - data_gateway/data_gateway.py | 2 +- tests/__init__.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/data_gateway/cli.py b/data_gateway/cli.py index 2a87ba1b..be33c49d 100644 --- a/data_gateway/cli.py +++ b/data_gateway/cli.py @@ -46,7 +46,6 @@ def gateway_cli(logger_uri, log_level): logger=logger, handler=get_remote_handler(logger_uri=logger_uri), log_level=log_level.upper(), - include_thread_name=True, include_process_name=True, ) diff --git a/data_gateway/data_gateway.py b/data_gateway/data_gateway.py index b81c8bb0..e8aa7a7b 100644 --- a/data_gateway/data_gateway.py +++ b/data_gateway/data_gateway.py @@ -16,7 +16,7 @@ logger = multiprocessing.get_logger() -apply_log_handler(logger=logger, include_process_name=True, include_thread_name=True) +apply_log_handler(logger=logger, include_process_name=True) class DataGateway: diff --git a/tests/__init__.py b/tests/__init__.py index 280da59e..ecb96109 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -3,7 +3,7 @@ from data_gateway.configuration import Configuration -apply_log_handler(include_thread_name=True, include_process_name=True) +apply_log_handler(include_process_name=True) TEST_PROJECT_NAME = "a-project-name" From 3165a94539b70718fc73209dc492798d6ac0b644 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 15 Feb 2022 10:57:20 +0000 Subject: [PATCH 64/84] ENH: Remove "Process" from process names skipci --- data_gateway/data_gateway.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data_gateway/data_gateway.py b/data_gateway/data_gateway.py index e8aa7a7b..b7bee815 100644 --- a/data_gateway/data_gateway.py +++ b/data_gateway/data_gateway.py @@ -101,7 +101,7 @@ def start(self, stop_when_no_more_data_after=False): stop_signal = multiprocessing.Value("i", 0) reader_process = multiprocessing.Process( - name="ReaderProcess", + name="Reader", target=self.packet_reader.read_packets, kwargs={ "serial_port": self.serial_port, @@ -112,7 +112,7 @@ def start(self, stop_when_no_more_data_after=False): ) parser_process = multiprocessing.Process( - name="ParserProcess", + name="Parser", target=self.packet_reader.parse_packets, kwargs={ "packet_queue": packet_queue, From f778e8509a679608a951bfa1cdd4ad2962a47ef8 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 15 Feb 2022 11:25:44 +0000 Subject: [PATCH 65/84] ENH: Use absolute path to output directory --- data_gateway/packet_reader.py | 4 ++-- data_gateway/persistence.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/data_gateway/packet_reader.py b/data_gateway/packet_reader.py index 0fde4a0f..1a464614 100644 --- a/data_gateway/packet_reader.py +++ b/data_gateway/packet_reader.py @@ -48,7 +48,7 @@ def __init__( self.save_locally = save_locally self.upload_to_cloud = upload_to_cloud self.session_subdirectory = str(hash(datetime.datetime.now()))[1:7] - self.output_directory = os.path.join(output_directory, self.session_subdirectory) + self.output_directory = os.path.abspath(os.path.join(output_directory, self.session_subdirectory)) self.window_size = window_size self.project_name = project_name self.bucket_name = bucket_name @@ -253,7 +253,7 @@ def _persist_configuration(self): configuration_dictionary = self.config.to_dict() if self.save_locally: - with open(os.path.abspath(os.path.join(self.output_directory, "configuration.json")), "w") as f: + with open(os.path.join(self.output_directory, "configuration.json"), "w") as f: json.dump(configuration_dictionary, f) if self.upload_to_cloud: diff --git a/data_gateway/persistence.py b/data_gateway/persistence.py index f9991bbc..e23968dd 100644 --- a/data_gateway/persistence.py +++ b/data_gateway/persistence.py @@ -12,7 +12,6 @@ logger = multiprocessing.get_logger() -# apply_log_handler(logger=logger) DEFAULT_OUTPUT_DIRECTORY = "data_gateway" @@ -159,7 +158,7 @@ def _persist_window(self, window=None): """ self._manage_storage() window = window or self.ready_window - window_path = os.path.abspath(os.path.join(".", self._generate_window_path())) + window_path = self._generate_window_path() with open(window_path, "w") as f: json.dump(window, f) From 0d4ad43f0d2d2630d09092effa28002061804615 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 15 Feb 2022 11:29:19 +0000 Subject: [PATCH 66/84] ENH: Optimise persistence log statements --- data_gateway/persistence.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/data_gateway/persistence.py b/data_gateway/persistence.py index e23968dd..0409e1ea 100644 --- a/data_gateway/persistence.py +++ b/data_gateway/persistence.py @@ -147,7 +147,7 @@ def __init__( self.storage_limit = storage_limit super().__init__(sensor_names, window_size, output_directory) os.makedirs(self.output_directory, exist_ok=True) - logger.info(f"Windows will be saved to {self.output_directory!r} at intervals of {self.window_size} seconds.") + logger.info("Windows will be saved to %r at intervals of %s seconds.", self.output_directory, self.window_size) def _persist_window(self, window=None): """Write a window of serialised data to disk, deleting the oldest window first if the storage limit has been @@ -163,12 +163,12 @@ def _persist_window(self, window=None): with open(window_path, "w") as f: json.dump(window, f) - logger.info(f"{self._file_prefix.capitalize()} {self._window_number} written to disk.") + logger.info("%s %d written to disk.", self._file_prefix.capitalize(), self._window_number) if self._save_csv_files: for sensor in window["sensor_data"]: csv_path = os.path.join(os.path.dirname(window_path), f"{sensor}.csv") - logger.info(f"Saving {sensor} data to csv file.") + logger.info("Saving %s data to csv file.", sensor) with open(csv_path, "w", newline="") as f: writer = csv.writer(f, delimiter=",") @@ -244,7 +244,7 @@ def __init__( self._backup_writer = BatchingFileWriter(sensor_names, window_size, output_directory=self._backup_directory) logger.info( - f"Windows will be uploaded to {self.output_directory!r} at intervals of {self.window_size} seconds." + "Windows will be uploaded to %r at intervals of %s seconds.", self.output_directory, self.window_size ) def _persist_window(self): @@ -272,7 +272,7 @@ def _persist_window(self): self._backup_writer._persist_window(window=self.ready_window) return - logger.info(f"{self._file_prefix.capitalize()} {self._window_number} uploaded to cloud.") + logger.info("%s %d uploaded to cloud.", self._file_prefix.capitalize(), self._window_number) if self.upload_backup_files: self._attempt_to_upload_backup_files() From e9fb9bd531a80df3caf247a2443ae235c6b5d4ae Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 15 Feb 2022 11:40:52 +0000 Subject: [PATCH 67/84] REF: Remove packet_reader parameter from Routine --- data_gateway/data_gateway.py | 1 - data_gateway/routine.py | 3 +-- tests/test_data_gateway/test_routine.py | 6 ------ 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/data_gateway/data_gateway.py b/data_gateway/data_gateway.py index b7bee815..0d4e5a5a 100644 --- a/data_gateway/data_gateway.py +++ b/data_gateway/data_gateway.py @@ -208,7 +208,6 @@ def _load_routine(self, routine_path): routine = Routine( **json.load(f), action=lambda command: self.serial_port.write((command + "\n").encode("utf_8")), - packet_reader=self.packet_reader, ) logger.debug("Loaded routine file from %r.", routine_path) diff --git a/data_gateway/routine.py b/data_gateway/routine.py index c73d0c36..eacce402 100644 --- a/data_gateway/routine.py +++ b/data_gateway/routine.py @@ -19,10 +19,9 @@ class Routine: :return None: """ - def __init__(self, commands, action, packet_reader, period=None, stop_after=None): + def __init__(self, commands, action, period=None, stop_after=None): self.commands = commands self.action = self._wrap_action_with_logger(action) - self.packet_reader = packet_reader self.period = period self.stop_after = stop_after diff --git a/tests/test_data_gateway/test_routine.py b/tests/test_data_gateway/test_routine.py index 5d7a66f9..d668a6c3 100644 --- a/tests/test_data_gateway/test_routine.py +++ b/tests/test_data_gateway/test_routine.py @@ -3,7 +3,6 @@ from unittest import TestCase from unittest.mock import patch -from data_gateway.packet_reader import PacketReader from data_gateway.routine import Routine @@ -18,7 +17,6 @@ def record_commands(command): routine = Routine( commands=[("first-command", 0.1), ("second-command", 0.3)], action=record_commands, - packet_reader=PacketReader(save_locally=False, upload_to_cloud=False), ) start_time = time.perf_counter() @@ -36,7 +34,6 @@ def test_error_raised_if_any_delay_is_greater_than_period(self): Routine( commands=[("first-command", 10), ("second-command", 0.3)], action=None, - packet_reader=PacketReader(save_locally=False, upload_to_cloud=False), period=1, ) @@ -46,7 +43,6 @@ def test_error_raised_if_stop_after_time_is_less_than_period(self): Routine( commands=[("first-command", 0.1), ("second-command", 0.3)], action=None, - packet_reader=PacketReader(save_locally=False, upload_to_cloud=False), period=1, stop_after=0.5, ) @@ -57,7 +53,6 @@ def test_warning_raised_if_stop_after_time_provided_without_a_period(self): Routine( commands=[("first-command", 10), ("second-command", 0.3)], action=None, - packet_reader=PacketReader(save_locally=False, upload_to_cloud=False), stop_after=0.5, ) @@ -76,7 +71,6 @@ def record_commands(command): routine = Routine( commands=[("first-command", 0.1), ("second-command", 0.3)], action=record_commands, - packet_reader=PacketReader(save_locally=False, upload_to_cloud=False), period=0.4, stop_after=1, ) From 5e2f0150f60a3df677c9b29ed8b46b299387d817 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 15 Feb 2022 12:11:24 +0000 Subject: [PATCH 68/84] FIX: Separate cloud output directory from local output directory --- data_gateway/data_gateway.py | 2 +- data_gateway/packet_reader.py | 15 +++-- tests/test_data_gateway/test_data_gateway.py | 67 ++++++++++---------- 3 files changed, 45 insertions(+), 39 deletions(-) diff --git a/data_gateway/data_gateway.py b/data_gateway/data_gateway.py index 0d4e5a5a..f009ddf3 100644 --- a/data_gateway/data_gateway.py +++ b/data_gateway/data_gateway.py @@ -225,7 +225,7 @@ def _send_commands_from_stdin_to_sensors(self, stop_signal): :return None: """ - commands_record_file = os.path.join(self.packet_reader.output_directory, "commands.txt") + commands_record_file = os.path.join(self.packet_reader.local_output_directory, "commands.txt") while stop_signal.value == 0: for line in sys.stdin: diff --git a/data_gateway/packet_reader.py b/data_gateway/packet_reader.py index 1a464614..f67520dd 100644 --- a/data_gateway/packet_reader.py +++ b/data_gateway/packet_reader.py @@ -48,7 +48,11 @@ def __init__( self.save_locally = save_locally self.upload_to_cloud = upload_to_cloud self.session_subdirectory = str(hash(datetime.datetime.now()))[1:7] - self.output_directory = os.path.abspath(os.path.join(output_directory, self.session_subdirectory)) + + self.cloud_output_directory = storage.path.join(output_directory, self.session_subdirectory) + self.local_output_directory = os.path.abspath(os.path.join(output_directory, self.session_subdirectory)) + os.makedirs(self.local_output_directory, exist_ok=True) + self.window_size = window_size self.project_name = project_name self.bucket_name = bucket_name @@ -61,7 +65,6 @@ def __init__( self.sleep = False self.sensor_time_offset = None - os.makedirs(self.output_directory, exist_ok=True) logger.warning("Timestamp synchronisation unavailable with current hardware; defaulting to using system clock.") def read_packets(self, serial_port, packet_queue, stop_signal): @@ -129,7 +132,7 @@ def parse_packets(self, packet_queue, stop_signal, stop_when_no_more_data_after= project_name=self.project_name, bucket_name=self.bucket_name, window_size=self.window_size, - output_directory=self.output_directory, + output_directory=self.cloud_output_directory, metadata={"data_gateway__configuration": self.config.to_dict()}, ) else: @@ -139,7 +142,7 @@ def parse_packets(self, packet_queue, stop_signal, stop_when_no_more_data_after= self.writer = BatchingFileWriter( sensor_names=self.config.sensor_names, window_size=self.window_size, - output_directory=self.output_directory, + output_directory=self.local_output_directory, save_csv_files=self.save_csv_files, ) else: @@ -253,14 +256,14 @@ def _persist_configuration(self): configuration_dictionary = self.config.to_dict() if self.save_locally: - with open(os.path.join(self.output_directory, "configuration.json"), "w") as f: + with open(os.path.join(self.local_output_directory, "configuration.json"), "w") as f: json.dump(configuration_dictionary, f) if self.upload_to_cloud: self.uploader.client.upload_from_string( string=json.dumps(configuration_dictionary), bucket_name=self.uploader.bucket_name, - path_in_bucket=storage.path.join(self.output_directory, "configuration.json"), + path_in_bucket=storage.path.join(self.cloud_output_directory, "configuration.json"), ) def _parse_sensor_packet_data(self, packet_type, payload, data): diff --git a/tests/test_data_gateway/test_data_gateway.py b/tests/test_data_gateway/test_data_gateway.py index fb890899..ac6676bd 100644 --- a/tests/test_data_gateway/test_data_gateway.py +++ b/tests/test_data_gateway/test_data_gateway.py @@ -1,11 +1,9 @@ -import shutil - -import coolname import json import os -import tempfile +import shutil from unittest.mock import patch +import coolname from octue.cloud import storage from octue.cloud.storage.client import GoogleCloudStorageClient @@ -63,13 +61,13 @@ def test_configuration_file_is_persisted(self): data_gateway.start(stop_when_no_more_data_after=0.1) # Check configuration file is present and valid locally. - with open(os.path.join(data_gateway.packet_reader.output_directory, "configuration.json")) as f: + with open(os.path.join(data_gateway.packet_reader.local_output_directory, "configuration.json")) as f: Configuration.from_dict(json.load(f)) # Check configuration file is present and valid on the cloud. configuration = self.storage_client.download_as_string( bucket_name=TEST_BUCKET_NAME, - path_in_bucket=storage.path.join(data_gateway.packet_reader.output_directory, "configuration.json"), + path_in_bucket=storage.path.join(data_gateway.packet_reader.cloud_output_directory, "configuration.json"), ) Configuration.from_dict(json.loads(configuration)) @@ -92,10 +90,12 @@ def test_data_gateway_with_baros_p_sensor(self): ) data_gateway.start(stop_when_no_more_data_after=0.1) - self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=["Baros_P"]) + self._check_data_is_written_to_files( + data_gateway.packet_reader.local_output_directory, sensor_names=["Baros_P"] + ) self._check_windows_are_uploaded_to_cloud( - data_gateway.packet_reader.output_directory, + data_gateway.packet_reader.cloud_output_directory, sensor_names=["Baros_P"], number_of_windows_to_check=1, ) @@ -108,7 +108,6 @@ def test_data_gateway_with_baros_t_sensor(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[0]))) serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[1]))) - data_gateway = DataGateway( serial_port, save_locally=True, @@ -118,10 +117,12 @@ def test_data_gateway_with_baros_t_sensor(self): bucket_name=TEST_BUCKET_NAME, ) data_gateway.start(stop_when_no_more_data_after=0.1) - self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=["Baros_T"]) + self._check_data_is_written_to_files( + data_gateway.packet_reader.local_output_directory, sensor_names=["Baros_T"] + ) self._check_windows_are_uploaded_to_cloud( - data_gateway.packet_reader.output_directory, + data_gateway.packet_reader.cloud_output_directory, sensor_names=["Baros_T"], number_of_windows_to_check=1, ) @@ -134,7 +135,6 @@ def test_data_gateway_with_diff_baros_sensor(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[0]))) serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[1]))) - data_gateway = DataGateway( serial_port, save_locally=True, @@ -146,12 +146,12 @@ def test_data_gateway_with_diff_baros_sensor(self): data_gateway.start(stop_when_no_more_data_after=0.1) self._check_data_is_written_to_files( - data_gateway.packet_reader.output_directory, + data_gateway.packet_reader.local_output_directory, sensor_names=["Diff_Baros"], ) self._check_windows_are_uploaded_to_cloud( - data_gateway.packet_reader.output_directory, + data_gateway.packet_reader.cloud_output_directory, sensor_names=["Diff_Baros"], number_of_windows_to_check=1, ) @@ -164,7 +164,6 @@ def test_data_gateway_with_mic_sensor(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[0]))) serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[1]))) - data_gateway = DataGateway( serial_port, save_locally=True, @@ -174,10 +173,10 @@ def test_data_gateway_with_mic_sensor(self): bucket_name=TEST_BUCKET_NAME, ) data_gateway.start(stop_when_no_more_data_after=0.1) - self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=["Mics"]) + self._check_data_is_written_to_files(data_gateway.packet_reader.local_output_directory, sensor_names=["Mics"]) self._check_windows_are_uploaded_to_cloud( - data_gateway.packet_reader.output_directory, + data_gateway.packet_reader.cloud_output_directory, sensor_names=["Mics"], number_of_windows_to_check=1, ) @@ -190,7 +189,6 @@ def test_data_gateway_with_acc_sensor(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[0]))) serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[1]))) - data_gateway = DataGateway( serial_port, save_locally=True, @@ -200,10 +198,11 @@ def test_data_gateway_with_acc_sensor(self): bucket_name=TEST_BUCKET_NAME, ) data_gateway.start(stop_when_no_more_data_after=0.1) - self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=["Acc"]) + + self._check_data_is_written_to_files(data_gateway.packet_reader.local_output_directory, sensor_names=["Acc"]) self._check_windows_are_uploaded_to_cloud( - data_gateway.packet_reader.output_directory, + data_gateway.packet_reader.cloud_output_directory, sensor_names=["Acc"], number_of_windows_to_check=1, ) @@ -216,7 +215,6 @@ def test_data_gateway_with_gyro_sensor(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[0]))) serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[1]))) - data_gateway = DataGateway( serial_port, save_locally=True, @@ -226,10 +224,11 @@ def test_data_gateway_with_gyro_sensor(self): bucket_name=TEST_BUCKET_NAME, ) data_gateway.start(stop_when_no_more_data_after=0.1) - self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=["Gyro"]) + + self._check_data_is_written_to_files(data_gateway.packet_reader.local_output_directory, sensor_names=["Gyro"]) self._check_windows_are_uploaded_to_cloud( - data_gateway.packet_reader.output_directory, + data_gateway.packet_reader.cloud_output_directory, sensor_names=["Gyro"], number_of_windows_to_check=1, ) @@ -242,7 +241,6 @@ def test_data_gateway_with_mag_sensor(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[0]))) serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[1]))) - data_gateway = DataGateway( serial_port, save_locally=True, @@ -252,10 +250,10 @@ def test_data_gateway_with_mag_sensor(self): bucket_name=TEST_BUCKET_NAME, ) data_gateway.start(stop_when_no_more_data_after=0.1) - self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=["Mag"]) + self._check_data_is_written_to_files(data_gateway.packet_reader.local_output_directory, sensor_names=["Mag"]) self._check_windows_are_uploaded_to_cloud( - data_gateway.packet_reader.output_directory, + data_gateway.packet_reader.cloud_output_directory, sensor_names=["Mag"], number_of_windows_to_check=1, ) @@ -268,7 +266,6 @@ def test_data_gateway_with_connections_statistics(self): serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[0]))) serial_port.write(data=b"".join((PACKET_KEY, packet_type, LENGTH, RANDOM_BYTES[1]))) - data_gateway = DataGateway( serial_port, save_locally=True, @@ -279,10 +276,12 @@ def test_data_gateway_with_connections_statistics(self): ) data_gateway.start(stop_when_no_more_data_after=0.1) - self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=["Constat"]) + self._check_data_is_written_to_files( + data_gateway.packet_reader.local_output_directory, sensor_names=["Constat"] + ) self._check_windows_are_uploaded_to_cloud( - data_gateway.packet_reader.output_directory, + data_gateway.packet_reader.cloud_output_directory, sensor_names=["Constat"], number_of_windows_to_check=1, ) @@ -314,7 +313,9 @@ def test_data_gateway_with_connections_statistics_in_sleep_mode(self): with patch("data_gateway.packet_reader.logger") as mock_logger: data_gateway.start(stop_when_no_more_data_after=0.1) - self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=["Constat"]) + self._check_data_is_written_to_files( + data_gateway.packet_reader.local_output_directory, sensor_names=["Constat"] + ) self.assertEqual(0, mock_logger.warning.call_count) def test_all_sensors_together(self): @@ -337,10 +338,12 @@ def test_all_sensors_together(self): ) data_gateway.start(stop_when_no_more_data_after=0.1) - self._check_data_is_written_to_files(data_gateway.packet_reader.output_directory, sensor_names=sensor_names) + self._check_data_is_written_to_files( + data_gateway.packet_reader.local_output_directory, sensor_names=sensor_names + ) self._check_windows_are_uploaded_to_cloud( - data_gateway.packet_reader.output_directory, + data_gateway.packet_reader.cloud_output_directory, sensor_names=sensor_names, number_of_windows_to_check=1, ) From 7ff1360510204dadf490b0a7cd30f0819d12e3b8 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 15 Feb 2022 12:15:22 +0000 Subject: [PATCH 69/84] REF: Factor out sending of stop signal --- data_gateway/__init__.py | 11 +++++++++++ data_gateway/data_gateway.py | 4 ++-- data_gateway/packet_reader.py | 8 +++----- data_gateway/routine.py | 8 ++++---- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/data_gateway/__init__.py b/data_gateway/__init__.py index 5636c833..1cccc635 100644 --- a/data_gateway/__init__.py +++ b/data_gateway/__init__.py @@ -3,3 +3,14 @@ __all__ = ("exceptions",) MICROPHONE_SENSOR_NAME = "Mics" + + +def stop_gateway(logger, stop_signal): + """Stop the gateway's multiple processes by sending the stop signal. + + :param logging.Logger logger: a logger to log that the stop signal has been sent + :param multiprocessing.Value stop_signal: a value of 0 means don't stop; a value of 1 means stop + :return None: + """ + logger.info("Sending stop signal.") + stop_signal.value = 1 diff --git a/data_gateway/data_gateway.py b/data_gateway/data_gateway.py index f009ddf3..024c0daf 100644 --- a/data_gateway/data_gateway.py +++ b/data_gateway/data_gateway.py @@ -8,6 +8,7 @@ import serial from octue.log_handlers import apply_log_handler +from data_gateway import stop_gateway from data_gateway.configuration import Configuration from data_gateway.dummy_serial import DummySerial from data_gateway.exceptions import DataMustBeSavedError @@ -236,8 +237,7 @@ def _send_commands_from_stdin_to_sensors(self, stop_signal): time.sleep(int(line.split(" ")[-1].strip())) elif line == "stop\n": self.serial_port.write(line.encode("utf_8")) - logger.info("Sending stop signal.") - stop_signal.value = 1 + stop_gateway(logger, stop_signal) break # Send the command to the node diff --git a/data_gateway/packet_reader.py b/data_gateway/packet_reader.py index f67520dd..4d03e051 100644 --- a/data_gateway/packet_reader.py +++ b/data_gateway/packet_reader.py @@ -7,7 +7,7 @@ from octue.cloud import storage -from data_gateway import MICROPHONE_SENSOR_NAME, exceptions +from data_gateway import MICROPHONE_SENSOR_NAME, exceptions, stop_gateway from data_gateway.configuration import Configuration from data_gateway.persistence import ( DEFAULT_OUTPUT_DIRECTORY, @@ -111,8 +111,7 @@ def read_packets(self, serial_port, packet_queue, stop_signal): pass finally: - logger.info("Sending stop signal.") - stop_signal.value = 1 + stop_gateway(logger, stop_signal) def parse_packets(self, packet_queue, stop_signal, stop_when_no_more_data_after=False): """Get packets from a thread-safe packet queue, check if a full payload has been received (i.e. correct length) @@ -213,8 +212,7 @@ def parse_packets(self, packet_queue, stop_signal, stop_when_no_more_data_after= pass finally: - logger.info("Sending stop signal.") - stop_signal.value = 1 + stop_gateway(logger, stop_signal) def update_handles(self, payload): """Update the Bluetooth handles object. Handles are updated every time a new Bluetooth connection is diff --git a/data_gateway/routine.py b/data_gateway/routine.py index eacce402..b2ef7c32 100644 --- a/data_gateway/routine.py +++ b/data_gateway/routine.py @@ -2,6 +2,8 @@ import sched import time +from data_gateway import stop_gateway + logger = multiprocessing.get_logger() @@ -55,8 +57,7 @@ def run(self, stop_signal): scheduler.run(blocking=True) if self.period is None: - logger.info("Sending stop signal.") - stop_signal.value = 1 + logger.info("Routine finished.") return elapsed_time = time.perf_counter() - cycle_start_time @@ -64,8 +65,7 @@ def run(self, stop_signal): if self.stop_after: if time.perf_counter() - start_time >= self.stop_after: - logger.info("Sending stop signal.") - stop_signal.value = 1 + stop_gateway(logger, stop_signal) return def _wrap_action_with_logger(self, action): From dbc00756fcdcd856bb0c9cf171e9873392b4edb4 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 15 Feb 2022 12:26:16 +0000 Subject: [PATCH 70/84] FIX: Ensure routines only run until stop command --- data_gateway/routine.py | 5 +++ tests/test_data_gateway/test_routine.py | 48 ++++++++++++++++++++----- 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/data_gateway/routine.py b/data_gateway/routine.py index b2ef7c32..f17203e5 100644 --- a/data_gateway/routine.py +++ b/data_gateway/routine.py @@ -54,6 +54,11 @@ def run(self, stop_signal): for command, delay in self.commands: scheduler.enter(delay=delay, priority=1, action=self.action, argument=(command,)) + # If a command is "stop", schedule stopping the gateway and then schedule no further commands. + if command == "stop": + scheduler.enter(delay=delay, priority=1, action=stop_gateway, argument=(logger, stop_signal)) + break + scheduler.run(blocking=True) if self.period is None: diff --git a/tests/test_data_gateway/test_routine.py b/tests/test_data_gateway/test_routine.py index d668a6c3..df315f27 100644 --- a/tests/test_data_gateway/test_routine.py +++ b/tests/test_data_gateway/test_routine.py @@ -6,13 +6,24 @@ from data_gateway.routine import Routine +def create_record_commands_action(): + """Create a list in which commands will be recorded when the `recorded_command` function is given as an action to + the routine. + + :return (list, callable): the list that actions are recorded in, and the function that causes them to be recorded + """ + recorded_commands = [] + + def record_commands(command): + recorded_commands.append((command, time.perf_counter())) + + return recorded_commands, record_commands + + class TestRoutine(TestCase): def test_routine_with_no_period_runs_commands_once(self): """Test that commands can be scheduled to run once when a period isn't given.""" - recorded_commands = [] - - def record_commands(command): - recorded_commands.append((command, time.perf_counter())) + recorded_commands, record_commands = create_record_commands_action() routine = Routine( commands=[("first-command", 0.1), ("second-command", 0.3)], @@ -63,10 +74,7 @@ def test_warning_raised_if_stop_after_time_provided_without_a_period(self): def test_routine_with_period(self): """Test that commands can be scheduled to repeat at the given period and then stop after a certain time.""" - recorded_commands = [] - - def record_commands(command): - recorded_commands.append((command, time.perf_counter())) + recorded_commands, record_commands = create_record_commands_action() routine = Routine( commands=[("first-command", 0.1), ("second-command", 0.3)], @@ -89,3 +97,27 @@ def record_commands(command): self.assertEqual(recorded_commands[3][0], "second-command") self.assertAlmostEqual(recorded_commands[3][1], start_time + 0.3 + routine.period, delta=0.2) + + def test_routine_only_runs_until_stop_command(self): + """Test that a routine only runs until the "stop" command is received.""" + recorded_commands, record_commands = create_record_commands_action() + + routine = Routine( + commands=[("first-command", 0.1), ("stop", 0.3), ("command-after-stop", 0.5)], + action=record_commands, + ) + + stop_signal = multiprocessing.Value("i", 0) + start_time = time.perf_counter() + + routine.run(stop_signal=stop_signal) + + self.assertEqual(len(recorded_commands), 2) + + self.assertEqual(recorded_commands[0][0], "first-command") + self.assertAlmostEqual(recorded_commands[0][1], start_time + 0.1, delta=0.2) + + self.assertEqual(recorded_commands[1][0], "stop") + self.assertAlmostEqual(recorded_commands[1][1], start_time + 0.3, delta=0.2) + + self.assertEqual(stop_signal.value, 1) From c868708f743926766335dcd0a32988a0cd4d93db Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 15 Feb 2022 12:29:59 +0000 Subject: [PATCH 71/84] FIX: Don't stop gateway when routine times out without stop command --- data_gateway/routine.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data_gateway/routine.py b/data_gateway/routine.py index f17203e5..942c62b1 100644 --- a/data_gateway/routine.py +++ b/data_gateway/routine.py @@ -62,7 +62,7 @@ def run(self, stop_signal): scheduler.run(blocking=True) if self.period is None: - logger.info("Routine finished.") + logger.info("Non-periodic routine finished.") return elapsed_time = time.perf_counter() - cycle_start_time @@ -70,7 +70,7 @@ def run(self, stop_signal): if self.stop_after: if time.perf_counter() - start_time >= self.stop_after: - stop_gateway(logger, stop_signal) + logger.info("Periodic routine stopped after given timeout of %ss.", self.stop_after) return def _wrap_action_with_logger(self, action): From 8196e1acdf213d086cd75580438177a081690b67 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 15 Feb 2022 12:43:15 +0000 Subject: [PATCH 72/84] TST: Improve test documentation --- tests/test_data_gateway/test_routine.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_data_gateway/test_routine.py b/tests/test_data_gateway/test_routine.py index df315f27..51144f15 100644 --- a/tests/test_data_gateway/test_routine.py +++ b/tests/test_data_gateway/test_routine.py @@ -7,7 +7,7 @@ def create_record_commands_action(): - """Create a list in which commands will be recorded when the `recorded_command` function is given as an action to + """Create a list in which commands will be recorded when the `record_commands` function is given as an action to the routine. :return (list, callable): the list that actions are recorded in, and the function that causes them to be recorded @@ -112,6 +112,7 @@ def test_routine_only_runs_until_stop_command(self): routine.run(stop_signal=stop_signal) + # Check that only the first two commands (i.e. up until the `stop` command) are scheduled and carried out. self.assertEqual(len(recorded_commands), 2) self.assertEqual(recorded_commands[0][0], "first-command") @@ -120,4 +121,5 @@ def test_routine_only_runs_until_stop_command(self): self.assertEqual(recorded_commands[1][0], "stop") self.assertAlmostEqual(recorded_commands[1][1], start_time + 0.3, delta=0.2) + # Check that the stop signal has been sent. self.assertEqual(stop_signal.value, 1) From 21b20544cab72d820f345174ba30485010124a25 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 15 Feb 2022 14:00:12 +0000 Subject: [PATCH 73/84] ENH: Update PacketReader logs; stop re-open of serial port on overflow --- data_gateway/packet_reader.py | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/data_gateway/packet_reader.py b/data_gateway/packet_reader.py index 4d03e051..aa0f2c0b 100644 --- a/data_gateway/packet_reader.py +++ b/data_gateway/packet_reader.py @@ -21,7 +21,8 @@ class PacketReader: - """A serial port packet reader. + """A serial port packet reader. Note that timestamp synchronisation is unavailable with the current sensor hardware + so the system clock is used instead. :param bool save_locally: save data windows locally :param bool upload_to_cloud: upload data windows to Google cloud @@ -65,8 +66,6 @@ def __init__( self.sleep = False self.sensor_time_offset = None - logger.warning("Timestamp synchronisation unavailable with current hardware; defaulting to using system clock.") - def read_packets(self, serial_port, packet_queue, stop_signal): """Read packets from a serial port and send them to the parser thread for processing and persistence. @@ -75,7 +74,7 @@ def read_packets(self, serial_port, packet_queue, stop_signal): :return None: """ try: - logger.info("Beginning reading packets from serial port.") + logger.info("Packet reader process started.") while stop_signal.value == 0: serial_data = serial_port.read() @@ -89,7 +88,6 @@ def read_packets(self, serial_port, packet_queue, stop_signal): packet_type = str(int.from_bytes(serial_port.read(), self.config.endian)) length = int.from_bytes(serial_port.read(), self.config.endian) packet = serial_port.read(length) - logger.debug("Read packet from serial port.") if packet_type == str(self.config.type_handle_def): self.update_handles(packet) @@ -97,12 +95,7 @@ def read_packets(self, serial_port, packet_queue, stop_signal): # Check for bytes in serial input buffer. A full buffer results in overflow. if serial_port.in_waiting == self.config.serial_buffer_rx_size: - logger.warning( - "Buffer is full: %d bytes waiting. Re-opening serial port, to avoid overflow", - serial_port.in_waiting, - ) - serial_port.close() - serial_port.open() + logger.warning("Serial port buffer is full - buffer overflow may occur, resulting in data loss.") continue packet_queue.put({"packet_type": packet_type, "packet": packet}) @@ -123,7 +116,7 @@ def parse_packets(self, packet_queue, stop_signal, stop_when_no_more_data_after= :param float|bool stop_when_no_more_data_after: the number of seconds after receiving no data to stop the gateway (mainly for testing); if `False`, no limit is applied :return None: """ - logger.info("Beginning parsing packets from serial port.") + logger.info("Packet parser process started.") if self.upload_to_cloud: self.uploader = BatchingUploader( @@ -175,8 +168,6 @@ def parse_packets(self, packet_queue, stop_signal, stop_when_no_more_data_after= break continue - logger.debug("Received packet for parsing.") - if packet_type not in self.handles: logger.error("Received packet with unknown type: %s", packet_type) continue @@ -244,7 +235,7 @@ def update_handles(self, payload): logger.info("Successfully updated handles.") return - logger.error("Handle error: %s %s", start_handle, end_handle) + logger.error("Handle error: start handle is %s, end handle is %s.", start_handle, end_handle) def _persist_configuration(self): """Persist the configuration to disk and/or cloud storage. @@ -470,7 +461,7 @@ def _check_for_packet_loss(self, sensor_name, timestamp, previous_timestamp): return if previous_timestamp[sensor_name] == -1: - logger.info("Received first %s packet" % sensor_name) + logger.info("Received first %s packet." % sensor_name) else: expected_current_timestamp = ( previous_timestamp[sensor_name] From 290b3c9ffb1f7b3e9e50cb428bbc4551a0caf934 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 15 Feb 2022 15:51:32 +0000 Subject: [PATCH 74/84] ENH: Adjust DataGateway logging; stop if error in commands thread --- data_gateway/data_gateway.py | 70 +++++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 28 deletions(-) diff --git a/data_gateway/data_gateway.py b/data_gateway/data_gateway.py index 024c0daf..650ee2cd 100644 --- a/data_gateway/data_gateway.py +++ b/data_gateway/data_gateway.py @@ -21,19 +21,26 @@ class DataGateway: - """A class for running the data gateway for collecting wind turbine sensor data. The gateway is run with multiple - threads reading from the serial port which put the packets they read into a queue for a single parser thread to - process and persist. An additional thread is run for sending commands to the sensors either interactively or via a - routine. If a "stop" signal is sent as a command, all threads are stopped and any data in the current window is - persisted. - - :param str|serial.Serial serial_port: the name of the serial port to use or a `serial.Serial` instance - :param str configuration_path: the path to a JSON configuration file for reading and parsing data + """A class for running the data gateway to collect wind turbine sensor data. The gateway is run as three processes: + 1. The `MainProcess` process, which starts the other two processes and sends commands to the serial port (via a + separate thread) interactively or through a routine + 2. The `Reader` process, which reads packets from the serial port and puts them on a queue + 3. The `Parser` process, which takes packets off the queue, parses them, and persists them + + All processes and threads are stopped and any data in the current window is persisted if: + - A "stop" signal is sent as a command interactively or in a routine + - An error is raised in any process or thread + - A `KeyboardInterrupt` is raised (i.e. the user presses `Ctrl + C`) + - No more data is received by the `Parser` process after `stop_when_no_more_data_after` seconds (if it is set in the + `DataGateway.run` method) + + :param str|serial.Serial serial_port: the name of the serial port or a `serial.Serial` instance to read from + :param str configuration_path: the path to a JSON configuration file for the packet reader :param str routine_path: the path to a JSON routine file containing sensor commands to be run automatically - :param bool save_locally: if `True`, save data windows locally + :param bool save_locally: if `True`, save data windows to disk locally :param bool upload_to_cloud: if `True`, upload data windows to Google Cloud Storage :param bool interactive: if `True`, allow commands entered into `stdin` to be sent to the sensors in real time - :param str output_directory: the directory in which to save data in the cloud bucket or local file system + :param str output_directory: the name of the directory in which to save data in the cloud bucket or local file system :param float window_size: the period in seconds at which data is persisted :param str|None project_name: the name of the Google Cloud project to upload to :param str|None bucket_name: the name of the Google Cloud bucket to upload to @@ -61,8 +68,8 @@ def __init__( ): if not save_locally and not upload_to_cloud: raise DataMustBeSavedError( - "Data from the gateway must either be saved locally or uploaded to the cloud. Please adjust the CLI " - "options provided." + "Data from the gateway must either be saved locally or uploaded to the cloud. Please adjust the " + "parameters provided." ) self.interactive = interactive @@ -97,7 +104,6 @@ def start(self, stop_when_no_more_data_after=False): :param float|bool stop_when_no_more_data_after: the number of seconds after receiving no data to stop the gateway (mainly for testing); if `False`, no limit is applied :return None: """ - logger.info("Starting data gateway.") packet_queue = multiprocessing.Queue() stop_signal = multiprocessing.Value("i", 0) @@ -150,7 +156,7 @@ def start(self, stop_when_no_more_data_after=False): time.sleep(5) def _load_configuration(self, configuration_path): - """Load a configuration from the path if it exists, otherwise load the default configuration. + """Load a configuration from the path if it exists; otherwise load the default configuration. :param str configuration_path: path to the configuration JSON file :return data_gateway.configuration.Configuration: @@ -159,11 +165,11 @@ def _load_configuration(self, configuration_path): with open(configuration_path) as f: configuration = Configuration.from_dict(json.load(f)) - logger.debug("Loaded configuration file from %r.", configuration_path) + logger.info("Loaded configuration file from %r.", configuration_path) return configuration configuration = Configuration() - logger.debug("No configuration file provided - using default configuration.") + logger.info("No configuration file provided - using default configuration.") return configuration def _get_serial_port(self, serial_port, configuration, use_dummy_serial_port): @@ -211,7 +217,7 @@ def _load_routine(self, routine_path): action=lambda command: self.serial_port.write((command + "\n").encode("utf_8")), ) - logger.debug("Loaded routine file from %r.", routine_path) + logger.info("Loaded routine file from %r.", routine_path) return routine logger.debug( @@ -228,17 +234,25 @@ def _send_commands_from_stdin_to_sensors(self, stop_signal): """ commands_record_file = os.path.join(self.packet_reader.local_output_directory, "commands.txt") - while stop_signal.value == 0: - for line in sys.stdin: - with open(commands_record_file, "a") as f: - f.write(line) + try: + while stop_signal.value == 0: + for line in sys.stdin: + with open(commands_record_file, "a") as f: + f.write(line) + + # The `sleep` command is mainly for facilitating testing. + if line.startswith("sleep") and line.endswith("\n"): + time.sleep(int(line.split(" ")[-1].strip())) + continue + + if line == "stop\n": + self.serial_port.write(line.encode("utf_8")) + stop_gateway(logger, stop_signal) + break - if line.startswith("sleep") and line.endswith("\n"): - time.sleep(int(line.split(" ")[-1].strip())) - elif line == "stop\n": + # Send the command to the node. self.serial_port.write(line.encode("utf_8")) - stop_gateway(logger, stop_signal) - break - # Send the command to the node - self.serial_port.write(line.encode("utf_8")) + except Exception as e: + stop_gateway(logger, stop_signal) + raise e From c264f7aeb44d9ac1ba0528f53555b0f639cff982 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 15 Feb 2022 15:53:15 +0000 Subject: [PATCH 75/84] ENH: Stop the gateway if there's an error in a Routine execution --- data_gateway/routine.py | 50 ++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/data_gateway/routine.py b/data_gateway/routine.py index 942c62b1..1d874ed1 100644 --- a/data_gateway/routine.py +++ b/data_gateway/routine.py @@ -41,37 +41,45 @@ def __init__(self, commands, action, period=None, stop_after=None): logger.warning("The `stop_after` parameter is ignored unless `period` is also given.") def run(self, stop_signal): - """Send the commands to the action after the given delays, repeating if a period was given. + """Send the commands to the action after the given delays, repeating if a period was given. The routine will + stop before the next run if the stop signal is received (i.e. if the `stop_signal.value` is set to 1 in another + process). + :param multiprocessing.Value stop_signal: a value of 0 means don't stop; a value of 1 means stop :return None: """ - scheduler = sched.scheduler(time.perf_counter) - start_time = time.perf_counter() + try: + scheduler = sched.scheduler(time.perf_counter) + start_time = time.perf_counter() - while stop_signal.value == 0: - cycle_start_time = time.perf_counter() + while stop_signal.value == 0: + cycle_start_time = time.perf_counter() - for command, delay in self.commands: - scheduler.enter(delay=delay, priority=1, action=self.action, argument=(command,)) + for command, delay in self.commands: + scheduler.enter(delay=delay, priority=1, action=self.action, argument=(command,)) - # If a command is "stop", schedule stopping the gateway and then schedule no further commands. - if command == "stop": - scheduler.enter(delay=delay, priority=1, action=stop_gateway, argument=(logger, stop_signal)) - break + # If a command is "stop", schedule stopping the gateway and then schedule no further commands. + if command == "stop": + scheduler.enter(delay=delay, priority=1, action=stop_gateway, argument=(logger, stop_signal)) + break - scheduler.run(blocking=True) + scheduler.run(blocking=True) - if self.period is None: - logger.info("Non-periodic routine finished.") - return + if self.period is None: + logger.info("Non-periodic routine finished.") + return - elapsed_time = time.perf_counter() - cycle_start_time - time.sleep(self.period - elapsed_time) + elapsed_time = time.perf_counter() - cycle_start_time + time.sleep(self.period - elapsed_time) - if self.stop_after: - if time.perf_counter() - start_time >= self.stop_after: - logger.info("Periodic routine stopped after given timeout of %ss.", self.stop_after) - return + if self.stop_after: + if time.perf_counter() - start_time >= self.stop_after: + logger.info("Periodic routine stopped after given timeout of %ss.", self.stop_after) + return + + except Exception as e: + stop_gateway(logger, stop_signal) + raise e def _wrap_action_with_logger(self, action): """Wrap the given action so that when it's run on a command, the command is logged. From 1c04e10e9004f5e16bb2badaa92d403ff89fa447 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 15 Feb 2022 15:56:22 +0000 Subject: [PATCH 76/84] TST: Update test --- tests/test_data_gateway/test_packet_reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_data_gateway/test_packet_reader.py b/tests/test_data_gateway/test_packet_reader.py index 626f181c..e6844d42 100644 --- a/tests/test_data_gateway/test_packet_reader.py +++ b/tests/test_data_gateway/test_packet_reader.py @@ -26,7 +26,7 @@ def test_error_is_logged_if_unknown_sensor_type_packet_is_received(self): stop_when_no_more_data_after=0.1, ) - self.assertIn("Received packet with unknown type: ", mock_logger.method_calls[2].args[0]) + self.assertIn("Received packet with unknown type: ", mock_logger.method_calls[1].args[0]) def test_update_handles_fails_if_start_and_end_handles_are_incorrect(self): """Test that an error is raised if the start and end handles are incorrect when trying to update handles.""" From 12fd49441fe30705a1367334f65783ccb818ce82 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 15 Feb 2022 16:01:51 +0000 Subject: [PATCH 77/84] ENH: Enforce existing files for relevant options in CLI --- data_gateway/cli.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/data_gateway/cli.py b/data_gateway/cli.py index be33c49d..bdbc996f 100644 --- a/data_gateway/cli.py +++ b/data_gateway/cli.py @@ -60,14 +60,14 @@ def gateway_cli(logger_uri, log_level): ) @click.option( "--config-file", - type=click.Path(dir_okay=False), + type=click.Path(dir_okay=False, exists=True), default="config.json", show_default=True, help="Path to your Aerosense deployment configuration JSON file.", ) @click.option( "--routine-file", - type=click.Path(dir_okay=False), + type=click.Path(dir_okay=False, exists=True), default="routine.json", show_default=True, help="Path to sensor command routine JSON file.", @@ -181,7 +181,7 @@ def start( @gateway_cli.command() @click.option( "--config-file", - type=click.Path(), + type=click.Path(dir_okay=False, exists=True), default="configuration.json", help="A path to a JSON configuration file.", ) @@ -243,7 +243,7 @@ def create_installation(config_file): @gateway_cli.command() @click.option( "--config-file", - type=click.Path(), + type=click.Path(dir_okay=False, exists=True), default="config.json", show_default=True, help="Path to your Aerosense deployment configuration file.", From 3c64b579de15cd0fda1feca1d9acd20feed28bfe Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 15 Feb 2022 16:04:46 +0000 Subject: [PATCH 78/84] ENH: Log a warning if no routine is provided in non-interactive mode --- data_gateway/data_gateway.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/data_gateway/data_gateway.py b/data_gateway/data_gateway.py index 650ee2cd..20ac9761 100644 --- a/data_gateway/data_gateway.py +++ b/data_gateway/data_gateway.py @@ -210,20 +210,21 @@ def _load_routine(self, routine_path): if self.interactive: logger.warning("Sensor command routine files are ignored in interactive mode.") return - else: - with open(routine_path) as f: - routine = Routine( - **json.load(f), - action=lambda command: self.serial_port.write((command + "\n").encode("utf_8")), - ) - - logger.info("Loaded routine file from %r.", routine_path) - return routine - - logger.debug( - "No routine file found at %r - no commands will be sent to the sensors unless given in interactive mode.", - routine_path, - ) + + with open(routine_path) as f: + routine = Routine( + **json.load(f), + action=lambda command: self.serial_port.write((command + "\n").encode("utf_8")), + ) + + logger.info("Loaded routine file from %r.", routine_path) + return routine + + if not self.interactive: + logger.warning( + "No routine was provided and interactive mode is off - no commands will be sent to the sensors in this " + "session." + ) def _send_commands_from_stdin_to_sensors(self, stop_signal): """Send commands from `stdin` to the sensors until the "stop" command is received or the packet reader is From 3e8542dd1fc905819e0ed244c1631b37ea0eb4e4 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 15 Feb 2022 16:35:02 +0000 Subject: [PATCH 79/84] REV: Revert "ENH: Enforce existing files for relevant options in CLI This reverts commit 12fd49441fe30705a1367334f65783ccb818ce82. --- data_gateway/cli.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/data_gateway/cli.py b/data_gateway/cli.py index bdbc996f..be33c49d 100644 --- a/data_gateway/cli.py +++ b/data_gateway/cli.py @@ -60,14 +60,14 @@ def gateway_cli(logger_uri, log_level): ) @click.option( "--config-file", - type=click.Path(dir_okay=False, exists=True), + type=click.Path(dir_okay=False), default="config.json", show_default=True, help="Path to your Aerosense deployment configuration JSON file.", ) @click.option( "--routine-file", - type=click.Path(dir_okay=False, exists=True), + type=click.Path(dir_okay=False), default="routine.json", show_default=True, help="Path to sensor command routine JSON file.", @@ -181,7 +181,7 @@ def start( @gateway_cli.command() @click.option( "--config-file", - type=click.Path(dir_okay=False, exists=True), + type=click.Path(), default="configuration.json", help="A path to a JSON configuration file.", ) @@ -243,7 +243,7 @@ def create_installation(config_file): @gateway_cli.command() @click.option( "--config-file", - type=click.Path(dir_okay=False, exists=True), + type=click.Path(), default="config.json", show_default=True, help="Path to your Aerosense deployment configuration file.", From 43cb4a4c2f85e14c7e4801731c3879b3503e2f41 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 15 Feb 2022 17:04:34 +0000 Subject: [PATCH 80/84] FIX: Pass log level to multiprocessing logger --- data_gateway/cli.py | 8 +++++++- data_gateway/data_gateway.py | 9 +++++++++ tests/test_data_gateway/test_cli.py | 6 +++--- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/data_gateway/cli.py b/data_gateway/cli.py index be33c49d..7e1efdd0 100644 --- a/data_gateway/cli.py +++ b/data_gateway/cli.py @@ -15,6 +15,8 @@ SUPERVISORD_PROGRAM_NAME = "AerosenseGateway" CREATE_INSTALLATION_CLOUD_FUNCTION_URL = "https://europe-west6-aerosense-twined.cloudfunctions.net/create-installation" +global_cli_context = {} + logger = multiprocessing.get_logger() @@ -40,12 +42,15 @@ def gateway_cli(logger_uri, log_level): """ from octue.log_handlers import apply_log_handler, get_remote_handler + # Store log level to apply to multi-processed logger in `DataGateway` in the `start` command. + global_cli_context["log_level"] = log_level.upper() + # Stream logs to remote handler if required. if logger_uri is not None: apply_log_handler( logger=logger, handler=get_remote_handler(logger_uri=logger_uri), - log_level=log_level.upper(), + log_level=global_cli_context["log_level"], include_process_name=True, ) @@ -173,6 +178,7 @@ def start( label=label, save_csv_files=save_csv_files, use_dummy_serial_port=use_dummy_serial_port, + log_level=global_cli_context["log_level"], ) data_gateway.start() diff --git a/data_gateway/data_gateway.py b/data_gateway/data_gateway.py index 20ac9761..0257d5ed 100644 --- a/data_gateway/data_gateway.py +++ b/data_gateway/data_gateway.py @@ -1,4 +1,5 @@ import json +import logging import multiprocessing import os import sys @@ -19,6 +20,9 @@ logger = multiprocessing.get_logger() apply_log_handler(logger=logger, include_process_name=True) +# Ignore logs from the dummy serial port. +logging.getLogger("data_gateway.dummy_serial.dummy_serial").setLevel(logging.WARNING) + class DataGateway: """A class for running the data gateway to collect wind turbine sensor data. The gateway is run as three processes: @@ -65,6 +69,7 @@ def __init__( label=None, save_csv_files=False, use_dummy_serial_port=False, + log_level=logging.INFO, ): if not save_locally and not upload_to_cloud: raise DataMustBeSavedError( @@ -96,6 +101,10 @@ def __init__( self.routine = self._load_routine(routine_path=routine_path) + logger.setLevel(log_level) + for handler in logger.handlers: + handler.setLevel(log_level) + def start(self, stop_when_no_more_data_after=False): """Begin reading and persisting data from the serial port for the sensors at the installation defined in the configuration. In interactive mode, commands can be sent to the nodes/sensors via the serial port by typing diff --git a/tests/test_data_gateway/test_cli.py b/tests/test_data_gateway/test_cli.py index 3e23d7c3..170f210f 100644 --- a/tests/test_data_gateway/test_cli.py +++ b/tests/test_data_gateway/test_cli.py @@ -141,7 +141,7 @@ def test_with_routine(self): def test_log_level_can_be_set(self): """Test that the log level can be set.""" with tempfile.TemporaryDirectory() as temporary_directory: - with self.assertLogs(level="DEBUG") as mock_logger: + with mock.patch("octue.log_handlers.logging.StreamHandler.emit") as mock_emit: result = CliRunner().invoke( gateway_cli, [ @@ -161,8 +161,8 @@ def test_log_level_can_be_set(self): debug_message_found = False - for message in mock_logger.output: - if "DEBUG" in message: + for record in mock_emit.call_args_list: + if record.args[0].levelname == "DEBUG": debug_message_found = True break From 20f2718b8e1db388bb44cde1f74ada3853e390ae Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 15 Feb 2022 17:09:59 +0000 Subject: [PATCH 81/84] TST: Enable multiprocessing test coverage --- .coveragerc | 3 +++ tox.ini | 1 + 2 files changed, 4 insertions(+) diff --git a/.coveragerc b/.coveragerc index 3bd76b89..16f2cbe9 100644 --- a/.coveragerc +++ b/.coveragerc @@ -10,3 +10,6 @@ omit = venv/* tests/* */tests/* + +concurrency = multiprocessing +parallel = True diff --git a/tox.ini b/tox.ini index 5c13f9cc..49b4210e 100644 --- a/tox.ini +++ b/tox.ini @@ -7,6 +7,7 @@ setenv = PYTHONPATH = {toxinidir}:{toxinidir}/gateway commands = coverage run --source data_gateway -m unittest discover + coverage combine coverage report --show-missing coverage xml deps = -r requirements-dev.txt From 832d5e3bbcc37e4b4c0f0caa20c4a17ec780580c Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 15 Feb 2022 17:27:17 +0000 Subject: [PATCH 82/84] FIX: Set log level straight away in DataGateway --- data_gateway/data_gateway.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/data_gateway/data_gateway.py b/data_gateway/data_gateway.py index 0257d5ed..32a957be 100644 --- a/data_gateway/data_gateway.py +++ b/data_gateway/data_gateway.py @@ -71,6 +71,11 @@ def __init__( use_dummy_serial_port=False, log_level=logging.INFO, ): + # Set multiprocessed logger level. + logger.setLevel(log_level) + for handler in logger.handlers: + handler.setLevel(log_level) + if not save_locally and not upload_to_cloud: raise DataMustBeSavedError( "Data from the gateway must either be saved locally or uploaded to the cloud. Please adjust the " @@ -101,10 +106,6 @@ def __init__( self.routine = self._load_routine(routine_path=routine_path) - logger.setLevel(log_level) - for handler in logger.handlers: - handler.setLevel(log_level) - def start(self, stop_when_no_more_data_after=False): """Begin reading and persisting data from the serial port for the sensors at the installation defined in the configuration. In interactive mode, commands can be sent to the nodes/sensors via the serial port by typing From cd8efc81edf6c5890d8f9ed246726d4a67833cd9 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 15 Feb 2022 17:37:52 +0000 Subject: [PATCH 83/84] REV: Revert "TST: Enable multiprocessing test coverage" This reverts commit 20f2718b8e1db388bb44cde1f74ada3853e390ae. --- .coveragerc | 3 --- tox.ini | 1 - 2 files changed, 4 deletions(-) diff --git a/.coveragerc b/.coveragerc index 16f2cbe9..3bd76b89 100644 --- a/.coveragerc +++ b/.coveragerc @@ -10,6 +10,3 @@ omit = venv/* tests/* */tests/* - -concurrency = multiprocessing -parallel = True diff --git a/tox.ini b/tox.ini index 49b4210e..5c13f9cc 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,6 @@ setenv = PYTHONPATH = {toxinidir}:{toxinidir}/gateway commands = coverage run --source data_gateway -m unittest discover - coverage combine coverage report --show-missing coverage xml deps = -r requirements-dev.txt From c1439217fca0eb14d46f74eedc4d185e90a1ad40 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 15 Feb 2022 18:04:18 +0000 Subject: [PATCH 84/84] DEP: Use latest octue version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 97eca16e..d6248247 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ "click>=7.1.2", "pyserial==3.5", "python-slugify==5.0.2", - "octue @ https://github.com/octue/octue-sdk-python/archive/enhancement/allow-apply-log-handler-to-work-on-logger-instance.zip", + "octue==0.10.5", ], url="https://gitlab.com/windenergie-hsr/aerosense/digital-twin/data-gateway", license="MIT",