From 298bf423d2f406105780babe9bf068549bcb2473 Mon Sep 17 00:00:00 2001 From: Leigh Johnson Date: Sun, 7 Feb 2021 04:29:13 +0000 Subject: [PATCH 01/16] refactor WorkerManager to use mixins; update unit tests to pass against mixin format (in preparation for testing each mixin) --- octoprint_nanny/manager.py | 462 +----------------------------------- octoprint_nanny/settings.py | 191 +++++++++++++++ octoprint_nanny/workers.py | 312 ++++++++++++++++++++++++ tests/test_manager.py | 34 ++- 4 files changed, 536 insertions(+), 463 deletions(-) create mode 100644 octoprint_nanny/settings.py create mode 100644 octoprint_nanny/workers.py diff --git a/octoprint_nanny/manager.py b/octoprint_nanny/manager.py index 51cb7abf..bfb2f5f7 100644 --- a/octoprint_nanny/manager.py +++ b/octoprint_nanny/manager.py @@ -1,7 +1,6 @@ from datetime import datetime from time import sleep import aiohttp -import aiofiles import aioprocessing import asyncio import base64 @@ -19,201 +18,18 @@ import octoprint.filemanager from octoprint.events import Events -from octoprint_nanny.clients.websocket import WebSocketWorker -from octoprint_nanny.clients.rest import RestAPIClient, API_CLIENT_EXCEPTIONS -from octoprint_nanny.clients.mqtt import MQTTClient -from octoprint_nanny.predictor import ( - PredictWorker, - BOUNDING_BOX_PREDICT_EVENT, - ANNOTATED_IMAGE_EVENT, -) +from octoprint_nanny.clients.rest import API_CLIENT_EXCEPTIONS +from octoprint_nanny.workers import MultiWorkerMixin from octoprint_nanny.exceptions import PluginSettingsRequired from octoprint_nanny.clients.honeycomb import HoneycombTracer -import print_nanny_client import beeline -logger = logging.getLogger("octoprint.plugins.octoprint_nanny.manager") Events.PRINT_PROGRESS = "PrintProgress" -class PluginSettingsMemoizeMixin: - """ - Convenience methods/properties for accessing OctoPrint plugin settings and computed metadata - """ - - def __init__(self, plugin): - self.plugin = plugin - - # stateful clients and computed settings that require re-initialization when settings change - self._calibration = None - self._mqtt_client = None - self._telemetry_events = None - self._device_info = None - self._rest_client = None - - self.environment = {} - - @beeline.traced("PluginSettingsMemoize.reset_monitoring_settings") - def reset_monitoring_settings(self): - self._calibration = None - self._monitoring_frames_per_minute = None - - @beeline.traced("PluginSettingsMemoize.reset_device_settings_state") - @beeline.traced_thread - def reset_device_settings_state(self): - self._mqtt_client = None - self._device_info = None - - @beeline.traced("PluginSettingsMemoize.reset_rest_client_state") - @beeline.traced_thread - def reset_rest_client_state(self): - self._rest_client = None - - @beeline.traced(name="PluginSettingsMemoize.get_device_metadata") - @beeline.traced_thread - def get_device_metadata(self): - metadata = dict( - created_dt=datetime.now(pytz.timezone("America/Los_Angeles")), - environment=self.environment, - ) - metadata.update(self.device_info) - return metadata - - @beeline.traced(name="PluginSettingsMemoize.get_print_job_metadata") - @beeline.traced_thread - def get_print_job_metadata(self): - return dict( - printer_data=self.plugin._printer.get_current_data(), - printer_profile_data=self.plugin._printer_profile_manager.get_current_or_default(), - temperatures=self.plugin._printer.get_current_temperatures(), - printer_profile_id=self.shared.printer_profile_id, - print_job_id=self.shared.print_job_id, - ) - - @beeline.traced(name="PluginSettingsMemoize.on_environment_detected") - @beeline.traced_thread - def on_environment_detected(self, environment): - self.environment = environment - - @property - def device_cloudiot_name(self): - return self.plugin.get_setting("device_cloudiot_name") - - @property - def device_id(self): - return self.plugin.get_setting("device_id") - - @property - def device_info(self): - if self._device_info is None: - self._device_info = self.plugin.get_device_info() - return self._device_info - - @property - def device_serial(self): - return self.plugin.get_setting("device_serial") - - @property - def device_cloudiot_id(self): - return self.plugin.get_setting("device_cloudiot_id") - - @property - def device_private_key(self): - return self.plugin.get_setting("device_private_key") - - @property - def device_public_key(self): - return self.plugin.get_setting("device_public_key") - - @property - def gcp_root_ca(self): - return self.plugin.get_setting("gcp_root_ca") - - @property - def api_url(self): - return self.plugin.get_setting("api_url") - - @property - def auth_token(self): - return self.plugin.get_setting("auth_token") - - @property - def ws_url(self): - return self.plugin.get_setting("ws_url") - - @property - def snapshot_url(self): - return self.plugin.get_setting("snapshot_url") - - @property - def user_id(self): - return self.plugin.get_setting("user_id") - - @property - def calibration(self): - if self._calibration is None: - self._calibration = PredictWorker.calc_calibration( - self.plugin.get_setting("calibrate_x0"), - self.plugin.get_setting("calibrate_y0"), - self.plugin.get_setting("calibrate_x1"), - self.plugin.get_setting("calibrate_y1"), - ) - return self._calibration - - @property - def monitoring_frames_per_minute(self): - return self.plugin.get_setting("monitoring_frames_per_minute") - - @property - def rest_client(self): - if self.auth_token is None: - raise PluginSettingsRequired(f"auth_token is not set") - if self._rest_client is None: - self._rest_client = RestAPIClient( - auth_token=self.auth_token, api_url=self.api_url - ) - logger.info(f"RestAPIClient initialized with api_url={self.api_url}") - return self._rest_client - - def test_mqtt_settings(self): - if self.device_cloudiot_id is None or self.device_private_key is None: - raise PluginSettingsRequired( - f"Received None for device_cloudiot_id={self.device_cloudiot_id} or private_key_file={self.device_private_key}" - ) - return True - - @property - def mqtt_client(self): - self.test_mqtt_settings() - if self._mqtt_client is None: - self._mqtt_client = MQTTClient( - device_id=self.device_id, - device_cloudiot_id=self.device_cloudiot_id, - private_key_file=self.device_private_key, - ca_certs=self.gcp_root_ca, - remote_control_queue=self.remote_control_queue, - trace_context=self.get_device_metadata(), - ) - return self._mqtt_client - - @property - def telemetry_events(self): - if self.auth_token is None: - raise PluginSettingsRequired(f"auth_token is not set") - if self._telemetry_events is None: - loop = asyncio.get_event_loop() - self.telemetry_events = asyncio.run_coroutine_threadsafe( - self.rest_client.get_telemetry_events(), loop - ).result() - return self._telemetry_events - - def event_in_tracked_telemetry(self, event_type): - return event_type in self.telemetry_events - - -class WorkerManager(PluginSettingsMemoizeMixin): +class WorkerManager(MultiWorkerMixin): """ Manages PredictWorker, WebsocketWorker, RestWorker processes """ @@ -263,7 +79,7 @@ def __init__(self, plugin): self.remote_control_queue = self.manager.AioQueue() self._local_event_handlers = { - Events.PRINT_STARTED: self._handle_print_start, + Events.PRINT_STARTED: self.on_print_start, Events.PRINT_FAILED: self.stop_monitoring, Events.PRINT_DONE: self.stop_monitoring, Events.PRINT_CANCELLING: self.stop_monitoring, @@ -273,85 +89,12 @@ def __init__(self, plugin): Events.SHUTDOWN: self.shutdown, } - self._remote_control_event_handlers = { - MQTTClient.CONFIG_UPDATE_START: self.get_device_config - } self._monitoring_halt = None self.event_loop_thread = threading.Thread(target=self._event_loop_worker) self.event_loop_thread.daemon = True self.event_loop_thread.start() - @beeline.traced("WorkerManager.get_device_config") - async def get_device_config(self, topic, message): - device_config = print_nanny_client.ExperimentDeviceConfig(**message) - - labels = device_config.artifact.get("labels") - artifacts = device_config.artifact.get("artifacts") - version = device_config.artifact.get("version") - metadata = device_config.artifact.get("metadata") - - async def _download(session, url, filename): - async with session.get(url) as res: - async with aiofiles.open(filename, "w+") as f: - content = await res.text() - return await f.write(content) - - async def _data_file(content, filename): - async with aiofiles.open(filename, "w+") as f: - return await f.write(content) - - async with aiohttp.ClientSession() as session: - await _download( - session, - labels, - os.path.join(self.plugin.get_plugin_data_folder(), "labels.txt"), - ) - await _download( - session, - artifacts, - os.path.join(self.plugin.get_plugin_data_folder(), "model.tflite"), - ) - await _data_file( - version, - os.path.join(self.plugin.get_plugin_data_folder(), "version.txt"), - ) - await _data_file( - metadata, - os.path.join(self.plugin.get_plugin_data_folder(), "metadata.json"), - ) - - @beeline.traced("WorkerManager.init_monitoring_threads") - def init_monitoring_threads(self): - self._monitoring_halt = threading.Event() - - self.predict_worker = PredictWorker( - self.snapshot_url, - self.calibration, - self.octo_ws_queue, - self.pn_ws_queue, - self.telemetry_queue, - self.monitoring_frames_per_minute, - self._monitoring_halt, - self.plugin._event_bus, - trace_context=self.get_device_metadata(), - ) - - self.predict_worker_thread = threading.Thread(target=self.predict_worker.run) - self.predict_worker_thread.daemon = True - - self.websocket_worker = WebSocketWorker( - self.ws_url, - self.auth_token, - self.pn_ws_queue, - self.shared.print_job_id, - self.device_id, - self._monitoring_halt, - trace_context=self.get_device_metadata(), - ) - self.pn_ws_thread = threading.Thread(target=self.websocket_worker.run) - self.pn_ws_thread.daemon = True - def _event_loop_worker(self): loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) @@ -545,173 +288,6 @@ async def _publish_octoprint_event_telemetry(self, event): event.update(self.get_print_job_metadata()) self.mqtt_client.publish_octoprint_event(event) - async def _remote_control_receive_loop_forever(self): - logger.info("Started _remote_control_receive_loop_forever") - while not self._thread_halt.is_set(): - try: - await self._remote_control_receive_loop() - except PluginSettingsRequired: - pass - logger.info("Exiting soon _remote_control_receive_loop_forever") - - @beeline.traced("WorkerManager._handle_remote_control_command") - async def _handle_remote_control_command(self, topic, message): - event_type = message.get("octoprint_event_type") - - if event_type is None: - logger.warning("Ignoring received message where octoprint_event_type=None") - return - - command_id = message.get("remote_control_command_id") - - await self._remote_control_snapshot(command_id) - - metadata = self.get_device_metadata() - await self.rest_client.update_remote_control_command( - command_id, received=True, metadata=metadata - ) - - handler_fn = self._remote_control_event_handlers.get(event_type) - - logger.info( - f"Got handler_fn={handler_fn} from WorkerManager._remote_control_event_handlers for octoprint_event_type={event_type}" - ) - if handler_fn: - try: - if inspect.isawaitable(handler_fn): - await handler_fn(event=message, event_type=event_type) - else: - handler_fn(event=message, event_type=event_type) - - metadata = self.get_device_metadata() - # set success state - await self.rest_client.update_remote_control_command( - command_id, - success=True, - metadata=metadata, - ) - except Exception as e: - logger.error(f"Error calling handler_fn {handler_fn} \n {e}") - metadata = self.get_device_metadata() - await self.rest_client.update_remote_control_command( - command_id, - success=False, - metadata=metadata, - ) - - @beeline.traced("WorkerManager._remote_control_receive_loop") - async def _remote_control_receive_loop(self): - - trace = self._honeycomb_tracer.start_trace() - span = self._honeycomb_tracer.start_span( - {"name": "WorkerManager.remote_control_queue.coro_get"} - ) - payload = await self.remote_control_queue.coro_get() - self._honeycomb_tracer.add_context(dict(event=payload)) - self._honeycomb_tracer.finish_span(span) - - topic = payload.get("topic") - - if topic is None: - logger.warning("Ignoring received message where topic=None") - - elif topic == self.mqtt_client.remote_control_command_topic: - await self._handle_remote_control_command(**payload) - - elif topic == self.mqtt_client.mqtt_config_topic: - await self.get_device_config(**payload) - - else: - logging.error( - f"No handler for topic={topic} in _remote_control_receive_loop" - ) - - self._honeycomb_tracer.finish_trace(trace) - - @beeline.traced("WorkerManager._remote_control_snapshot") - async def _remote_control_snapshot(self, command_id): - async with aiohttp.ClientSession() as session: - res = await session.get(self.snapshot_url) - snapshot_io = io.BytesIO(await res.read()) - - return await self.rest_client.create_snapshot( - image=snapshot_io, command=command_id - ) - - @beeline.traced("WorkerManager._telemetry_queue_send_loop") - async def _telemetry_queue_send_loop(self): - try: - span = self._honeycomb_tracer.start_span( - {"name": "WorkerManager.telemetry_queue.coro_get"} - ) - - event = await self.telemetry_queue.coro_get() - - self._honeycomb_tracer.add_context(dict(event=event)) - self._honeycomb_tracer.finish_span(span) - - event_type = event.get("event_type") - if event_type is None: - logger.warning( - "Ignoring enqueued msg without type declared {event}".format( - event=event - ) - ) - return - - if event_type == BOUNDING_BOX_PREDICT_EVENT: - await self._publish_bounding_box_telemetry(event) - return - - if self.event_in_tracked_telemetry(event_type): - await self._publish_octoprint_event_telemetry(event) - else: - if event_type not in self.MUTED_EVENTS: - logger.warning(f"Discarding {event_type} with payload {event}") - return - - # run local handler fn - handler_fn = self._local_event_handlers.get(event_type) - if handler_fn: - - if inspect.isawaitable(handler_fn): - await handler_fn(**event) - else: - handler_fn(**event) - except API_CLIENT_EXCEPTIONS as e: - logger.error(f"REST client raised exception {e}", exc_info=True) - - async def _telemetry_queue_send_loop_forever(self): - """ - Publishes telemetry events via HTTP - """ - logger.info("Started _telemetry_queue_send_loop_forever") - while not self._thread_halt.is_set(): - try: - await self._telemetry_queue_send_loop() - except PluginSettingsRequired as e: - pass - logging.info("Exiting soon _telemetry_queue_send_loop_forever") - - @beeline.traced("WorkerManager.stop_monitoring") - def stop_monitoring(self, event_type=None, **kwargs): - """ - joins and terminates dedicated prediction and pn websocket processes - """ - logging.info( - f"WorkerManager.stop_monitoring called by event_type={event_type} event={kwargs}" - ) - self.monitoring_active = False - - asyncio.run_coroutine_threadsafe( - self.rest_client.update_octoprint_device( - self.device_id, monitoring_active=False - ), - self.loop, - ).result() - - self.stop_monitoring_threads() - @beeline.traced("WorkerManager.shutdown") def shutdown(self): self.stop_monitoring() @@ -726,31 +302,9 @@ def shutdown(self): self.stop_worker_threads() self._honeycomb_tracer.on_shutdown() - @beeline.traced("WorkerManager.start_monitoring") - def start_monitoring(self, event_type=None, **kwargs): - """ - starts prediction and pn websocket processes - """ - logging.info( - f"WorkerManager.start_monitoring called by event_type={event_type} event={kwargs}" - ) - self.monitoring_active = True - - asyncio.run_coroutine_threadsafe( - self.rest_client.update_octoprint_device( - self.device_id, monitoring_active=True - ), - self.loop, - ).result() - - self.init_monitoring_threads() - self.start_monitoring_threads() - - @beeline.traced("WorkerManager._handle_print_start") - async def _handle_print_start(self, event_type, event_data, **kwargs): - logger.info( - f"_handle_print_start called for {event_type} with data {event_data}" - ) + @beeline.traced("WorkerManager.on_print_start") + async def on_print_start(self, event_type, event_data, **kwargs): + logger.info(f"on_print_start called for {event_type} with data {event_data}") try: current_profile = ( self.plugin._printer_profile_manager.get_current_or_default() @@ -777,7 +331,7 @@ async def _handle_print_start(self, event_type, event_data, **kwargs): self.shared.print_job_id = print_job.id except API_CLIENT_EXCEPTIONS as e: - logger.error(f"_handle_print_start API called failed {e}", exc_info=True) + logger.error(f"on_print_start API called failed {e}", exc_info=True) return if self.plugin.get_setting("auto_start"): diff --git a/octoprint_nanny/settings.py b/octoprint_nanny/settings.py new file mode 100644 index 00000000..180ca6a5 --- /dev/null +++ b/octoprint_nanny/settings.py @@ -0,0 +1,191 @@ +import asyncio +from datetime import datetime +import logging +import pytz + +from octoprint_nanny.predictor import ( + PredictWorker, +) + +import beeline +from octoprint_nanny.clients.mqtt import MQTTClient +from octoprint_nanny.clients.rest import RestAPIClient, API_CLIENT_EXCEPTIONS +from octoprint_nanny.clients.websocket import WebSocketWorker +from octoprint_nanny.exceptions import PluginSettingsRequired + +logger = logging.getLogger("octoprint.plugins.octoprint_nanny.manager") + + +class PluginSettingsMemoizeMixin: + """ + Convenience methods/properties for accessing OctoPrint plugin settings and computed metadata + """ + + def __init__(self, plugin): + self.plugin = plugin + + # stateful clients and computed settings that require re-initialization when settings change + self._calibration = None + self._mqtt_client = None + self._telemetry_events = None + self._device_info = None + self._rest_client = None + + self.environment = {} + + @beeline.traced("PluginSettingsMemoize.reset_monitoring_settings") + def reset_monitoring_settings(self): + self._calibration = None + self._monitoring_frames_per_minute = None + + @beeline.traced("PluginSettingsMemoize.reset_device_settings_state") + @beeline.traced_thread + def reset_device_settings_state(self): + self._mqtt_client = None + self._device_info = None + + @beeline.traced("PluginSettingsMemoize.reset_rest_client_state") + @beeline.traced_thread + def reset_rest_client_state(self): + self._rest_client = None + + @beeline.traced(name="PluginSettingsMemoize.get_device_metadata") + @beeline.traced_thread + def get_device_metadata(self): + metadata = dict( + created_dt=datetime.now(pytz.timezone("UTC")), + environment=self.environment, + ) + metadata.update(self.device_info) + return metadata + + @beeline.traced(name="PluginSettingsMemoize.get_print_job_metadata") + @beeline.traced_thread + def get_print_job_metadata(self): + return dict( + printer_data=self.plugin._printer.get_current_data(), + printer_profile_data=self.plugin._printer_profile_manager.get_current_or_default(), + temperatures=self.plugin._printer.get_current_temperatures(), + printer_profile_id=self.shared.printer_profile_id, + print_job_id=self.shared.print_job_id, + ) + + @beeline.traced(name="PluginSettingsMemoize.on_environment_detected") + @beeline.traced_thread + def on_environment_detected(self, environment): + self.environment = environment + + @property + def device_cloudiot_name(self): + return self.plugin.get_setting("device_cloudiot_name") + + @property + def device_id(self): + return self.plugin.get_setting("device_id") + + @property + def device_info(self): + if self._device_info is None: + self._device_info = self.plugin.get_device_info() + return self._device_info + + @property + def device_serial(self): + return self.plugin.get_setting("device_serial") + + @property + def device_cloudiot_id(self): + return self.plugin.get_setting("device_cloudiot_id") + + @property + def device_private_key(self): + return self.plugin.get_setting("device_private_key") + + @property + def device_public_key(self): + return self.plugin.get_setting("device_public_key") + + @property + def gcp_root_ca(self): + return self.plugin.get_setting("gcp_root_ca") + + @property + def api_url(self): + return self.plugin.get_setting("api_url") + + @property + def auth_token(self): + return self.plugin.get_setting("auth_token") + + @property + def ws_url(self): + return self.plugin.get_setting("ws_url") + + @property + def snapshot_url(self): + return self.plugin.get_setting("snapshot_url") + + @property + def user_id(self): + return self.plugin.get_setting("user_id") + + @property + def calibration(self): + if self._calibration is None: + self._calibration = PredictWorker.calc_calibration( + self.plugin.get_setting("calibrate_x0"), + self.plugin.get_setting("calibrate_y0"), + self.plugin.get_setting("calibrate_x1"), + self.plugin.get_setting("calibrate_y1"), + ) + return self._calibration + + @property + def monitoring_frames_per_minute(self): + return self.plugin.get_setting("monitoring_frames_per_minute") + + @property + def rest_client(self): + if self.auth_token is None: + raise PluginSettingsRequired(f"auth_token is not set") + if self._rest_client is None: + self._rest_client = RestAPIClient( + auth_token=self.auth_token, api_url=self.api_url + ) + logger.info(f"RestAPIClient initialized with api_url={self.api_url}") + return self._rest_client + + def test_mqtt_settings(self): + if self.device_cloudiot_id is None or self.device_private_key is None: + raise PluginSettingsRequired( + f"Received None for device_cloudiot_id={self.device_cloudiot_id} or private_key_file={self.device_private_key}" + ) + return True + + @property + def mqtt_client(self): + self.test_mqtt_settings() + if self._mqtt_client is None: + self._mqtt_client = MQTTClient( + device_id=self.device_id, + device_cloudiot_id=self.device_cloudiot_id, + private_key_file=self.device_private_key, + ca_certs=self.gcp_root_ca, + remote_control_queue=self.remote_control_queue, + trace_context=self.get_device_metadata(), + ) + return self._mqtt_client + + @property + def telemetry_events(self): + if self.auth_token is None: + raise PluginSettingsRequired(f"auth_token is not set") + if self._telemetry_events is None: + loop = asyncio.get_event_loop() + self.telemetry_events = asyncio.run_coroutine_threadsafe( + self.rest_client.get_telemetry_events(), loop + ).result() + return self._telemetry_events + + def event_in_tracked_telemetry(self, event_type): + return event_type in self.telemetry_events diff --git a/octoprint_nanny/workers.py b/octoprint_nanny/workers.py new file mode 100644 index 00000000..8e8ca13c --- /dev/null +++ b/octoprint_nanny/workers.py @@ -0,0 +1,312 @@ +import aiofiles +import aiohttp +import io +import inspect +import logging +import os +import threading + + +import beeline + +import print_nanny_client + +from octoprint_nanny.exceptions import PluginSettingsRequired +from octoprint_nanny.clients.honeycomb import HoneycombTracer +from octoprint_nanny.clients.rest import RestAPIClient, API_CLIENT_EXCEPTIONS +from octoprint_nanny.predictor import ( + PredictWorker, + BOUNDING_BOX_PREDICT_EVENT, + ANNOTATED_IMAGE_EVENT, +) +from octoprint_nanny.settings import PluginSettingsMemoizeMixin + +logger = logging.getLogger("octoprint.plugins.octoprint_nanny.") + + +class ReceiveWorkerMixin(PluginSettingsMemoizeMixin): + async def _remote_control_receive_loop_forever(self): + logger.info("Started _remote_control_receive_loop_forever") + while not self._thread_halt.is_set(): + try: + await self._remote_control_receive_loop() + except PluginSettingsRequired: + pass + logger.info("Exiting soon _remote_control_receive_loop_forever") + + @beeline.traced("WorkerManager._remote_control_snapshot") + async def _remote_control_snapshot(self, command_id): + async with aiohttp.ClientSession() as session: + res = await session.get(self.snapshot_url) + snapshot_io = io.BytesIO(await res.read()) + + return await self.rest_client.create_snapshot( + image=snapshot_io, command=command_id + ) + + @beeline.traced("WorkerManager._handle_remote_control_command") + async def _handle_remote_control_command(self, topic, message): + event_type = message.get("octoprint_event_type") + + if event_type is None: + logger.warning("Ignoring received message where octoprint_event_type=None") + return + + command_id = message.get("remote_control_command_id") + + await self._remote_control_snapshot(command_id) + + metadata = self.get_device_metadata() + await self.rest_client.update_remote_control_command( + command_id, received=True, metadata=metadata + ) + + handler_fn = self._remote_control_event_handlers.get(event_type) + + logger.info( + f"Got handler_fn={handler_fn} from WorkerManager._remote_control_event_handlers for octoprint_event_type={event_type}" + ) + if handler_fn: + try: + if inspect.isawaitable(handler_fn): + await handler_fn(event=message, event_type=event_type) + else: + handler_fn(event=message, event_type=event_type) + + metadata = self.get_device_metadata() + # set success state + await self.rest_client.update_remote_control_command( + command_id, + success=True, + metadata=metadata, + ) + except Exception as e: + logger.error(f"Error calling handler_fn {handler_fn} \n {e}") + metadata = self.get_device_metadata() + await self.rest_client.update_remote_control_command( + command_id, + success=False, + metadata=metadata, + ) + + @beeline.traced("WorkerManager._remote_control_receive_loop") + async def _remote_control_receive_loop(self): + + trace = self._honeycomb_tracer.start_trace() + span = self._honeycomb_tracer.start_span( + {"name": "WorkerManager.remote_control_queue.coro_get"} + ) + payload = await self.remote_control_queue.coro_get() + self._honeycomb_tracer.add_context(dict(event=payload)) + self._honeycomb_tracer.finish_span(span) + + topic = payload.get("topic") + + if topic is None: + logger.warning("Ignoring received message where topic=None") + + elif topic == self.mqtt_client.remote_control_command_topic: + await self._handle_remote_control_command(**payload) + + elif topic == self.mqtt_client.mqtt_config_topic: + await self.get_device_config(**payload) + + else: + logging.error( + f"No handler for topic={topic} in _remote_control_receive_loop" + ) + + self._honeycomb_tracer.finish_trace(trace) + + @beeline.traced("WorkerManager.get_device_config") + async def get_device_config(self, topic, message): + device_config = print_nanny_client.ExperimentDeviceConfig(**message) + + labels = device_config.artifact.get("labels") + artifacts = device_config.artifact.get("artifacts") + version = device_config.artifact.get("version") + metadata = device_config.artifact.get("metadata") + + async def _download(session, url, filename): + async with session.get(url) as res: + async with aiofiles.open(filename, "w+") as f: + content = await res.text() + return await f.write(content) + + async def _data_file(content, filename): + async with aiofiles.open(filename, "w+") as f: + return await f.write(content) + + async with aiohttp.ClientSession() as session: + await _download( + session, + labels, + os.path.join(self.plugin.get_plugin_data_folder(), "labels.txt"), + ) + await _download( + session, + artifacts, + os.path.join(self.plugin.get_plugin_data_folder(), "model.tflite"), + ) + await _data_file( + version, + os.path.join(self.plugin.get_plugin_data_folder(), "version.txt"), + ) + await _data_file( + metadata, + os.path.join(self.plugin.get_plugin_data_folder(), "metadata.json"), + ) + + +class TelemetryWorkerMixin: + @beeline.traced("WorkerManager._publish_octoprint_event_telemetry") + async def _publish_octoprint_event_telemetry(self, event): + event_type = event.get("event_type") + logger.info(f"_publish_octoprint_event_telemetry {event}") + event.update( + dict( + user_id=self.user_id, + device_id=self.device_id, + device_cloudiot_name=self.device_cloudiot_name, + ) + ) + event.update(self.get_device_metadata()) + + if event_type in self.PRINT_JOB_EVENTS: + event.update(self.get_print_job_metadata()) + self.mqtt_client.publish_octoprint_event(event) + + @beeline.traced("WorkerManager._telemetry_queue_send_loop") + async def _telemetry_queue_send_loop(self): + try: + span = self._honeycomb_tracer.start_span( + {"name": "WorkerManager.telemetry_queue.coro_get"} + ) + + event = await self.telemetry_queue.coro_get() + + self._honeycomb_tracer.add_context(dict(event=event)) + self._honeycomb_tracer.finish_span(span) + + event_type = event.get("event_type") + if event_type is None: + logger.warning( + "Ignoring enqueued msg without type declared {event}".format( + event=event + ) + ) + return + + if event_type == BOUNDING_BOX_PREDICT_EVENT: + await self._publish_bounding_box_telemetry(event) + return + + if self.event_in_tracked_telemetry(event_type): + await self._publish_octoprint_event_telemetry(event) + else: + if event_type not in self.MUTED_EVENTS: + logger.warning(f"Discarding {event_type} with payload {event}") + return + + # run local handler fn + handler_fn = self._local_event_handlers.get(event_type) + if handler_fn: + + if inspect.isawaitable(handler_fn): + await handler_fn(**event) + else: + handler_fn(**event) + except API_CLIENT_EXCEPTIONS as e: + logger.error(f"REST client raised exception {e}", exc_info=True) + + async def _telemetry_queue_send_loop_forever(self): + """ + Publishes telemetry events via HTTP + """ + logger.info("Started _telemetry_queue_send_loop_forever") + while not self._thread_halt.is_set(): + try: + await self._telemetry_queue_send_loop() + except PluginSettingsRequired as e: + pass + logging.info("Exiting soon _telemetry_queue_send_loop_forever") + + +class MonitoringWorkerMixin: + @beeline.traced("WorkerManager.init_monitoring_threads") + def init_monitoring_threads(self): + self._monitoring_halt = threading.Event() + + self.predict_worker = PredictWorker( + self.snapshot_url, + self.calibration, + self.octo_ws_queue, + self.pn_ws_queue, + self.telemetry_queue, + self.monitoring_frames_per_minute, + self._monitoring_halt, + self.plugin._event_bus, + trace_context=self.get_device_metadata(), + ) + + self.predict_worker_thread = threading.Thread(target=self.predict_worker.run) + self.predict_worker_thread.daemon = True + + self.websocket_worker = WebSocketWorker( + self.ws_url, + self.auth_token, + self.pn_ws_queue, + self.shared.print_job_id, + self.device_id, + self._monitoring_halt, + trace_context=self.get_device_metadata(), + ) + self.pn_ws_thread = threading.Thread(target=self.websocket_worker.run) + self.pn_ws_thread.daemon = True + + @beeline.traced("WorkerManager.stop_monitoring") + def stop_monitoring(self, event_type=None, **kwargs): + """ + joins and terminates dedicated prediction and pn websocket processes + """ + logging.info( + f"WorkerManager.stop_monitoring called by event_type={event_type} event={kwargs}" + ) + self.monitoring_active = False + + asyncio.run_coroutine_threadsafe( + self.rest_client.update_octoprint_device( + self.device_id, monitoring_active=False + ), + self.loop, + ).result() + + self.stop_monitoring_threads() + + @beeline.traced("WorkerManager.start_monitoring") + def start_monitoring(self, event_type=None, **kwargs): + """ + starts prediction and pn websocket processes + """ + logging.info( + f"WorkerManager.start_monitoring called by event_type={event_type} event={kwargs}" + ) + self.monitoring_active = True + + asyncio.run_coroutine_threadsafe( + self.rest_client.update_octoprint_device( + self.device_id, monitoring_active=True + ), + self.loop, + ).result() + + self.init_monitoring_threads() + self.start_monitoring_threads() + + +class MultiWorkerMixin( + TelemetryWorkerMixin, + ReceiveWorkerMixin, + MonitoringWorkerMixin, +): + pass diff --git a/tests/test_manager.py b/tests/test_manager.py index b5382494..ef5f1835 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -1,6 +1,8 @@ import asyncio import pytest +from unittest.mock import PropertyMock + import octoprint_nanny.plugins # import DEFAULT_SETTINGS, OctoPrintNannyPlugin from octoprint_nanny.manager import WorkerManager from octoprint_nanny.exceptions import PluginSettingsRequired @@ -47,7 +49,7 @@ async def test_telemetry_queue_send_loop_valid_octoprint_event(mocker): mocker.patch.object(WorkerManager, "telemetry_events") mocker.patch.object(WorkerManager, "event_in_tracked_telemetry", return_value=True) - mock_handle_print_start = mocker.patch.object(WorkerManager, "_handle_print_start") + mock_on_print_start = mocker.patch.object(WorkerManager, "on_print_start") manager = WorkerManager(plugin) @@ -62,7 +64,7 @@ async def test_telemetry_queue_send_loop_valid_octoprint_event(mocker): await manager._telemetry_queue_send_loop() mock_publish_octoprint_event_telemetry.assert_called_once_with(event) - mock_handle_print_start.assert_called_once_with( + mock_on_print_start.assert_called_once_with( event_data=event["event_data"], event_type=event["event_type"] ) @@ -108,6 +110,13 @@ async def test_remote_control_receive_loop_valid_event(mocker): mock_rest_client.update_remote_control_command.return_value = asyncio.Future() mock_rest_client.update_remote_control_command.return_value.set_result("foo") + mock_mqtt_client = mocker.patch.object(WorkerManager, "mqtt_client") + + topic = "remote-control-topic" + type(mock_mqtt_client).remote_control_command_topic = PropertyMock( + return_value=topic + ) + mocker.patch.object(WorkerManager, "get_device_metadata", return_value={}) mock_start_monitoring = mocker.patch.object(WorkerManager, "start_monitoring") @@ -118,28 +127,35 @@ async def test_remote_control_receive_loop_valid_event(mocker): } command = { - "octoprint_event_type": "octoprint_nanny_plugin_monitoring_start", - "command": "MonitoringStart", - "remote_control_command_id": 1, + "message": { + "octoprint_event_type": "octoprint_nanny_plugin_monitoring_start", + "command": "MonitoringStart", + "remote_control_command_id": 1, + }, + "topic": topic, } manager.remote_control_queue.put_nowait(command) await manager._remote_control_receive_loop() mock_remote_control_snapshot.assert_called_once_with( - command["remote_control_command_id"] + command["message"]["remote_control_command_id"] ) mock_rest_client.update_remote_control_command.assert_has_calls( [ mocker.call( - command["remote_control_command_id"], received=True, metadata={} + command["message"]["remote_control_command_id"], + received=True, + metadata={}, ), mocker.call( - command["remote_control_command_id"], success=True, metadata={} + command["message"]["remote_control_command_id"], + success=True, + metadata={}, ), ] ) mock_start_monitoring.assert_called_once_with( - event=command, event_type=command["octoprint_event_type"] + event=command["message"], event_type=command["message"]["octoprint_event_type"] ) From 6c55efbbc3c15f9f75f579c56fbb68482440582a Mon Sep 17 00:00:00 2001 From: Leigh Johnson Date: Sun, 7 Feb 2021 20:37:19 +0000 Subject: [PATCH 02/16] add octoprint_nanny.workers module --- octoprint_nanny/workers/__init__.py | 0 octoprint_nanny/workers/monitoring.py | 89 +++++++ octoprint_nanny/workers/mqtt.py | 325 ++++++++++++++++++++++++++ octoprint_nanny/workers/websocket.py | 117 ++++++++++ 4 files changed, 531 insertions(+) create mode 100644 octoprint_nanny/workers/__init__.py create mode 100644 octoprint_nanny/workers/monitoring.py create mode 100644 octoprint_nanny/workers/mqtt.py create mode 100644 octoprint_nanny/workers/websocket.py diff --git a/octoprint_nanny/workers/__init__.py b/octoprint_nanny/workers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/octoprint_nanny/workers/monitoring.py b/octoprint_nanny/workers/monitoring.py new file mode 100644 index 00000000..d6424655 --- /dev/null +++ b/octoprint_nanny/workers/monitoring.py @@ -0,0 +1,89 @@ +import asyncio +import logging +import threading + +import beeline +from octoprint_nanny.workers.websocket import WebSocketWorker +from octoprint_nanny.predictor import ( + PredictWorker, + BOUNDING_BOX_PREDICT_EVENT, + ANNOTATED_IMAGE_EVENT, +) +from octoprint_nanny.settings import PluginSettingsMemoizeMixin + +logger = logging.getLogger("octoprint.plugins.octoprint_nanny.workers.monitoring") + + +class MonitoringWorker(PluginSettingsMemoizeMixin): + """ + Wrapper for PredictWorker and WebsocketWorker + """ + + def __init__(self, plugin, octo_ws_queue, pn_ws_queue, mqtt_send_queue): + + super().__init__(plugin) + self.halt = threading.Event() + self.octo_ws_queue = octo_ws_queue + self.pn_ws_queue = pn_ws_queue + self.mqtt_send_queue = mqtt_send_queue + + @beeline.traced() + def init(self): + + self.predict_worker = PredictWorker( + self.snapshot_url, + self.calibration, + self.octo_ws_queue, + self.pn_ws_queue, + self.mqtt_send_queue, + self.monitoring_frames_per_minute, + self.halt, + self.plugin._event_bus, + trace_context=self.get_device_metadata(), + ) + + self.predict_worker_thread = threading.Thread(target=self.predict_worker.run) + self.predict_worker_thread.daemon = True + + self.websocket_worker = WebSocketWorker( + self.ws_url, + self.auth_token, + self.pn_ws_queue, + self.device_id, + self.halt, + trace_context=self.get_device_metadata(), + ) + self.websocket_worker_thread = threading.Thread( + target=self.websocket_worker.run + ) + self.websocket_worker_thread.daemon = True + + @beeline.traced() + async def stop(self, event_type=None, **kwargs): + """ + joins and terminates dedicated prediction and pn websocket processes + """ + logging.info( + f"WorkerManager.stop_monitoring called by event_type={event_type} event={kwargs}" + ) + self.active = False + + await self.rest_client.update_octoprint_device(self.device_id, active=False) + self.halt.set() + self.predict_worker_thread.join() + self.websocket_worker_thrad.join() + + @beeline.traced("WorkerManager.start_monitoring") + async def start(self, event_type=None, **kwargs): + """ + starts prediction and pn websocket processes + """ + logging.info( + f"WorkerManager.start_monitoring called by event_type={event_type} event={kwargs}" + ) + self.active = True + + await self.rest_client.update_octoprint_device(self.device_id, active=True) + self.init() + self.predict_worker_thread.start() + self.websocket_worker_thread.start() diff --git a/octoprint_nanny/workers/mqtt.py b/octoprint_nanny/workers/mqtt.py new file mode 100644 index 00000000..c4f22bdd --- /dev/null +++ b/octoprint_nanny/workers/mqtt.py @@ -0,0 +1,325 @@ +import aiofiles +import aiohttp +import asyncio +import concurrent +import io +import inspect +import logging +import os +import threading + +import logging + +import beeline + +import print_nanny_client + +from octoprint.events import Events + +from octoprint_nanny.clients.rest import API_CLIENT_EXCEPTIONS +from octoprint_nanny.exceptions import PluginSettingsRequired +from octoprint_nanny.settings import PluginSettingsMemoizeMixin +from octoprint_nanny.clients.honeycomb import HoneycombTracer +from octoprint_nanny.predictor import BOUNDING_BOX_PREDICT_EVENT + +logger = logging.getLogger("octoprint.plugins.octoprint_nanny.workers.mqtt") + + +class MQTTClientWorker(PluginSettingsMemoizeMixin): + def __init__(self, plugin, halt): + + super().__init__(plugin) + self.halt = halt + + @beeline.traced() + def run(self): + try: + return self.mqtt_client.run(self.halt) + except PluginSettingsRequired: + pass + logger.warning("MQTTClientWorker will exit soon") + + +class MQTTPublishWorker(PluginSettingsMemoizeMixin): + """ + Run a worker thread to handle publishing MQTT messages + """ + + PRINT_JOB_EVENTS = [ + Events.ERROR, + Events.PRINT_CANCELLING, + Events.PRINT_CANCELLED, + Events.PRINT_DONE, + Events.PRINT_FAILED, + Events.PRINT_PAUSED, + Events.PRINT_RESUMED, + Events.PRINT_STARTED, + ] + # do not warn when the following events are skipped on telemetry update + MUTED_EVENTS = [Events.Z_CHANGE, "plugin_octoprint_nanny_predict_done"] + + def __init__(self, plugin, halt, queue): + + super().__init__(plugin) + self.halt = halt + self.queue = queue + self._callbacks = {} + self._honeycomb_tracer = HoneycombTracer(service_name="octoprint_plugin") + + @beeline.traced() + def run(self): + """ + Telemetry worker's event loop is exposed as WorkerManager.loop + this permits other threads to schedule work in this event loop with asyncio.run_coroutine_threadsafe() + """ + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + loop.set_default_executor(concurrent.futures.ThreadPoolExecutor(max_workers=4)) + + return loop.run_until_complete(asyncio.ensure_future(self.loop_forever())) + + @beeline.traced() + async def _publish_octoprint_event_telemetry(self, event): + event_type = event.get("event_type") + logger.info(f"_publish_octoprint_event_telemetry {event}") + event.update( + dict( + user_id=self.user_id, + device_id=self.device_id, + device_cloudiot_name=self.device_cloudiot_name, + ) + ) + event.update(self.get_device_metadata()) + + if event_type in self.PRINT_JOB_EVENTS: + event.update(self.get_print_job_metadata()) + self.mqtt_client.publish_octoprint_event(event) + + @beeline.traced() + async def _publish_bounding_box_telemetry(self, event): + event.update( + dict( + user_id=self.user_id, + device_id=self.device_id, + device_cloudiot_name=self.device_cloudiot_name, + ) + ) + self.mqtt_client.publish_bounding_boxes(event) + + @beeline.traced() + async def _loop(self): + try: + span = self._honeycomb_tracer.start_span( + {"name": "WorkerManager.queue.coro_get"} + ) + + try: + event = self.queue.get_nowait() + except queue.Empty: + return + + self._honeycomb_tracer.add_context(dict(event=event)) + self._honeycomb_tracer.finish_span(span) + + event_type = event.get("event_type") + if event_type is None: + logger.warning( + "Ignoring enqueued msg without type declared {event}".format( + event=event + ) + ) + return + + if event_type == BOUNDING_BOX_PREDICT_EVENT: + await self._publish_bounding_box_telemetry(event) + return + + if self.event_in_tracked_telemetry(event_type): + await self._publish_octoprint_event_telemetry(event) + else: + if event_type not in self.MUTED_EVENTS: + logger.warning(f"Discarding {event_type} with payload {event}") + return + + # run local handler fn + handler_fn = self._local_event_handlers.get(event_type) + if handler_fn: + + if inspect.isawaitable(handler_fn): + await handler_fn(**event) + else: + handler_fn(**event) + except API_CLIENT_EXCEPTIONS as e: + logger.error(f"REST client raised exception {e}", exc_info=True) + + async def loop_forever(self): + """ + Publishes telemetry events via HTTP + """ + logger.info("Started loop_forever") + while not self.halt.is_set(): + try: + await self._loop() + except PluginSettingsRequired as e: + pass + logging.info("Exiting soon loop_forever") + + +class MQTTReceiveWorker(PluginSettingsMemoizeMixin): + def __init__(self, plugin, halt, queue): + + super().__init__(plugin) + self.halt = halt + self.queue = queue + self._callbacks = {} + self._honeycomb_tracer = HoneycombTracer(service_name="octoprint_plugin") + + @beeline.traced() + def run(self): + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) + self.loop.set_default_executor( + concurrent.futures.ThreadPoolExecutor(max_workers=4) + ) + + return self.loop.run_until_complete(asyncio.ensure_future(self.loop_forever())) + + def register_callbacks(self, callbacks): + for k, v in callbacks.items(): + if self._callbacks.get(k) is None: + self._callbacks[k] = [v] + else: + self._callbacks[k].append(v) + logging.info(f"Registered MQTTReceiveWorker._callbacks {self._callbacks}") + return self._callbacks + + async def loop_forever(self): + logger.info("Started loop_forever") + while not self.halt.is_set(): + try: + await self._loop() + except PluginSettingsRequired: + pass + logger.info("Exiting soon MQTTReceiveWorker.loop_forever") + + @beeline.traced() + async def _remote_control_snapshot(self, command_id): + async with aiohttp.ClientSession() as session: + res = await session.get(self.snapshot_url) + snapshot_io = io.BytesIO(await res.read()) + + return await self.rest_client.create_snapshot( + image=snapshot_io, command=command_id + ) + + @beeline.traced() + async def _handle_remote_control_command(self, topic, message): + event_type = message.get("octoprint_event_type") + + if event_type is None: + logger.warning("Ignoring received message where octoprint_event_type=None") + return + + command_id = message.get("remote_control_command_id") + + await self._remote_control_snapshot(command_id) + + metadata = self.get_device_metadata() + await self.rest_client.update_remote_control_command( + command_id, received=True, metadata=metadata + ) + + handler_fns = self._callbacks.get(event_type) + + logger.info( + f"Got handler_fn={handler_fns} from WorkerManager._callbacks for octoprint_event_type={event_type}" + ) + if handler_fns is not None: + for handler_fn in handler_fns: + try: + if inspect.isawaitable(handler_fn): + await handler_fn(event=message, event_type=event_type) + else: + handler_fn(event=message, event_type=event_type) + + metadata = self.get_device_metadata() + # set success state + await self.rest_client.update_remote_control_command( + command_id, + success=True, + metadata=metadata, + ) + except Exception as e: + logger.error(f"Error calling handler_fn {handler_fn} \n {e}") + metadata = self.get_device_metadata() + await self.rest_client.update_remote_control_command( + command_id, + success=False, + metadata=metadata, + ) + + @beeline.traced() + async def _loop(self): + + trace = self._honeycomb_tracer.start_trace() + span = self._honeycomb_tracer.start_span( + {"name": "WorkerManager.queue.coro_get"} + ) + payload = await self.queue.coro_get() + self._honeycomb_tracer.add_context(dict(event=payload)) + self._honeycomb_tracer.finish_span(span) + + topic = payload.get("topic") + + if topic is None: + logger.warning("Ignoring received message where topic=None") + + elif topic == self.mqtt_client.remote_control_command_topic: + await self._handle_remote_control_command(**payload) + + elif topic == self.mqtt_client.config_topic: + await self._handle_config_update(**payload) + + else: + logging.error(f"No handler for topic={topic} in _loop") + + self._honeycomb_tracer.finish_trace(trace) + + @beeline.traced() + async def _handle_config_update(self, topic, message): + device_config = print_nanny_client.ExperimentDeviceConfig(**message) + + labels = device_config.artifact.get("labels") + artifacts = device_config.artifact.get("artifacts") + version = device_config.artifact.get("version") + metadata = device_config.artifact.get("metadata") + + async def _download(session, url, filename): + async with session.get(url) as res: + async with aiofiles.open(filename, "w+") as f: + content = await res.text() + return await f.write(content) + + async def _data_file(content, filename): + async with aiofiles.open(filename, "w+") as f: + return await f.write(content) + + async with aiohttp.ClientSession() as session: + await _download( + session, + labels, + os.path.join(self.plugin.get_plugin_data_folder(), "labels.txt"), + ) + await _download( + session, + artifacts, + os.path.join(self.plugin.get_plugin_data_folder(), "model.tflite"), + ) + await _data_file( + version, + os.path.join(self.plugin.get_plugin_data_folder(), "version.txt"), + ) + await _data_file( + metadata, + os.path.join(self.plugin.get_plugin_data_folder(), "metadata.json"), + ) diff --git a/octoprint_nanny/workers/websocket.py b/octoprint_nanny/workers/websocket.py new file mode 100644 index 00000000..434c6219 --- /dev/null +++ b/octoprint_nanny/workers/websocket.py @@ -0,0 +1,117 @@ +import aiohttp +import asyncio +import hashlib +import json +import logging +import queue +import websockets +import urllib +import asyncio +import os +import threading +import aioprocessing +import multiprocessing +import signal +import sys +import os + +import beeline + +from octoprint_nanny.utils.encoder import NumpyEncoder +from octoprint_nanny.clients.honeycomb import HoneycombTracer + +# @ todo configure logger from ~/.octoprint/logging.yaml +logger = logging.getLogger("octoprint.plugins.octoprint_nanny.clients.websocket") + + +class WebSocketWorker: + """ + Relays prediction and image buffers from PredictWorker + to websocket connection + + Restart proc on api_url and auth_token settings change + """ + + def __init__( + self, + base_url, + auth_token, + producer, + device_id, + halt, + trace_context={}, + ): + + if not isinstance(producer, multiprocessing.managers.BaseProxy): + raise ValueError( + "producer should be an instance of aioprocessing.managers.AioQueueProxy" + ) + + self._device_id = device_id + self._base_url = base_url + self._url = f"{base_url}{device_id}/video/upload/" + self._auth_token = auth_token + self._producer = producer + + self._extra_headers = (("Authorization", f"Bearer {self._auth_token}"),) + self._halt = halt + self._honeycomb_tracer = HoneycombTracer(service_name="octoprint_plugin") + self._honeycomb_tracer.add_global_context(trace_context) + + def _signal_handler(self, received_signal, _): + logger.warning(f"Received signal {received_signal}") + self._halt.set() + sys.exit(0) + + def encode(self, msg): + return json.dumps(msg, cls=NumpyEncoder) + + @beeline.traced("WebSocketWorker.ping") + async def ping(self, msg=None): + async with websockets.connect( + self._url, extra_headers=self._extra_headers + ) as websocket: + if msg is None: + msg = {"event_type": "ping"} + msg = self.encode(msg) + await websocket.send(msg) + return await websocket.recv() + + @beeline.traced("WebSocketWorker.send") + async def send(self, msg=None): + async with websockets.connect( + self._url, extra_headers=self._extra_headers + ) as websocket: + if msg is None: + msg = {"event_type": "ping"} + msg = self.encode(msg) + await websocket.send(msg) + + def run(self): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + loop.run_until_complete(self.relay_loop()) + + async def relay_loop(self): + logging.info(f"Initializing websocket {self._url}") + async with websockets.connect( + self._url, extra_headers=self._extra_headers + ) as websocket: + logger.info(f"Websocket connected {websocket}") + while not self._halt.is_set(): + trace = self._honeycomb_tracer.start_trace() + span = self._honeycomb_tracer.start_span( + context={"name": "WebSocketWorker._producer.coro_get"} + ) + msg = await self._producer.coro_get() + self._honeycomb_tracer.finish_span(span) + + event_type = msg.get("event_type") + if event_type == "annotated_image": + encoded_msg = self.encode(msg=msg) + await websocket.send(encoded_msg) + else: + logger.warning(f"Invalid event_type {event_type}, msg ignored") + + self._honeycomb_tracer.finish_trace(trace) + logger.warning("Halt event set, worker will exit soon") From 7c72037281d9c2c82e34b7ea2954558561084f47 Mon Sep 17 00:00:00 2001 From: Leigh Johnson Date: Sun, 7 Feb 2021 20:38:27 +0000 Subject: [PATCH 03/16] rm first pass on octoprint_nanny/workers.py (moved to octoprint_nanny/workers/*.py) --- octoprint_nanny/workers.py | 312 ------------------------------------- 1 file changed, 312 deletions(-) delete mode 100644 octoprint_nanny/workers.py diff --git a/octoprint_nanny/workers.py b/octoprint_nanny/workers.py deleted file mode 100644 index 8e8ca13c..00000000 --- a/octoprint_nanny/workers.py +++ /dev/null @@ -1,312 +0,0 @@ -import aiofiles -import aiohttp -import io -import inspect -import logging -import os -import threading - - -import beeline - -import print_nanny_client - -from octoprint_nanny.exceptions import PluginSettingsRequired -from octoprint_nanny.clients.honeycomb import HoneycombTracer -from octoprint_nanny.clients.rest import RestAPIClient, API_CLIENT_EXCEPTIONS -from octoprint_nanny.predictor import ( - PredictWorker, - BOUNDING_BOX_PREDICT_EVENT, - ANNOTATED_IMAGE_EVENT, -) -from octoprint_nanny.settings import PluginSettingsMemoizeMixin - -logger = logging.getLogger("octoprint.plugins.octoprint_nanny.") - - -class ReceiveWorkerMixin(PluginSettingsMemoizeMixin): - async def _remote_control_receive_loop_forever(self): - logger.info("Started _remote_control_receive_loop_forever") - while not self._thread_halt.is_set(): - try: - await self._remote_control_receive_loop() - except PluginSettingsRequired: - pass - logger.info("Exiting soon _remote_control_receive_loop_forever") - - @beeline.traced("WorkerManager._remote_control_snapshot") - async def _remote_control_snapshot(self, command_id): - async with aiohttp.ClientSession() as session: - res = await session.get(self.snapshot_url) - snapshot_io = io.BytesIO(await res.read()) - - return await self.rest_client.create_snapshot( - image=snapshot_io, command=command_id - ) - - @beeline.traced("WorkerManager._handle_remote_control_command") - async def _handle_remote_control_command(self, topic, message): - event_type = message.get("octoprint_event_type") - - if event_type is None: - logger.warning("Ignoring received message where octoprint_event_type=None") - return - - command_id = message.get("remote_control_command_id") - - await self._remote_control_snapshot(command_id) - - metadata = self.get_device_metadata() - await self.rest_client.update_remote_control_command( - command_id, received=True, metadata=metadata - ) - - handler_fn = self._remote_control_event_handlers.get(event_type) - - logger.info( - f"Got handler_fn={handler_fn} from WorkerManager._remote_control_event_handlers for octoprint_event_type={event_type}" - ) - if handler_fn: - try: - if inspect.isawaitable(handler_fn): - await handler_fn(event=message, event_type=event_type) - else: - handler_fn(event=message, event_type=event_type) - - metadata = self.get_device_metadata() - # set success state - await self.rest_client.update_remote_control_command( - command_id, - success=True, - metadata=metadata, - ) - except Exception as e: - logger.error(f"Error calling handler_fn {handler_fn} \n {e}") - metadata = self.get_device_metadata() - await self.rest_client.update_remote_control_command( - command_id, - success=False, - metadata=metadata, - ) - - @beeline.traced("WorkerManager._remote_control_receive_loop") - async def _remote_control_receive_loop(self): - - trace = self._honeycomb_tracer.start_trace() - span = self._honeycomb_tracer.start_span( - {"name": "WorkerManager.remote_control_queue.coro_get"} - ) - payload = await self.remote_control_queue.coro_get() - self._honeycomb_tracer.add_context(dict(event=payload)) - self._honeycomb_tracer.finish_span(span) - - topic = payload.get("topic") - - if topic is None: - logger.warning("Ignoring received message where topic=None") - - elif topic == self.mqtt_client.remote_control_command_topic: - await self._handle_remote_control_command(**payload) - - elif topic == self.mqtt_client.mqtt_config_topic: - await self.get_device_config(**payload) - - else: - logging.error( - f"No handler for topic={topic} in _remote_control_receive_loop" - ) - - self._honeycomb_tracer.finish_trace(trace) - - @beeline.traced("WorkerManager.get_device_config") - async def get_device_config(self, topic, message): - device_config = print_nanny_client.ExperimentDeviceConfig(**message) - - labels = device_config.artifact.get("labels") - artifacts = device_config.artifact.get("artifacts") - version = device_config.artifact.get("version") - metadata = device_config.artifact.get("metadata") - - async def _download(session, url, filename): - async with session.get(url) as res: - async with aiofiles.open(filename, "w+") as f: - content = await res.text() - return await f.write(content) - - async def _data_file(content, filename): - async with aiofiles.open(filename, "w+") as f: - return await f.write(content) - - async with aiohttp.ClientSession() as session: - await _download( - session, - labels, - os.path.join(self.plugin.get_plugin_data_folder(), "labels.txt"), - ) - await _download( - session, - artifacts, - os.path.join(self.plugin.get_plugin_data_folder(), "model.tflite"), - ) - await _data_file( - version, - os.path.join(self.plugin.get_plugin_data_folder(), "version.txt"), - ) - await _data_file( - metadata, - os.path.join(self.plugin.get_plugin_data_folder(), "metadata.json"), - ) - - -class TelemetryWorkerMixin: - @beeline.traced("WorkerManager._publish_octoprint_event_telemetry") - async def _publish_octoprint_event_telemetry(self, event): - event_type = event.get("event_type") - logger.info(f"_publish_octoprint_event_telemetry {event}") - event.update( - dict( - user_id=self.user_id, - device_id=self.device_id, - device_cloudiot_name=self.device_cloudiot_name, - ) - ) - event.update(self.get_device_metadata()) - - if event_type in self.PRINT_JOB_EVENTS: - event.update(self.get_print_job_metadata()) - self.mqtt_client.publish_octoprint_event(event) - - @beeline.traced("WorkerManager._telemetry_queue_send_loop") - async def _telemetry_queue_send_loop(self): - try: - span = self._honeycomb_tracer.start_span( - {"name": "WorkerManager.telemetry_queue.coro_get"} - ) - - event = await self.telemetry_queue.coro_get() - - self._honeycomb_tracer.add_context(dict(event=event)) - self._honeycomb_tracer.finish_span(span) - - event_type = event.get("event_type") - if event_type is None: - logger.warning( - "Ignoring enqueued msg without type declared {event}".format( - event=event - ) - ) - return - - if event_type == BOUNDING_BOX_PREDICT_EVENT: - await self._publish_bounding_box_telemetry(event) - return - - if self.event_in_tracked_telemetry(event_type): - await self._publish_octoprint_event_telemetry(event) - else: - if event_type not in self.MUTED_EVENTS: - logger.warning(f"Discarding {event_type} with payload {event}") - return - - # run local handler fn - handler_fn = self._local_event_handlers.get(event_type) - if handler_fn: - - if inspect.isawaitable(handler_fn): - await handler_fn(**event) - else: - handler_fn(**event) - except API_CLIENT_EXCEPTIONS as e: - logger.error(f"REST client raised exception {e}", exc_info=True) - - async def _telemetry_queue_send_loop_forever(self): - """ - Publishes telemetry events via HTTP - """ - logger.info("Started _telemetry_queue_send_loop_forever") - while not self._thread_halt.is_set(): - try: - await self._telemetry_queue_send_loop() - except PluginSettingsRequired as e: - pass - logging.info("Exiting soon _telemetry_queue_send_loop_forever") - - -class MonitoringWorkerMixin: - @beeline.traced("WorkerManager.init_monitoring_threads") - def init_monitoring_threads(self): - self._monitoring_halt = threading.Event() - - self.predict_worker = PredictWorker( - self.snapshot_url, - self.calibration, - self.octo_ws_queue, - self.pn_ws_queue, - self.telemetry_queue, - self.monitoring_frames_per_minute, - self._monitoring_halt, - self.plugin._event_bus, - trace_context=self.get_device_metadata(), - ) - - self.predict_worker_thread = threading.Thread(target=self.predict_worker.run) - self.predict_worker_thread.daemon = True - - self.websocket_worker = WebSocketWorker( - self.ws_url, - self.auth_token, - self.pn_ws_queue, - self.shared.print_job_id, - self.device_id, - self._monitoring_halt, - trace_context=self.get_device_metadata(), - ) - self.pn_ws_thread = threading.Thread(target=self.websocket_worker.run) - self.pn_ws_thread.daemon = True - - @beeline.traced("WorkerManager.stop_monitoring") - def stop_monitoring(self, event_type=None, **kwargs): - """ - joins and terminates dedicated prediction and pn websocket processes - """ - logging.info( - f"WorkerManager.stop_monitoring called by event_type={event_type} event={kwargs}" - ) - self.monitoring_active = False - - asyncio.run_coroutine_threadsafe( - self.rest_client.update_octoprint_device( - self.device_id, monitoring_active=False - ), - self.loop, - ).result() - - self.stop_monitoring_threads() - - @beeline.traced("WorkerManager.start_monitoring") - def start_monitoring(self, event_type=None, **kwargs): - """ - starts prediction and pn websocket processes - """ - logging.info( - f"WorkerManager.start_monitoring called by event_type={event_type} event={kwargs}" - ) - self.monitoring_active = True - - asyncio.run_coroutine_threadsafe( - self.rest_client.update_octoprint_device( - self.device_id, monitoring_active=True - ), - self.loop, - ).result() - - self.init_monitoring_threads() - self.start_monitoring_threads() - - -class MultiWorkerMixin( - TelemetryWorkerMixin, - ReceiveWorkerMixin, - MonitoringWorkerMixin, -): - pass From 7220b55aa795c61f65da5bee91d2d4fcdef4e2eb Mon Sep 17 00:00:00 2001 From: Leigh Johnson Date: Sun, 7 Feb 2021 20:38:49 +0000 Subject: [PATCH 04/16] fix settings module loggert --- octoprint_nanny/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octoprint_nanny/settings.py b/octoprint_nanny/settings.py index 180ca6a5..e28c1f67 100644 --- a/octoprint_nanny/settings.py +++ b/octoprint_nanny/settings.py @@ -13,7 +13,7 @@ from octoprint_nanny.clients.websocket import WebSocketWorker from octoprint_nanny.exceptions import PluginSettingsRequired -logger = logging.getLogger("octoprint.plugins.octoprint_nanny.manager") +logger = logging.getLogger("octoprint.plugins.octoprint_nanny.settings") class PluginSettingsMemoizeMixin: From 7d6fe54b5c849234fe071d8cf4ad85dbb12eb56a Mon Sep 17 00:00:00 2001 From: Leigh Johnson Date: Mon, 8 Feb 2021 01:28:31 +0000 Subject: [PATCH 05/16] adjust config topic property in MQTTClient --- octoprint_nanny/clients/mqtt.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/octoprint_nanny/clients/mqtt.py b/octoprint_nanny/clients/mqtt.py index 39fc0d9f..a5c4ae88 100644 --- a/octoprint_nanny/clients/mqtt.py +++ b/octoprint_nanny/clients/mqtt.py @@ -106,12 +106,12 @@ def __init__( self.client.on_unsubscribe = self._on_unsubscribe # device receives configuration updates on this topic - self.mqtt_config_topic = f"/devices/{self.device_cloudiot_id}/config" + self.config_topic = f"/devices/{self.device_cloudiot_id}/config" # device receives commands on this topic - self.mqtt_command_topic = f"/devices/{self.device_cloudiot_id}/commands/#" + self.commands_topic = f"/devices/{self.device_cloudiot_id}/commands/#" # remote_control app commmands are routed to this subfolder - self.remote_control_command_topic = ( + self.remote_control_commands_topic = ( f"/devices/{self.device_cloudiot_id}/commands/remote_control" ) # this permits routing on a per-app basis, e.g. @@ -137,22 +137,22 @@ def __init__( ## @beeline.traced("MQTTClient._on_message") def _on_message(self, client, userdata, message): - if message.topic == self.remote_control_command_topic: + if message.topic == self.remote_control_commands_topic: parsed_message = json.loads(message.payload.decode("utf-8")) logger.info( f"Received remote control command on topic={message.topic} payload={parsed_message}" ) self.remote_control_queue.put_nowait( - {"topic": self.remote_control_command_topic, "message": parsed_message} + {"topic": self.remote_control_commands_topic, "message": parsed_message} ) # callback to api to indicate command was received - elif message.topic == self.mqtt_config_topic: + elif message.topic == self.config_topic: parsed_message = json.loads(message.payload.decode("utf-8")) logger.info( f"Received config update on topic={message.topic} payload={parsed_message}" ) self.remote_control_queue.put_nowait( - {"topic": self.mqtt_config_topic, "message": parsed_message} + {"topic": self.config_topic, "message": parsed_message} ) else: logger.info( @@ -200,13 +200,13 @@ def _on_connect(self, client, userdata, flags, rc): should_backoff = False minimum_backoff_time = 1 logger.info("Device successfully connected to MQTT broker") - self.client.subscribe(self.mqtt_config_topic, qos=1) + self.client.subscribe(self.config_topic, qos=1) logger.info( - f"Subscribing to config updates device_cloudiot_id={self.device_cloudiot_id} to topic {self.mqtt_config_topic}" + f"Subscribing to config updates device_cloudiot_id={self.device_cloudiot_id} to topic {self.config_topic}" ) - self.client.subscribe(self.mqtt_command_topic, qos=1) + self.client.subscribe(self.commands_topic, qos=1) logger.info( - f"Subscribing to remote commands device_cloudiot_id={self.device_cloudiot_id} to topic {self.mqtt_command_topic}" + f"Subscribing to remote commands device_cloudiot_id={self.device_cloudiot_id} to topic {self.commands_topic}" ) else: logger.error(f"Connection refused by MQTT broker with reason code rc={rc}") From 2c1e965b7aa083d9e3be9e0f24af06f64d10a6b0 Mon Sep 17 00:00:00 2001 From: Leigh Johnson Date: Mon, 8 Feb 2021 01:29:06 +0000 Subject: [PATCH 06/16] finish first pass on refactoring workers into octoprint_nanny/workers/* modules --- octoprint_nanny/clients/websocket.py | 119 ---------- octoprint_nanny/manager.py | 298 +++++++------------------- octoprint_nanny/predictor.py | 1 - octoprint_nanny/workers/monitoring.py | 100 +++++---- octoprint_nanny/workers/mqtt.py | 140 +++++++++--- tests/test_manager.py | 2 +- 6 files changed, 243 insertions(+), 417 deletions(-) delete mode 100644 octoprint_nanny/clients/websocket.py diff --git a/octoprint_nanny/clients/websocket.py b/octoprint_nanny/clients/websocket.py deleted file mode 100644 index 66baaadf..00000000 --- a/octoprint_nanny/clients/websocket.py +++ /dev/null @@ -1,119 +0,0 @@ -import aiohttp -import asyncio -import hashlib -import json -import logging -import queue -import websockets -import urllib -import asyncio -import os -import threading -import aioprocessing -import multiprocessing -import signal -import sys -import os - -import beeline - -from octoprint_nanny.utils.encoder import NumpyEncoder -from octoprint_nanny.clients.honeycomb import HoneycombTracer - -# @ todo configure logger from ~/.octoprint/logging.yaml -logger = logging.getLogger("octoprint.plugins.octoprint_nanny.clients.websocket") - - -class WebSocketWorker: - """ - Relays prediction and image buffers from PredictWorker - to websocket connection - - Restart proc on api_url and auth_token settings change - """ - - def __init__( - self, - base_url, - auth_token, - producer, - print_job_id, - device_id, - halt, - trace_context={}, - ): - - if not isinstance(producer, multiprocessing.managers.BaseProxy): - raise ValueError( - "producer should be an instance of aioprocessing.managers.AioQueueProxy" - ) - - self._print_job_id = print_job_id - self._device_id = device_id - self._base_url = base_url - self._url = f"{base_url}{device_id}/video/upload/" - self._auth_token = auth_token - self._producer = producer - - self._extra_headers = (("Authorization", f"Bearer {self._auth_token}"),) - self._halt = halt - self._honeycomb_tracer = HoneycombTracer(service_name="octoprint_plugin") - self._honeycomb_tracer.add_global_context(trace_context) - - def _signal_handler(self, received_signal, _): - logger.warning(f"Received signal {received_signal}") - self._halt.set() - sys.exit(0) - - def encode(self, msg): - return json.dumps(msg, cls=NumpyEncoder) - - @beeline.traced("WebSocketWorker.ping") - async def ping(self, msg=None): - async with websockets.connect( - self._url, extra_headers=self._extra_headers - ) as websocket: - if msg is None: - msg = {"event_type": "ping"} - msg = self.encode(msg) - await websocket.send(msg) - return await websocket.recv() - - @beeline.traced("WebSocketWorker.send") - async def send(self, msg=None): - async with websockets.connect( - self._url, extra_headers=self._extra_headers - ) as websocket: - if msg is None: - msg = {"event_type": "ping"} - msg = self.encode(msg) - await websocket.send(msg) - - def run(self): - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - loop.run_until_complete(self.relay_loop()) - - async def relay_loop(self): - logging.info(f"Initializing websocket {self._url}") - async with websockets.connect( - self._url, extra_headers=self._extra_headers - ) as websocket: - logger.info(f"Websocket connected {websocket}") - while not self._halt.is_set(): - trace = self._honeycomb_tracer.start_trace() - span = self._honeycomb_tracer.start_span( - context={"name": "WebSocketWorker._producer.coro_get"} - ) - msg = await self._producer.coro_get() - self._honeycomb_tracer.finish_span(span) - - event_type = msg.get("event_type") - if event_type == "annotated_image": - encoded_msg = self.encode(msg=msg) - await websocket.send(encoded_msg) - else: - logger.warning(f"Invalid event_type {event_type}, msg ignored") - - self._honeycomb_tracer.finish_trace(trace) - logger.warning("Halt event set, worker will exit soon") diff --git a/octoprint_nanny/manager.py b/octoprint_nanny/manager.py index bfb2f5f7..2a947957 100644 --- a/octoprint_nanny/manager.py +++ b/octoprint_nanny/manager.py @@ -16,85 +16,82 @@ import threading import uuid -import octoprint.filemanager from octoprint.events import Events +import octoprint.filemanager + from octoprint_nanny.clients.rest import API_CLIENT_EXCEPTIONS -from octoprint_nanny.workers import MultiWorkerMixin -from octoprint_nanny.exceptions import PluginSettingsRequired + +from octoprint_nanny.exceptions import PluginSettingsRequired +from octoprint_nanny.settings import PluginSettingsMemoize from octoprint_nanny.clients.honeycomb import HoneycombTracer +from octoprint_nanny.workers.mqtt import MQTTManager +from octoprint_nanny.workers.monitoring import MonitoringManager import beeline Events.PRINT_PROGRESS = "PrintProgress" +logger = logging.getLogger("octoprint.plugins.octoprint_nanny.manager") -class WorkerManager(MultiWorkerMixin): +class WorkerManager: """ - Manages PredictWorker, WebsocketWorker, RestWorker processes + Coordinates MQTTManager and MonitoringManager classes """ - MAX_BACKOFF = 256 - BACKOFF = 2 - - PRINT_JOB_EVENTS = [ - Events.ERROR, - Events.PRINT_CANCELLING, - Events.PRINT_CANCELLED, - Events.PRINT_DONE, - Events.PRINT_FAILED, - Events.PRINT_PAUSED, - Events.PRINT_RESUMED, - Events.PRINT_STARTED, - ] - - # do not warn when the following events are skipped on telemetry update - MUTED_EVENTS = [Events.Z_CHANGE, "plugin_octoprint_nanny_predict_done"] - - EVENT_PREFIX = "plugin_octoprint_nanny_" - def __init__(self, plugin): - super().__init__(plugin) + self.event_loop_thread = threading.Thread(target=self._event_loop_worker) + self.event_loop_thread.daemon = True + self.event_loop_thread.start() + + plugin_settings = PluginSettingsMemoize(plugin) + self.plugin_settings = plugin_settings self._honeycomb_tracer = HoneycombTracer(service_name="octoprint_plugin") self.plugin = plugin self.manager = aioprocessing.AioManager() self.shared = self.manager.Namespace() - # proxy objects - self.shared.printer_profile_id = None - self.shared.print_job_id = None - self.shared.calibration = None - self.monitoring_active = False - # outbound telemetry to GCP MQTT bridge - self.telemetry_queue = self.manager.AioQueue() # images streamed to octoprint front-end over websocket - self.octo_ws_queue = self.manager.AioQueue() + octo_ws_queue = self.manager.AioQueue() + self.octo_ws_queue = octo_ws_queue # images streamed to webapp asgi over websocket - self.pn_ws_queue = self.manager.AioQueue() - # inbound commands from MQTT - self.remote_control_queue = self.manager.AioQueue() + pn_ws_queue = self.manager.AioQueue() + self.pn_ws_queue = pn_ws_queue + + # outbound telemetry messages to MQTT bridge + mqtt_send_queue = self.manager.AioQueue() + self.mqtt_send_queue = mqtt_send_queue + # inbound MQTT command and config messages from MQTT bridge + mqtt_receive_queue = self.manager.AioQueue() + self.mqtt_receive_queue = mqtt_receive_queue + + self.mqtt_manager = MQTTManager( + mqtt_send_queue, mqtt_receive_queue, plugin_settings, plugin + ) + self.monitoring_manager = MonitoringManager( + octo_ws_queue, + pn_ws_queue, + mqtt_send_queue, + plugin_settings, + self.plugin._event_bus, + ) - self._local_event_handlers = { + # local callback/handler functions for events published via telemetry queue + self._mqtt_send_queue_callbacks = { Events.PRINT_STARTED: self.on_print_start, - Events.PRINT_FAILED: self.stop_monitoring, - Events.PRINT_DONE: self.stop_monitoring, - Events.PRINT_CANCELLING: self.stop_monitoring, - Events.PRINT_CANCELLED: self.stop_monitoring, - Events.PRINT_PAUSED: self.stop_monitoring, - Events.PRINT_RESUMED: self.stop_monitoring, + Events.PRINT_FAILED: self.monitoring_manager.stop, + Events.PRINT_DONE: self.monitoring_manager.stop, + Events.PRINT_CANCELLING: self.monitoring_manager.stop, + Events.PRINT_CANCELLED: self.monitoring_manager.stop, + Events.PRINT_PAUSED: self.monitoring_manager.stop, + Events.PRINT_RESUMED: self.monitoring_manager.start, Events.SHUTDOWN: self.shutdown, } - self._monitoring_halt = None - - self.event_loop_thread = threading.Thread(target=self._event_loop_worker) - self.event_loop_thread.daemon = True - self.event_loop_thread.start() - def _event_loop_worker(self): loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) @@ -102,119 +99,48 @@ def _event_loop_worker(self): self.loop = loop return loop.run_forever() - @beeline.traced("WorkerManager.init_worker_threads") - def init_worker_threads(self): - self._thread_halt = threading.Event() - - self.telemetry_worker_thread = threading.Thread(target=self._telemetry_worker) - self.telemetry_worker_thread.daemon = True - - # daemonized thread for MQTT worker thread - self.mqtt_worker_thread = threading.Thread(target=self._mqtt_worker) - self.mqtt_worker_thread.daemon = True - - # daemonized thread for handling inbound commands received via MQTT - self.remote_control_worker_thread = threading.Thread( - target=self._remote_control_worker - ) - self.remote_control_worker_thread.daemon = True - - @beeline.traced("WorkerManager.start_monitoring_threads") - def start_monitoring_threads(self): - self.predict_worker_thread.start() - self.pn_ws_thread.start() - - @beeline.traced("WorkerManager.start_worker_threads") - def start_worker_threads(self): - self.mqtt_worker_thread.start() - self.telemetry_worker_thread.start() - self.remote_control_worker_thread.start() - - @beeline.traced("WorkerManager.stop_monitoring_threads") - def stop_monitoring_threads(self): - logger.warning("Setting halt signal for monitoring worker threads") - if self._monitoring_halt is not None: - self._monitoring_halt.set() - logger.info("Waiting for WorkerManager.predict_worker_thread to drain") - self.predict_worker_thread.join() - - logger.info("Waiting for WorkerManger.pn_ws_thread to drain") - self.pn_ws_thread.join() - - @beeline.traced("WorkerManager.stop_worker_threads") - def stop_worker_threads(self): - logger.warning("Setting halt signal for telemetry worker threads") - self._thread_halt.set() - self.plugin._event_bus.fire(Events.PLUGIN_OCTOPRINT_NANNY_WORKER_STOP) - logger.info( - "Waiting for WorkerMangager.mqtt_client network connection to close" - ) - - try: - while self.mqtt_client.client.is_connected(): - self.mqtt_client.client.disconnect() - logger.info("Waiting for WorkerManager.mqtt_worker_thread to drain") - self.mqtt_client.client.disconnect() - self.mqtt_client.client.loop_stop() - except PluginSettingsRequired: - pass - self.mqtt_worker_thread.join() - - logger.info("Waiting for WorkerManager.remote_control_worker_thread to drain") - self.remote_control_queue.put_nowait({"event_type": "halt"}) - self.remote_control_worker_thread.join(timeout=10) - - logger.info("Waiting for WorkerManager.telemetry_worker_thread to drain") - self.telemetry_worker_thread.join(timeout=10) - logger.info("Finished halting WorkerManager threads") - - @beeline.traced("WorkerManager._register_plugin_event_handlers") + @beeline.traced() def _register_plugin_event_handlers(self): """ Events.PLUGIN_OCTOPRINT_NANNY* events are not available on Events until plugin is fully initialized """ - self._local_event_handlers.update( - { - Events.PLUGIN_OCTOPRINT_NANNY_MONITORING_START: self.start_monitoring, - Events.PLUGIN_OCTOPRINT_NANNY_MONITORING_STOP: self.stop_monitoring, - } - ) - self._remote_control_event_handlers.update( - { - Events.PLUGIN_OCTOPRINT_NANNY_MONITORING_START: self.start_monitoring, - Events.PLUGIN_OCTOPRINT_NANNY_MONITORING_STOP: self.stop_monitoring, - Events.PLUGIN_OCTOPRINT_NANNY_SNAPSHOT: self.on_snapshot, - } - ) - - @beeline.traced("WorkerManager.on_settings_initialized") + pass + # self._local_event_handlers.update( + # { + # Events.PLUGIN_OCTOPRINT_NANNY_MONITORING_START: self.start_monitoring, + # Events.PLUGIN_OCTOPRINT_NANNY_MONITORING_STOP: self.stop_monitoring, + # } + # ) + # self._remote_control_event_handlers.update( + # { + # Events.PLUGIN_OCTOPRINT_NANNY_MONITORING_START: self.start_monitoring, + # Events.PLUGIN_OCTOPRINT_NANNY_MONITORING_STOP: self.stop_monitoring, + # } + # ) + + @beeline.traced() def on_settings_initialized(self): self._honeycomb_tracer.add_global_context(self.get_device_metadata()) - self.init_worker_threads() + # self._register_plugin_event_handlers() + self.mqtt_manager.start() - # register plugin event handlers - self._register_plugin_event_handlers() - self.start_worker_threads() - - @beeline.traced("WorkerManager.on_snapshot") - def on_snapshot(self, *args, **kwargs): - logger.info(f"WorkerManager.on_snapshot called with {args} {kwargs}") - - @beeline.traced("WorkerManager.apply_device_registration") + @beeline.traced() def apply_device_registration(self): + self.mqtt_manager.stop() logger.info("Resetting WorkerManager device registration state") - self.stop_worker_threads() - self.reset_device_settings_state() - self.init_worker_threads() - self.start_worker_threads() + self.plugin_settings.reset_device_settings_state() + self.mqtt_manager.start() - @beeline.traced("WorkerManager.apply_auth") + @beeline.traced() def apply_auth(self): logger.info("Resetting WorkerManager user auth state") - self.reset_rest_client_state() + self.mqtt_manager.stop() + self.plugin_settings.reset_rest_client_state() + self.mqtt_manager.start() - @beeline.traced("WorkerManager.apply_monitoring_settings") + @beeline.traced() def apply_monitoring_settings(self): + self.reset_monitoring_settings() logger.info( "Stopping any existing monitoring processes to apply new calibration" @@ -226,69 +152,7 @@ def apply_monitoring_settings(self): ) self.start_monitoring() - @beeline.traced("WorkerManager._mqtt_worker") - def _mqtt_worker(self): - try: - return self.mqtt_client.run(self._thread_halt) - except PluginSettingsRequired: - pass - logger.warning("WorkerManager._mqtt_worker exiting") - - @beeline.traced("WorkerManager._telemetry_worker") - def _telemetry_worker(self): - """ - Telemetry worker's event loop is exposed as WorkerManager.loop - this permits other threads to schedule work in this event loop with asyncio.run_coroutine_threadsafe() - """ - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - loop.set_default_executor(concurrent.futures.ThreadPoolExecutor(max_workers=4)) - - return loop.run_until_complete( - asyncio.ensure_future(self._telemetry_queue_send_loop_forever()) - ) - - @beeline.traced("WorkerManager._remote_control_worker") - def _remote_control_worker(self): - self.remote_control_loop = asyncio.new_event_loop() - asyncio.set_event_loop(self.remote_control_loop) - self.remote_control_loop.set_default_executor( - concurrent.futures.ThreadPoolExecutor(max_workers=4) - ) - - return self.remote_control_loop.run_until_complete( - asyncio.ensure_future(self._remote_control_receive_loop_forever()) - ) - - @beeline.traced("WorkerManager._publish_bounding_box_telemetry") - async def _publish_bounding_box_telemetry(self, event): - event.update( - dict( - user_id=self.user_id, - device_id=self.device_id, - device_cloudiot_name=self.device_cloudiot_name, - ) - ) - self.mqtt_client.publish_bounding_boxes(event) - - @beeline.traced("WorkerManager._publish_octoprint_event_telemetry") - async def _publish_octoprint_event_telemetry(self, event): - event_type = event.get("event_type") - logger.info(f"_publish_octoprint_event_telemetry {event}") - event.update( - dict( - user_id=self.user_id, - device_id=self.device_id, - device_cloudiot_name=self.device_cloudiot_name, - ) - ) - event.update(self.get_device_metadata()) - - if event_type in self.PRINT_JOB_EVENTS: - event.update(self.get_print_job_metadata()) - self.mqtt_client.publish_octoprint_event(event) - - @beeline.traced("WorkerManager.shutdown") + @beeline.traced() def shutdown(self): self.stop_monitoring() @@ -302,17 +166,15 @@ def shutdown(self): self.stop_worker_threads() self._honeycomb_tracer.on_shutdown() - @beeline.traced("WorkerManager.on_print_start") + @beeline.traced() async def on_print_start(self, event_type, event_data, **kwargs): logger.info(f"on_print_start called for {event_type} with data {event_data}") try: current_profile = ( self.plugin._printer_profile_manager.get_current_or_default() ) - printer_profile = ( - await self.plugin.rest_client.update_or_create_printer_profile( - current_profile, self.device_id - ) + printer_profile = await self.rest_client.update_or_create_printer_profile( + current_profile, self.device_id ) self.shared.printer_profile_id = printer_profile.id @@ -320,11 +182,11 @@ async def on_print_start(self, event_type, event_data, **kwargs): gcode_file_path = self.plugin._file_manager.path_on_disk( octoprint.filemanager.FileDestinations.LOCAL, event_data["path"] ) - gcode_file = await self.plugin.rest_client.update_or_create_gcode_file( + gcode_file = await self.rest_client.update_or_create_gcode_file( event_data, gcode_file_path, self.device_id ) - print_job = await self.plugin.rest_client.create_print_job( + print_job = await self.rest_client.create_print_job( event_data, gcode_file.id, printer_profile.id, self.device_id ) diff --git a/octoprint_nanny/predictor.py b/octoprint_nanny/predictor.py index 1545f6de..b637ba61 100644 --- a/octoprint_nanny/predictor.py +++ b/octoprint_nanny/predictor.py @@ -38,7 +38,6 @@ from typing_extensions import TypedDict from typing import Optional -# @ todo configure logger from ~/.octoprint/logging.yaml logger = logging.getLogger("octoprint.plugins.octoprint_nanny.predictor") BOUNDING_BOX_PREDICT_EVENT = "bounding_box_predict" diff --git a/octoprint_nanny/workers/monitoring.py b/octoprint_nanny/workers/monitoring.py index d6424655..49d1a863 100644 --- a/octoprint_nanny/workers/monitoring.py +++ b/octoprint_nanny/workers/monitoring.py @@ -14,76 +14,72 @@ logger = logging.getLogger("octoprint.plugins.octoprint_nanny.workers.monitoring") -class MonitoringWorker(PluginSettingsMemoizeMixin): - """ - Wrapper for PredictWorker and WebsocketWorker - """ +class MonitoringManager: + def __init__( + self, + octo_ws_queue, + pn_ws_queue, + mqtt_send_queue, + plugin_settings, + plugin_event_bus, + ): - def __init__(self, plugin, octo_ws_queue, pn_ws_queue, mqtt_send_queue): - - super().__init__(plugin) self.halt = threading.Event() self.octo_ws_queue = octo_ws_queue self.pn_ws_queue = pn_ws_queue self.mqtt_send_queue = mqtt_send_queue + self.plugin_settings = plugin_settings + self.plugin_event_bus = plugin_event_bus + self.rest_client = plugin_settings.rest_client @beeline.traced() - def init(self): + def _drain(self): + self.halt.set() + + for worker in self._worker_threads: + logger.info(f"Waiting for worker={worker} thread to drain") + worker.join() - self.predict_worker = PredictWorker( - self.snapshot_url, - self.calibration, + def _reset(self): + self.halt = threading.Event() + self._predict_worker = PredictWorker( + self.plugin_settings.snapshot_url, + self.plugin_settings.calibration, self.octo_ws_queue, self.pn_ws_queue, self.mqtt_send_queue, - self.monitoring_frames_per_minute, + self.plugin_settings.monitoring_frames_per_minute, self.halt, - self.plugin._event_bus, - trace_context=self.get_device_metadata(), + self.plugin_event_bus, + trace_context=self.plugin_settings.get_device_metadata(), ) - - self.predict_worker_thread = threading.Thread(target=self.predict_worker.run) - self.predict_worker_thread.daemon = True - - self.websocket_worker = WebSocketWorker( - self.ws_url, - self.auth_token, + self._websocket_worker = WebSocketWorker( + self.plugin_settings.ws_url, + self.plugin_settings.auth_token, self.pn_ws_queue, - self.device_id, + self.plugin_settings.device_id, self.halt, - trace_context=self.get_device_metadata(), + trace_context=self.plugin_settings.get_device_metadata(), ) - self.websocket_worker_thread = threading.Thread( - target=self.websocket_worker.run - ) - self.websocket_worker_thread.daemon = True + self._workers = [self._predict_worker, self._websocket_worker] + self._worker_threads = [] @beeline.traced() - async def stop(self, event_type=None, **kwargs): - """ - joins and terminates dedicated prediction and pn websocket processes - """ - logging.info( - f"WorkerManager.stop_monitoring called by event_type={event_type} event={kwargs}" - ) - self.active = False - - await self.rest_client.update_octoprint_device(self.device_id, active=False) - self.halt.set() - self.predict_worker_thread.join() - self.websocket_worker_thrad.join() + async def start(self): + self._reset() - @beeline.traced("WorkerManager.start_monitoring") - async def start(self, event_type=None, **kwargs): - """ - starts prediction and pn websocket processes - """ - logging.info( - f"WorkerManager.start_monitoring called by event_type={event_type} event={kwargs}" + for worker in self._workers: + thread = threading.Thread(target=worker.run) + thread.daemon = True + self._worker_threads.append(thread) + thread.start() + await self.rest_client.update_octoprint_device( + self.plugin_settings.device_id, active=True ) - self.active = True - await self.rest_client.update_octoprint_device(self.device_id, active=True) - self.init() - self.predict_worker_thread.start() - self.websocket_worker_thread.start() + @beeline.traced() + async def stop(self): + self._drain() + await self.rest_client.update_octoprint_device( + self.plugin_settings.device_id, active=False + ) diff --git a/octoprint_nanny/workers/mqtt.py b/octoprint_nanny/workers/mqtt.py index c4f22bdd..c5c3a1fd 100644 --- a/octoprint_nanny/workers/mqtt.py +++ b/octoprint_nanny/workers/mqtt.py @@ -1,5 +1,6 @@ import aiofiles import aiohttp +import aioprocessing import asyncio import concurrent import io @@ -7,6 +8,7 @@ import logging import os import threading +import queue import logging @@ -18,17 +20,99 @@ from octoprint_nanny.clients.rest import API_CLIENT_EXCEPTIONS from octoprint_nanny.exceptions import PluginSettingsRequired -from octoprint_nanny.settings import PluginSettingsMemoizeMixin +from octoprint_nanny.settings import PluginSettingsMemoize + from octoprint_nanny.clients.honeycomb import HoneycombTracer from octoprint_nanny.predictor import BOUNDING_BOX_PREDICT_EVENT logger = logging.getLogger("octoprint.plugins.octoprint_nanny.workers.mqtt") -class MQTTClientWorker(PluginSettingsMemoizeMixin): - def __init__(self, plugin, halt): +class MQTTManager: + def __init__( + self, + mqtt_send_queue: aioprocessing.Queue, + mqtt_receive_queue: aioprocessing.Queue, + plugin_settings: PluginSettingsMemoize, + plugin, + ): + + super().__init__(self) + self.plugin_settings = plugin_settings + self.mqtt_client = plugin_settings.mqtt_client + self.mqtt_send_queue = mqtt_send_queue + self.mqtt_receive_queue = mqtt_receive_queue + halt = threading.Event() + self.halt = halt + self.plugin = plugin + + self._workers = [] + self._worker_threads = [] + + def _drain(self): + """ + Halt running workers and wait pending work + """ + self.halt.set() + + try: + logger.info("Waiting for MQTTManager.mqtt_client network loop to finish") + while self._client_worker.mqtt_client.client.is_connected(): + self.mqtt_client.client.disconnect() + logger.info("Stopping MQTTManager.mqtt_client network loop") + self.mqtt_client.client.loop_stop() + except PluginSettingsRequired: + pass + + for worker in self._worker_threads: + logger.info(f"Waiting for worker={worker} thread to drain") + worker.join() + + def _reset(self): + self.halt = threading.Event() + self._publisher_worker = MQTTPublisherWorker( + self.halt, self.mqtt_send_queue, self.plugin_settings + ) + self._subscriber_worker = MQTTSubscriberWorker( + self.halt, self.mqtt_receive_queue, self.plugin_settings, self.plugin + ) + self._client_worker = MQTTClientWorker( + self.halt, self.plugin_settings.mqtt_client + ) + + self._workers = [ + self._client_worker, + self._publisher_worker, + self._subscriber_worker, + ] + self._worker_threads = [] + + def start(self): + """ + (re)initialize and start worker threads + """ + logger.info("MQTTManager.start was called") + self._reset() + for worker in self._workers: + thread = threading.Thread(target=worker.run) + thread.daemon = True + self._worker_threads.append(thread) + thread.start() + + def stop(self): + logger.info("MQTTManager.stop was called") + self._drain() + + +class MQTTClientWorker: + """ + Manages paho MQTT client's network loop on behalf of Publisher/Subscriber Workers + https://www.eclipse.org/paho/index.php?page=clients/python/docs/index.php#network-loop + """ + + def __init__(self, halt, mqtt_client): - super().__init__(plugin) + self.mqtt_client = mqtt_client self.halt = halt @beeline.traced() @@ -40,9 +124,9 @@ def run(self): logger.warning("MQTTClientWorker will exit soon") -class MQTTPublishWorker(PluginSettingsMemoizeMixin): +class MQTTPublisherWorker: """ - Run a worker thread to handle publishing MQTT messages + Run a worker thread dedicated to publishing OctoPrint events to MQTT bridge """ PRINT_JOB_EVENTS = [ @@ -58,11 +142,12 @@ class MQTTPublishWorker(PluginSettingsMemoizeMixin): # do not warn when the following events are skipped on telemetry update MUTED_EVENTS = [Events.Z_CHANGE, "plugin_octoprint_nanny_predict_done"] - def __init__(self, plugin, halt, queue): + def __init__(self, halt, queue, plugin_settings): - super().__init__(plugin) self.halt = halt self.queue = queue + self.plugin_settings = plugin_settings + self.mqtt_client = plugin_settings.mqtt_client self._callbacks = {} self._honeycomb_tracer = HoneycombTracer(service_name="octoprint_plugin") @@ -84,24 +169,24 @@ async def _publish_octoprint_event_telemetry(self, event): logger.info(f"_publish_octoprint_event_telemetry {event}") event.update( dict( - user_id=self.user_id, - device_id=self.device_id, - device_cloudiot_name=self.device_cloudiot_name, + user_id=self.plugin_settings.user_id, + device_id=self.plugin_settings.device_id, + device_cloudiot_name=self.plugin_settings.device_cloudiot_name, ) ) - event.update(self.get_device_metadata()) + event.update(self.plugin_settings.get_device_metadata()) if event_type in self.PRINT_JOB_EVENTS: - event.update(self.get_print_job_metadata()) + event.update(self.plugin_settings.get_print_job_metadata()) self.mqtt_client.publish_octoprint_event(event) @beeline.traced() async def _publish_bounding_box_telemetry(self, event): event.update( dict( - user_id=self.user_id, - device_id=self.device_id, - device_cloudiot_name=self.device_cloudiot_name, + user_id=self.plugin_settings.user_id, + device_id=self.plugin_settings.device_id, + device_cloudiot_name=self.plugin_settings.device_cloudiot_name, ) ) self.mqtt_client.publish_bounding_boxes(event) @@ -142,7 +227,7 @@ async def _loop(self): return # run local handler fn - handler_fn = self._local_event_handlers.get(event_type) + handler_fn = self._callbacks.get(event_type) if handler_fn: if inspect.isawaitable(handler_fn): @@ -165,12 +250,15 @@ async def loop_forever(self): logging.info("Exiting soon loop_forever") -class MQTTReceiveWorker(PluginSettingsMemoizeMixin): - def __init__(self, plugin, halt, queue): +class MQTTSubscriberWorker: + def __init__(self, plugin, halt, queue, plugin_settings, plugin): - super().__init__(plugin) self.halt = halt self.queue = queue + self.plugin_settings = plugin_settings + self.plugin = plugin + self.mqtt_client = plugin_settings.mqtt_client + self.rest_client = plugin_settings.rest_client self._callbacks = {} self._honeycomb_tracer = HoneycombTracer(service_name="octoprint_plugin") @@ -190,7 +278,7 @@ def register_callbacks(self, callbacks): self._callbacks[k] = [v] else: self._callbacks[k].append(v) - logging.info(f"Registered MQTTReceiveWorker._callbacks {self._callbacks}") + logging.info(f"Registered MQTTSubscribeWorker._callbacks {self._callbacks}") return self._callbacks async def loop_forever(self): @@ -200,12 +288,12 @@ async def loop_forever(self): await self._loop() except PluginSettingsRequired: pass - logger.info("Exiting soon MQTTReceiveWorker.loop_forever") + logger.info("Exiting soon MQTTSubscribeWorker.loop_forever") @beeline.traced() async def _remote_control_snapshot(self, command_id): async with aiohttp.ClientSession() as session: - res = await session.get(self.snapshot_url) + res = await session.get(self.plugin_settings.snapshot_url) snapshot_io = io.BytesIO(await res.read()) return await self.rest_client.create_snapshot( @@ -224,7 +312,7 @@ async def _handle_remote_control_command(self, topic, message): await self._remote_control_snapshot(command_id) - metadata = self.get_device_metadata() + metadata = self.plugin_settings.get_device_metadata() await self.rest_client.update_remote_control_command( command_id, received=True, metadata=metadata ) @@ -242,7 +330,7 @@ async def _handle_remote_control_command(self, topic, message): else: handler_fn(event=message, event_type=event_type) - metadata = self.get_device_metadata() + metadata = self.plugin_settings.get_device_metadata() # set success state await self.rest_client.update_remote_control_command( command_id, @@ -251,7 +339,7 @@ async def _handle_remote_control_command(self, topic, message): ) except Exception as e: logger.error(f"Error calling handler_fn {handler_fn} \n {e}") - metadata = self.get_device_metadata() + metadata = self.plugin_settings.get_device_metadata() await self.rest_client.update_remote_control_command( command_id, success=False, diff --git a/tests/test_manager.py b/tests/test_manager.py index ef5f1835..c7d65f73 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -95,7 +95,7 @@ async def test_telemetry_queue_send_loop_bounding_box_predict(mocker): @pytest.mark.asyncio -async def test_remote_control_receive_loop_valid_event(mocker): +async def test_remote_control_receive_loop_valid_octoprint_event(mocker): plugin = octoprint_nanny.plugins.OctoPrintNannyPlugin() plugin.get_setting = get_default_setting From 17f7a7f6dd6c470f42d60e88de0309050ad114b3 Mon Sep 17 00:00:00 2001 From: Leigh Johnson Date: Mon, 8 Feb 2021 01:35:38 +0000 Subject: [PATCH 07/16] test_websocket.py tests passing --- octoprint_nanny/manager.py | 14 +++++------ octoprint_nanny/settings.py | 3 +-- octoprint_nanny/workers/monitoring.py | 7 +++--- octoprint_nanny/workers/mqtt.py | 22 +++++++++--------- .../OctoPrintNannyPlugin | Bin 0 -> 6 bytes tests/test_websocket.py | 7 +++--- 6 files changed, 25 insertions(+), 28 deletions(-) create mode 100644 tests/mocks/test_remote_control_receive_loop_valid_octoprint_event/octoprint_nanny.plugins/OctoPrintNannyPlugin diff --git a/octoprint_nanny/manager.py b/octoprint_nanny/manager.py index 2a947957..036acdd1 100644 --- a/octoprint_nanny/manager.py +++ b/octoprint_nanny/manager.py @@ -99,7 +99,7 @@ def _event_loop_worker(self): self.loop = loop return loop.run_forever() - @beeline.traced() + @beeline.traced def _register_plugin_event_handlers(self): """ Events.PLUGIN_OCTOPRINT_NANNY* events are not available on Events until plugin is fully initialized @@ -118,27 +118,27 @@ def _register_plugin_event_handlers(self): # } # ) - @beeline.traced() + @beeline.traced def on_settings_initialized(self): self._honeycomb_tracer.add_global_context(self.get_device_metadata()) # self._register_plugin_event_handlers() self.mqtt_manager.start() - @beeline.traced() + @beeline.traced def apply_device_registration(self): self.mqtt_manager.stop() logger.info("Resetting WorkerManager device registration state") self.plugin_settings.reset_device_settings_state() self.mqtt_manager.start() - @beeline.traced() + @beeline.traced def apply_auth(self): logger.info("Resetting WorkerManager user auth state") self.mqtt_manager.stop() self.plugin_settings.reset_rest_client_state() self.mqtt_manager.start() - @beeline.traced() + @beeline.traced def apply_monitoring_settings(self): self.reset_monitoring_settings() @@ -152,7 +152,7 @@ def apply_monitoring_settings(self): ) self.start_monitoring() - @beeline.traced() + @beeline.traced def shutdown(self): self.stop_monitoring() @@ -166,7 +166,7 @@ def shutdown(self): self.stop_worker_threads() self._honeycomb_tracer.on_shutdown() - @beeline.traced() + @beeline.traced async def on_print_start(self, event_type, event_data, **kwargs): logger.info(f"on_print_start called for {event_type} with data {event_data}") try: diff --git a/octoprint_nanny/settings.py b/octoprint_nanny/settings.py index e28c1f67..94d7338b 100644 --- a/octoprint_nanny/settings.py +++ b/octoprint_nanny/settings.py @@ -10,13 +10,12 @@ import beeline from octoprint_nanny.clients.mqtt import MQTTClient from octoprint_nanny.clients.rest import RestAPIClient, API_CLIENT_EXCEPTIONS -from octoprint_nanny.clients.websocket import WebSocketWorker from octoprint_nanny.exceptions import PluginSettingsRequired logger = logging.getLogger("octoprint.plugins.octoprint_nanny.settings") -class PluginSettingsMemoizeMixin: +class PluginSettingsMemoize: """ Convenience methods/properties for accessing OctoPrint plugin settings and computed metadata """ diff --git a/octoprint_nanny/workers/monitoring.py b/octoprint_nanny/workers/monitoring.py index 49d1a863..03cfa379 100644 --- a/octoprint_nanny/workers/monitoring.py +++ b/octoprint_nanny/workers/monitoring.py @@ -9,7 +9,6 @@ BOUNDING_BOX_PREDICT_EVENT, ANNOTATED_IMAGE_EVENT, ) -from octoprint_nanny.settings import PluginSettingsMemoizeMixin logger = logging.getLogger("octoprint.plugins.octoprint_nanny.workers.monitoring") @@ -32,7 +31,7 @@ def __init__( self.plugin_event_bus = plugin_event_bus self.rest_client = plugin_settings.rest_client - @beeline.traced() + @beeline.traced def _drain(self): self.halt.set() @@ -64,7 +63,7 @@ def _reset(self): self._workers = [self._predict_worker, self._websocket_worker] self._worker_threads = [] - @beeline.traced() + @beeline.traced async def start(self): self._reset() @@ -77,7 +76,7 @@ async def start(self): self.plugin_settings.device_id, active=True ) - @beeline.traced() + @beeline.traced async def stop(self): self._drain() await self.rest_client.update_octoprint_device( diff --git a/octoprint_nanny/workers/mqtt.py b/octoprint_nanny/workers/mqtt.py index c5c3a1fd..b3c6e619 100644 --- a/octoprint_nanny/workers/mqtt.py +++ b/octoprint_nanny/workers/mqtt.py @@ -115,7 +115,7 @@ def __init__(self, halt, mqtt_client): self.mqtt_client = mqtt_client self.halt = halt - @beeline.traced() + @beeline.traced def run(self): try: return self.mqtt_client.run(self.halt) @@ -151,7 +151,7 @@ def __init__(self, halt, queue, plugin_settings): self._callbacks = {} self._honeycomb_tracer = HoneycombTracer(service_name="octoprint_plugin") - @beeline.traced() + @beeline.traced def run(self): """ Telemetry worker's event loop is exposed as WorkerManager.loop @@ -163,7 +163,7 @@ def run(self): return loop.run_until_complete(asyncio.ensure_future(self.loop_forever())) - @beeline.traced() + @beeline.traced async def _publish_octoprint_event_telemetry(self, event): event_type = event.get("event_type") logger.info(f"_publish_octoprint_event_telemetry {event}") @@ -180,7 +180,7 @@ async def _publish_octoprint_event_telemetry(self, event): event.update(self.plugin_settings.get_print_job_metadata()) self.mqtt_client.publish_octoprint_event(event) - @beeline.traced() + @beeline.traced async def _publish_bounding_box_telemetry(self, event): event.update( dict( @@ -191,7 +191,7 @@ async def _publish_bounding_box_telemetry(self, event): ) self.mqtt_client.publish_bounding_boxes(event) - @beeline.traced() + @beeline.traced async def _loop(self): try: span = self._honeycomb_tracer.start_span( @@ -251,7 +251,7 @@ async def loop_forever(self): class MQTTSubscriberWorker: - def __init__(self, plugin, halt, queue, plugin_settings, plugin): + def __init__(self, halt, queue, plugin_settings, plugin): self.halt = halt self.queue = queue @@ -262,7 +262,7 @@ def __init__(self, plugin, halt, queue, plugin_settings, plugin): self._callbacks = {} self._honeycomb_tracer = HoneycombTracer(service_name="octoprint_plugin") - @beeline.traced() + @beeline.traced def run(self): self.loop = asyncio.new_event_loop() asyncio.set_event_loop(self.loop) @@ -290,7 +290,7 @@ async def loop_forever(self): pass logger.info("Exiting soon MQTTSubscribeWorker.loop_forever") - @beeline.traced() + @beeline.traced async def _remote_control_snapshot(self, command_id): async with aiohttp.ClientSession() as session: res = await session.get(self.plugin_settings.snapshot_url) @@ -300,7 +300,7 @@ async def _remote_control_snapshot(self, command_id): image=snapshot_io, command=command_id ) - @beeline.traced() + @beeline.traced async def _handle_remote_control_command(self, topic, message): event_type = message.get("octoprint_event_type") @@ -346,7 +346,7 @@ async def _handle_remote_control_command(self, topic, message): metadata=metadata, ) - @beeline.traced() + @beeline.traced async def _loop(self): trace = self._honeycomb_tracer.start_trace() @@ -373,7 +373,7 @@ async def _loop(self): self._honeycomb_tracer.finish_trace(trace) - @beeline.traced() + @beeline.traced async def _handle_config_update(self, topic, message): device_config = print_nanny_client.ExperimentDeviceConfig(**message) diff --git a/tests/mocks/test_remote_control_receive_loop_valid_octoprint_event/octoprint_nanny.plugins/OctoPrintNannyPlugin b/tests/mocks/test_remote_control_receive_loop_valid_octoprint_event/octoprint_nanny.plugins/OctoPrintNannyPlugin new file mode 100644 index 0000000000000000000000000000000000000000..c5414f5f556649464cc5ea91f72028df5ec88f1d GIT binary patch literal 6 NcmZo*t}SHH0{{k!0iXZ? literal 0 HcmV?d00001 diff --git a/tests/test_websocket.py b/tests/test_websocket.py index 921b6d4b..8ac94363 100644 --- a/tests/test_websocket.py +++ b/tests/test_websocket.py @@ -4,7 +4,7 @@ import aiohttp import urllib from datetime import datetime -from octoprint_nanny.clients.websocket import WebSocketWorker +from octoprint_nanny.workers.websocket import WebSocketWorker from octoprint_nanny.predictor import PredictWorker import pytz import threading @@ -12,14 +12,13 @@ @pytest.fixture def ws_client(mocker): - mocker.patch("octoprint_nanny.clients.websocket.asyncio") + mocker.patch("octoprint_nanny.workers.websocket.asyncio") m = aioprocessing.AioManager() return WebSocketWorker( "ws://localhost:8000/ws/", "3a833ac48104772a349254690cae747e826886f1", m.Queue(), 1, - 1, threading.Event(), {}, ) @@ -46,7 +45,7 @@ def predict_worker(mocker): def test_wrong_queue_type_raises(): with pytest.raises(ValueError): WebSocketWorker( - "http://foo.com/ws/", "api_team", queue.Queue(), 1, 1, threading.Event(), {} + "http://foo.com/ws/", "api_team", queue.Queue(), 1, threading.Event(), {} ) From 5009b2d4d21962850214e1e8840922e862dc641a Mon Sep 17 00:00:00 2001 From: Leigh Johnson Date: Mon, 8 Feb 2021 03:53:40 +0000 Subject: [PATCH 08/16] test passing: test_mqtt_send_queue_valid_octoprint_event --- octoprint_nanny/clients/mqtt.py | 8 +- octoprint_nanny/manager.py | 14 +- octoprint_nanny/plugins.py | 6 +- octoprint_nanny/settings.py | 20 +- octoprint_nanny/workers/monitoring.py | 12 +- octoprint_nanny/workers/mqtt.py | 110 ++++++----- setup.cfg | 4 +- tests/{ => clients}/test_mqtt.py | 0 .../OctoPrintNannyPlugin | Bin 309 -> 0 bytes .../OctoPrintNannyPlugin | Bin 505 -> 0 bytes .../OctoPrintNannyPlugin | Bin 6 -> 0 bytes .../OctoPrintNannyPlugin | Bin 309 -> 0 bytes .../OctoPrintNannyPlugin | Bin 309 -> 0 bytes tests/test_manager.py | 183 +++++++++--------- 14 files changed, 189 insertions(+), 168 deletions(-) rename tests/{ => clients}/test_mqtt.py (100%) delete mode 100644 tests/mocks/test_default_settings_client_states/octoprint_nanny.plugins/OctoPrintNannyPlugin delete mode 100644 tests/mocks/test_remote_control_receive_loop_valid_event/octoprint_nanny.plugins/OctoPrintNannyPlugin delete mode 100644 tests/mocks/test_remote_control_receive_loop_valid_octoprint_event/octoprint_nanny.plugins/OctoPrintNannyPlugin delete mode 100644 tests/mocks/test_telemetry_queue_send_loop_bounding_box_predict/octoprint_nanny.plugins/OctoPrintNannyPlugin delete mode 100644 tests/mocks/test_telemetry_queue_send_loop_valid_octoprint_event/octoprint_nanny.plugins/OctoPrintNannyPlugin diff --git a/octoprint_nanny/clients/mqtt.py b/octoprint_nanny/clients/mqtt.py index a5c4ae88..4a69874f 100644 --- a/octoprint_nanny/clients/mqtt.py +++ b/octoprint_nanny/clients/mqtt.py @@ -56,7 +56,7 @@ def __init__( private_key_file: str, ca_certs, algorithm="RS256", - remote_control_queue=None, + mqtt_receive_queue=None, mqtt_bridge_hostname=MQTT_BRIDGE_HOSTNAME, mqtt_bridge_port=MQTT_BRIDGE_PORT, on_connect=None, @@ -91,7 +91,7 @@ def __init__( self.region = region self.algorithm = algorithm - self.remote_control_queue = remote_control_queue + self.mqtt_receive_queue = mqtt_receive_queue self._honeycomb_tracer = HoneycombTracer(service_name="octoprint_plugin") self.client = mqtt.Client(client_id=client_id, protocol=mqtt.MQTTv311) @@ -142,7 +142,7 @@ def _on_message(self, client, userdata, message): logger.info( f"Received remote control command on topic={message.topic} payload={parsed_message}" ) - self.remote_control_queue.put_nowait( + self.mqtt_receive_queue.put_nowait( {"topic": self.remote_control_commands_topic, "message": parsed_message} ) # callback to api to indicate command was received @@ -151,7 +151,7 @@ def _on_message(self, client, userdata, message): logger.info( f"Received config update on topic={message.topic} payload={parsed_message}" ) - self.remote_control_queue.put_nowait( + self.mqtt_receive_queue.put_nowait( {"topic": self.config_topic, "message": parsed_message} ) else: diff --git a/octoprint_nanny/manager.py b/octoprint_nanny/manager.py index 036acdd1..ea3b508b 100644 --- a/octoprint_nanny/manager.py +++ b/octoprint_nanny/manager.py @@ -45,8 +45,6 @@ def __init__(self, plugin): self.event_loop_thread.daemon = True self.event_loop_thread.start() - plugin_settings = PluginSettingsMemoize(plugin) - self.plugin_settings = plugin_settings self._honeycomb_tracer = HoneycombTracer(service_name="octoprint_plugin") self.plugin = plugin @@ -69,9 +67,9 @@ def __init__(self, plugin): mqtt_receive_queue = self.manager.AioQueue() self.mqtt_receive_queue = mqtt_receive_queue - self.mqtt_manager = MQTTManager( - mqtt_send_queue, mqtt_receive_queue, plugin_settings, plugin - ) + plugin_settings = PluginSettingsMemoize(plugin, mqtt_receive_queue) + self.plugin_settings = plugin_settings + self.monitoring_manager = MonitoringManager( octo_ws_queue, pn_ws_queue, @@ -80,6 +78,9 @@ def __init__(self, plugin): self.plugin._event_bus, ) + self.mqtt_manager = MQTTManager( + mqtt_send_queue, mqtt_receive_queue, plugin_settings, plugin + ) # local callback/handler functions for events published via telemetry queue self._mqtt_send_queue_callbacks = { Events.PRINT_STARTED: self.on_print_start, @@ -91,6 +92,9 @@ def __init__(self, plugin): Events.PRINT_RESUMED: self.monitoring_manager.start, Events.SHUTDOWN: self.shutdown, } + self.mqtt_manager.publisher_worker.register_callbacks( + self._mqtt_send_queue_callbacks + ) def _event_loop_worker(self): loop = asyncio.new_event_loop() diff --git a/octoprint_nanny/plugins.py b/octoprint_nanny/plugins.py index 1ea54410..96d1dbe3 100644 --- a/octoprint_nanny/plugins.py +++ b/octoprint_nanny/plugins.py @@ -110,12 +110,10 @@ def __init__(self, *args, **kwargs): self._worker_manager = WorkerManager(plugin=self) self._honeycomb_tracer = HoneycombTracer(service_name="octoprint_plugin") - @beeline.traced("OctoPrintNannyPlugin.get_setting") - @beeline.traced_thread def get_setting(self, key): return self._settings.get([key]) - @beeline.traced("OctoPrintNannyPlugin._test_api_auth") + @beeline.traced @beeline.traced_thread async def _test_api_auth(self, auth_token, api_url): rest_client = RestAPIClient(auth_token=auth_token, api_url=api_url) @@ -128,7 +126,7 @@ async def _test_api_auth(self, auth_token, api_url): self._logger.error(f"_test_api_auth API call failed {e}") self._settings.set(["auth_valid"], False) - @beeline.traced("OctoPrintNannyPlugin._cpuinfo") + @beeline.traced def _cpuinfo(self) -> dict: """ Dict from /proc/cpu diff --git a/octoprint_nanny/settings.py b/octoprint_nanny/settings.py index 94d7338b..b03edf69 100644 --- a/octoprint_nanny/settings.py +++ b/octoprint_nanny/settings.py @@ -20,9 +20,9 @@ class PluginSettingsMemoize: Convenience methods/properties for accessing OctoPrint plugin settings and computed metadata """ - def __init__(self, plugin): + def __init__(self, plugin, mqtt_receive_queue): self.plugin = plugin - + self.mqtt_receive_queue = mqtt_receive_queue # stateful clients and computed settings that require re-initialization when settings change self._calibration = None self._mqtt_client = None @@ -32,23 +32,23 @@ def __init__(self, plugin): self.environment = {} - @beeline.traced("PluginSettingsMemoize.reset_monitoring_settings") + @beeline.traced def reset_monitoring_settings(self): self._calibration = None self._monitoring_frames_per_minute = None - @beeline.traced("PluginSettingsMemoize.reset_device_settings_state") + @beeline.traced @beeline.traced_thread def reset_device_settings_state(self): self._mqtt_client = None self._device_info = None - @beeline.traced("PluginSettingsMemoize.reset_rest_client_state") + @beeline.traced @beeline.traced_thread def reset_rest_client_state(self): self._rest_client = None - @beeline.traced(name="PluginSettingsMemoize.get_device_metadata") + @beeline.traced @beeline.traced_thread def get_device_metadata(self): metadata = dict( @@ -58,18 +58,16 @@ def get_device_metadata(self): metadata.update(self.device_info) return metadata - @beeline.traced(name="PluginSettingsMemoize.get_print_job_metadata") + @beeline.traced @beeline.traced_thread def get_print_job_metadata(self): return dict( printer_data=self.plugin._printer.get_current_data(), printer_profile_data=self.plugin._printer_profile_manager.get_current_or_default(), temperatures=self.plugin._printer.get_current_temperatures(), - printer_profile_id=self.shared.printer_profile_id, - print_job_id=self.shared.print_job_id, ) - @beeline.traced(name="PluginSettingsMemoize.on_environment_detected") + @beeline.traced @beeline.traced_thread def on_environment_detected(self, environment): self.environment = environment @@ -170,7 +168,7 @@ def mqtt_client(self): device_cloudiot_id=self.device_cloudiot_id, private_key_file=self.device_private_key, ca_certs=self.gcp_root_ca, - remote_control_queue=self.remote_control_queue, + mqtt_receive_queue=self.mqtt_receive_queue, trace_context=self.get_device_metadata(), ) return self._mqtt_client diff --git a/octoprint_nanny/workers/monitoring.py b/octoprint_nanny/workers/monitoring.py index 03cfa379..f8c6d30f 100644 --- a/octoprint_nanny/workers/monitoring.py +++ b/octoprint_nanny/workers/monitoring.py @@ -29,9 +29,8 @@ def __init__( self.mqtt_send_queue = mqtt_send_queue self.plugin_settings = plugin_settings self.plugin_event_bus = plugin_event_bus - self.rest_client = plugin_settings.rest_client - @beeline.traced + @beeline.traced("MonitoringManager._drain") def _drain(self): self.halt.set() @@ -39,6 +38,7 @@ def _drain(self): logger.info(f"Waiting for worker={worker} thread to drain") worker.join() + @beeline.traced("MonitoringManager._reset") def _reset(self): self.halt = threading.Event() self._predict_worker = PredictWorker( @@ -63,7 +63,7 @@ def _reset(self): self._workers = [self._predict_worker, self._websocket_worker] self._worker_threads = [] - @beeline.traced + @beeline.traced("MonitoringManager.start") async def start(self): self._reset() @@ -72,13 +72,13 @@ async def start(self): thread.daemon = True self._worker_threads.append(thread) thread.start() - await self.rest_client.update_octoprint_device( + await self.plugin_settings.rest_client.update_octoprint_device( self.plugin_settings.device_id, active=True ) - @beeline.traced + @beeline.traced("MonitoringManager.stop") async def stop(self): self._drain() - await self.rest_client.update_octoprint_device( + await self.plugin_settings.rest_client.update_octoprint_device( self.plugin_settings.device_id, active=False ) diff --git a/octoprint_nanny/workers/mqtt.py b/octoprint_nanny/workers/mqtt.py index b3c6e619..b1ae009e 100644 --- a/octoprint_nanny/workers/mqtt.py +++ b/octoprint_nanny/workers/mqtt.py @@ -37,18 +37,24 @@ def __init__( plugin, ): - super().__init__(self) self.plugin_settings = plugin_settings - self.mqtt_client = plugin_settings.mqtt_client self.mqtt_send_queue = mqtt_send_queue self.mqtt_receive_queue = mqtt_receive_queue halt = threading.Event() self.halt = halt self.plugin = plugin + self.publisher_worker = MQTTPublisherWorker( + self.halt, self.mqtt_send_queue, self.plugin_settings + ) + self.subscriber_worker = MQTTSubscriberWorker( + self.halt, self.mqtt_receive_queue, self.plugin_settings, self.plugin + ) + self.client_worker = MQTTClientWorker(self.halt, self.plugin_settings) self._workers = [] self._worker_threads = [] + @beeline.traced("MQTTManager._drain") def _drain(self): """ Halt running workers and wait pending work @@ -57,10 +63,10 @@ def _drain(self): try: logger.info("Waiting for MQTTManager.mqtt_client network loop to finish") - while self._client_worker.mqtt_client.client.is_connected(): - self.mqtt_client.client.disconnect() + while self.client_worker.plugin_settings.mqtt_client.client.is_connected(): + self.plugin_settings.plugin_settings.mqtt_client.client.disconnect() logger.info("Stopping MQTTManager.mqtt_client network loop") - self.mqtt_client.client.loop_stop() + self.plugin_settings.mqtt_client.client.loop_stop() except PluginSettingsRequired: pass @@ -68,25 +74,27 @@ def _drain(self): logger.info(f"Waiting for worker={worker} thread to drain") worker.join() + @beeline.traced("MQTTManager._reset") def _reset(self): self.halt = threading.Event() - self._publisher_worker = MQTTPublisherWorker( + self.publisher_worker = MQTTPublisherWorker( self.halt, self.mqtt_send_queue, self.plugin_settings ) - self._subscriber_worker = MQTTSubscriberWorker( + self.subscriber_worker = MQTTSubscriberWorker( self.halt, self.mqtt_receive_queue, self.plugin_settings, self.plugin ) - self._client_worker = MQTTClientWorker( + self.client_worker = MQTTClientWorker( self.halt, self.plugin_settings.mqtt_client ) self._workers = [ - self._client_worker, - self._publisher_worker, - self._subscriber_worker, + self.client_worker, + self.publisher_worker, + self.subscriber_worker, ] self._worker_threads = [] + @beeline.traced("MQTTManager.start") def start(self): """ (re)initialize and start worker threads @@ -99,6 +107,7 @@ def start(self): self._worker_threads.append(thread) thread.start() + @beeline.traced("MQTTManager.stop") def stop(self): logger.info("MQTTManager.stop was called") self._drain() @@ -110,15 +119,15 @@ class MQTTClientWorker: https://www.eclipse.org/paho/index.php?page=clients/python/docs/index.php#network-loop """ - def __init__(self, halt, mqtt_client): + def __init__(self, halt, plugin_settings): - self.mqtt_client = mqtt_client + self.plugin_settings = plugin_settings self.halt = halt - @beeline.traced + @beeline.traced("MQTTClientWorker.run") def run(self): try: - return self.mqtt_client.run(self.halt) + return self.plugin_settings.mqtt_client.run(self.halt) except PluginSettingsRequired: pass logger.warning("MQTTClientWorker will exit soon") @@ -147,11 +156,20 @@ def __init__(self, halt, queue, plugin_settings): self.halt = halt self.queue = queue self.plugin_settings = plugin_settings - self.mqtt_client = plugin_settings.mqtt_client self._callbacks = {} self._honeycomb_tracer = HoneycombTracer(service_name="octoprint_plugin") - @beeline.traced + @beeline.traced("MQTTPublisherWorker.register_callbacks") + def register_callbacks(self, callbacks): + for k, v in callbacks.items(): + if self._callbacks.get(k) is None: + self._callbacks[k] = [v] + else: + self._callbacks[k].append(v) + logging.info(f"Registered MQTTSubscribeWorker._callbacks {self._callbacks}") + return self._callbacks + + @beeline.traced("MQTTPublisherWorker.run") def run(self): """ Telemetry worker's event loop is exposed as WorkerManager.loop @@ -163,7 +181,7 @@ def run(self): return loop.run_until_complete(asyncio.ensure_future(self.loop_forever())) - @beeline.traced + @beeline.traced("MQTTPublisherWorker._publish_octoprint_event_telemetry") async def _publish_octoprint_event_telemetry(self, event): event_type = event.get("event_type") logger.info(f"_publish_octoprint_event_telemetry {event}") @@ -178,9 +196,9 @@ async def _publish_octoprint_event_telemetry(self, event): if event_type in self.PRINT_JOB_EVENTS: event.update(self.plugin_settings.get_print_job_metadata()) - self.mqtt_client.publish_octoprint_event(event) + self.plugin_settings.mqtt_client.publish_octoprint_event(event) - @beeline.traced + @beeline.traced("MQTTPublisherWorker._publish_bounding_box_telemetry") async def _publish_bounding_box_telemetry(self, event): event.update( dict( @@ -189,9 +207,9 @@ async def _publish_bounding_box_telemetry(self, event): device_cloudiot_name=self.plugin_settings.device_cloudiot_name, ) ) - self.mqtt_client.publish_bounding_boxes(event) + self.plugin_settings.mqtt_client.publish_bounding_boxes(event) - @beeline.traced + @beeline.traced("MQTTPublisherWorker._loop") async def _loop(self): try: span = self._honeycomb_tracer.start_span( @@ -219,21 +237,20 @@ async def _loop(self): await self._publish_bounding_box_telemetry(event) return - if self.event_in_tracked_telemetry(event_type): + if self.plugin_settings.event_in_tracked_telemetry(event_type): await self._publish_octoprint_event_telemetry(event) else: if event_type not in self.MUTED_EVENTS: logger.warning(f"Discarding {event_type} with payload {event}") return - # run local handler fn - handler_fn = self._callbacks.get(event_type) - if handler_fn: - - if inspect.isawaitable(handler_fn): - await handler_fn(**event) - else: - handler_fn(**event) + handler_fns = self._callbacks.get(event_type) + for handler_fn in handler_fns: + if handler_fn: + if inspect.isawaitable(handler_fn): + await handler_fn(**event) + else: + handler_fn(**event) except API_CLIENT_EXCEPTIONS as e: logger.error(f"REST client raised exception {e}", exc_info=True) @@ -257,12 +274,10 @@ def __init__(self, halt, queue, plugin_settings, plugin): self.queue = queue self.plugin_settings = plugin_settings self.plugin = plugin - self.mqtt_client = plugin_settings.mqtt_client - self.rest_client = plugin_settings.rest_client self._callbacks = {} self._honeycomb_tracer = HoneycombTracer(service_name="octoprint_plugin") - @beeline.traced + @beeline.traced("MQTTSubscriberWorker.run") def run(self): self.loop = asyncio.new_event_loop() asyncio.set_event_loop(self.loop) @@ -272,6 +287,7 @@ def run(self): return self.loop.run_until_complete(asyncio.ensure_future(self.loop_forever())) + @beeline.traced("MQTTSubscriberWorker.register_callbacks") def register_callbacks(self, callbacks): for k, v in callbacks.items(): if self._callbacks.get(k) is None: @@ -290,17 +306,17 @@ async def loop_forever(self): pass logger.info("Exiting soon MQTTSubscribeWorker.loop_forever") - @beeline.traced + @beeline.traced("MQTTSubscriberWorker._remote_control_snapshot") async def _remote_control_snapshot(self, command_id): async with aiohttp.ClientSession() as session: res = await session.get(self.plugin_settings.snapshot_url) snapshot_io = io.BytesIO(await res.read()) - return await self.rest_client.create_snapshot( + return await self.plugin_settings.rest_client.create_snapshot( image=snapshot_io, command=command_id ) - @beeline.traced + @beeline.traced("MQTTSubscriberWorker._handle_remote_control_command") async def _handle_remote_control_command(self, topic, message): event_type = message.get("octoprint_event_type") @@ -313,7 +329,7 @@ async def _handle_remote_control_command(self, topic, message): await self._remote_control_snapshot(command_id) metadata = self.plugin_settings.get_device_metadata() - await self.rest_client.update_remote_control_command( + await self.plugin_settings.rest_client.update_remote_control_command( command_id, received=True, metadata=metadata ) @@ -332,7 +348,7 @@ async def _handle_remote_control_command(self, topic, message): metadata = self.plugin_settings.get_device_metadata() # set success state - await self.rest_client.update_remote_control_command( + await self.plugin_settings.rest_client.update_remote_control_command( command_id, success=True, metadata=metadata, @@ -340,20 +356,24 @@ async def _handle_remote_control_command(self, topic, message): except Exception as e: logger.error(f"Error calling handler_fn {handler_fn} \n {e}") metadata = self.plugin_settings.get_device_metadata() - await self.rest_client.update_remote_control_command( + await self.plugin_settings.rest_client.update_remote_control_command( command_id, success=False, metadata=metadata, ) - @beeline.traced + @beeline.traced("MQTTSubscriberWorker._loop") async def _loop(self): trace = self._honeycomb_tracer.start_trace() span = self._honeycomb_tracer.start_span( {"name": "WorkerManager.queue.coro_get"} ) - payload = await self.queue.coro_get() + + try: + event = self.queue.get_nowait() + except queue.Empty: + return self._honeycomb_tracer.add_context(dict(event=payload)) self._honeycomb_tracer.finish_span(span) @@ -362,10 +382,10 @@ async def _loop(self): if topic is None: logger.warning("Ignoring received message where topic=None") - elif topic == self.mqtt_client.remote_control_command_topic: + elif topic == self.plugin_settings.mqtt_client.remote_control_command_topic: await self._handle_remote_control_command(**payload) - elif topic == self.mqtt_client.config_topic: + elif topic == self.plugin_settings.mqtt_client.config_topic: await self._handle_config_update(**payload) else: @@ -373,7 +393,7 @@ async def _loop(self): self._honeycomb_tracer.finish_trace(trace) - @beeline.traced + @beeline.traced("MQTTSubscriberWorker._handle_config_updat") async def _handle_config_update(self, topic, message): device_config = print_nanny_client.ExperimentDeviceConfig(**message) diff --git a/setup.cfg b/setup.cfg index 80012c51..d1be5824 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,11 +1,13 @@ [bumpversion] -current_version = 0.5.0 +current_version = 0.4.3 commit = True tag = True [tool:pytest] markers = webapp: marks webapp integration tests (deselect with -m "not webapp") + slow: marks webapp integration tests (deselect with -m "not slow") + env = HONEYCOMB_DEBUG=True addopts = -p no:warnings diff --git a/tests/test_mqtt.py b/tests/clients/test_mqtt.py similarity index 100% rename from tests/test_mqtt.py rename to tests/clients/test_mqtt.py diff --git a/tests/mocks/test_default_settings_client_states/octoprint_nanny.plugins/OctoPrintNannyPlugin b/tests/mocks/test_default_settings_client_states/octoprint_nanny.plugins/OctoPrintNannyPlugin deleted file mode 100644 index b025e5fd27007d8924b41729818bc550e17bca81..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 309 zcmZvW%}N6?6osR${fV`6 zjty6TqY+0Vj#|q5U@=w`UNl3OwZ=q^c|8(3zK7{Xo(F68V227)4VmRW5^n!D(if#S zE=bHuIrZM)?i2G@+?PW~N-_9ESiGni|K^gWlcAVh@K}BqGx8=_!h_M#nL|U;c5O@& TkA+$)boED@P+O)?-7Zx>C2?Js diff --git a/tests/mocks/test_remote_control_receive_loop_valid_event/octoprint_nanny.plugins/OctoPrintNannyPlugin b/tests/mocks/test_remote_control_receive_loop_valid_event/octoprint_nanny.plugins/OctoPrintNannyPlugin deleted file mode 100644 index c22bcdaa4971df21e9a271c9672fcde51df8437f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 505 zcmZut%TB{E5KIWP#XL&O8!oxEz2qA}6{kpm5Qp5X#96RuYG-2ykpogWP{CibjQ|OW zi=~|z@6OC3ctvjv&o_GdE%utsXW_i-O_WAvK(Fnluj9qgU{9(Xkl`| zq2Q$wvjSLN&r8d2zh!f1*@IuUFlkQgVVzN1{t}Pms*>YZJQ+*MFFUa>yGfspJ%Zyo zJgc^z2ZN{bZ@@FTCHr!3I$H!Ht5xBsO?K&ArLx+=RaKdap_g3Hs?m*b=C)GWU8h<% zw?+M^^e8mh69^Z|)P~xG>Ce}2#LHH}pP{l8b%g#quH9h};kg`q$f4Yq2a+Px(H9Ub Bn->58 diff --git a/tests/mocks/test_remote_control_receive_loop_valid_octoprint_event/octoprint_nanny.plugins/OctoPrintNannyPlugin b/tests/mocks/test_remote_control_receive_loop_valid_octoprint_event/octoprint_nanny.plugins/OctoPrintNannyPlugin deleted file mode 100644 index c5414f5f556649464cc5ea91f72028df5ec88f1d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6 NcmZo*t}SHH0{{k!0iXZ? diff --git a/tests/mocks/test_telemetry_queue_send_loop_bounding_box_predict/octoprint_nanny.plugins/OctoPrintNannyPlugin b/tests/mocks/test_telemetry_queue_send_loop_bounding_box_predict/octoprint_nanny.plugins/OctoPrintNannyPlugin deleted file mode 100644 index b025e5fd27007d8924b41729818bc550e17bca81..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 309 zcmZvW%}N6?6osR${fV`6 zjty6TqY+0Vj#|q5U@=w`UNl3OwZ=q^c|8(3zK7{Xo(F68V227)4VmRW5^n!D(if#S zE=bHuIrZM)?i2G@+?PW~N-_9ESiGni|K^gWlcAVh@K}BqGx8=_!h_M#nL|U;c5O@& TkA+$)boED@P+O)?-7Zx>C2?Js diff --git a/tests/mocks/test_telemetry_queue_send_loop_valid_octoprint_event/octoprint_nanny.plugins/OctoPrintNannyPlugin b/tests/mocks/test_telemetry_queue_send_loop_valid_octoprint_event/octoprint_nanny.plugins/OctoPrintNannyPlugin deleted file mode 100644 index b025e5fd27007d8924b41729818bc550e17bca81..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 309 zcmZvW%}N6?6osR${fV`6 zjty6TqY+0Vj#|q5U@=w`UNl3OwZ=q^c|8(3zK7{Xo(F68V227)4VmRW5^n!D(if#S zE=bHuIrZM)?i2G@+?PW~N-_9ESiGni|K^gWlcAVh@K}BqGx8=_!h_M#nL|U;c5O@& TkA+$)boED@P+O)?-7Zx>C2?Js diff --git a/tests/test_manager.py b/tests/test_manager.py index c7d65f73..a4478d7b 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -13,12 +13,6 @@ ) -@pytest.fixture(autouse=True) -def mock_plugin(automock): - with automock((octoprint_nanny.plugins, "OctoPrintNannyPlugin"), unlocked=True): - yield - - def get_default_setting(key): return octoprint_nanny.plugins.DEFAULT_SETTINGS[key] @@ -27,135 +21,140 @@ def test_default_settings_client_states(mocker): """ By default, accessing WorkerManger.rest_client and WorkerManger.mqtt_client should raise PluginSettingsRequired """ - plugin = octoprint_nanny.plugins.OctoPrintNannyPlugin() + plugin = mocker.Mock() plugin.get_setting = get_default_setting manager = WorkerManager(plugin) - assert manager.auth_token is None - assert manager.device_id is None + assert manager.plugin_settings.auth_token is None + assert manager.plugin_settings.device_id is None with pytest.raises(PluginSettingsRequired): - dir(manager.mqtt_client) + dir(manager.plugin_settings.mqtt_client) with pytest.raises(PluginSettingsRequired): - dir(manager.rest_client) + dir(manager.plugin_settings.rest_client) @pytest.mark.asyncio -async def test_telemetry_queue_send_loop_valid_octoprint_event(mocker): - plugin = octoprint_nanny.plugins.OctoPrintNannyPlugin() +async def test_mqtt_send_queue_valid_octoprint_event(mocker): + plugin = mocker.Mock() plugin.get_setting = get_default_setting - mocker.patch.object(WorkerManager, "test_mqtt_settings") + mocker.patch("octoprint_nanny.settings.PluginSettingsMemoize.test_mqtt_settings") - mocker.patch.object(WorkerManager, "telemetry_events") - mocker.patch.object(WorkerManager, "event_in_tracked_telemetry", return_value=True) + mocker.patch("octoprint_nanny.settings.PluginSettingsMemoize.telemetry_events") + mocker.patch( + "octoprint_nanny.settings.PluginSettingsMemoize.event_in_tracked_telemetry", + return_value=True, + ) mock_on_print_start = mocker.patch.object(WorkerManager, "on_print_start") manager = WorkerManager(plugin) - event = {"event_type": Events.PRINT_STARTED, "event_data": {}} - manager.telemetry_queue.put_nowait(event) + manager.mqtt_send_queue.put_nowait(event) mock_publish_octoprint_event_telemetry = mocker.patch.object( - manager, "_publish_octoprint_event_telemetry", return_value=asyncio.Future() + manager.mqtt_manager.publisher_worker, + "_publish_octoprint_event_telemetry", + return_value=asyncio.Future(), ) mock_publish_octoprint_event_telemetry.return_value.set_result("foo") - await manager._telemetry_queue_send_loop() + await manager.mqtt_manager.publisher_worker._loop() mock_publish_octoprint_event_telemetry.assert_called_once_with(event) + mock_on_print_start.assert_called_once_with( event_data=event["event_data"], event_type=event["event_type"] ) -@pytest.mark.asyncio -async def test_telemetry_queue_send_loop_bounding_box_predict(mocker): - plugin = octoprint_nanny.plugins.OctoPrintNannyPlugin() - plugin.get_setting = get_default_setting +# @pytest.mark.asyncio +# async def test_telemetry_queue_send_loop_bounding_box_predict(mocker): +# plugin = octoprint_nanny.plugins.OctoPrintNannyPlugin() +# plugin.get_setting = get_default_setting - mocker.patch.object(WorkerManager, "test_mqtt_settings") +# mocker.patch.object(WorkerManager, "test_mqtt_settings") - mocker.patch.object(WorkerManager, "telemetry_events") - mocker.patch.object(WorkerManager, "event_in_tracked_telemetry", return_value=True) +# mocker.patch.object(WorkerManager, "telemetry_events") +# mocker.patch.object(WorkerManager, "event_in_tracked_telemetry", return_value=True) - manager = WorkerManager(plugin) +# manager = WorkerManager(plugin) - event = {"event_type": BOUNDING_BOX_PREDICT_EVENT, "event_data": {}} - manager.telemetry_queue.put_nowait(event) +# event = {"event_type": BOUNDING_BOX_PREDICT_EVENT, "event_data": {}} +# manager.telemetry_queue.put_nowait(event) - mock_fn = mocker.patch.object( - manager, "_publish_bounding_box_telemetry", return_value=asyncio.Future() - ) - mock_fn.return_value.set_result("foo") +# mock_fn = mocker.patch.object( +# manager, "_publish_bounding_box_telemetry", return_value=asyncio.Future() +# ) +# mock_fn.return_value.set_result("foo") - await manager._telemetry_queue_send_loop() +# await manager._telemetry_queue_send_loop() - mock_fn.assert_called_once_with(event) +# mock_fn.assert_called_once_with(event) -@pytest.mark.asyncio -async def test_remote_control_receive_loop_valid_octoprint_event(mocker): - plugin = octoprint_nanny.plugins.OctoPrintNannyPlugin() - plugin.get_setting = get_default_setting +# @pytest.mark.asyncio +# async def test_remote_control_receive_loop_valid_octoprint_event(mocker): +# plugin = octoprint_nanny.plugins.OctoPrintNannyPlugin() +# plugin.get_setting = get_default_setting - mocker.patch.object(WorkerManager, "test_mqtt_settings") +# mocker.patch.object(WorkerManager, "test_mqtt_settings") - mock_remote_control_snapshot = mocker.patch.object( - WorkerManager, "_remote_control_snapshot", return_value=asyncio.Future() - ) - mock_remote_control_snapshot.return_value.set_result("foo") +# mock_remote_control_snapshot = mocker.patch.object( +# WorkerManager, "_remote_control_snapshot", return_value=asyncio.Future() +# ) +# mock_remote_control_snapshot.return_value.set_result("foo") - mock_rest_client = mocker.patch.object(WorkerManager, "rest_client") - mock_rest_client.update_remote_control_command.return_value = asyncio.Future() - mock_rest_client.update_remote_control_command.return_value.set_result("foo") +# mock_rest_client = mocker.patch.object(WorkerManager, "rest_client") +# mock_rest_client.update_remote_control_command.return_value = asyncio.Future() +# mock_rest_client.update_remote_control_command.return_value.set_result("foo") - mock_mqtt_client = mocker.patch.object(WorkerManager, "mqtt_client") +# mock_mqtt_client = mocker.patch.object(WorkerManager, "mqtt_client") - topic = "remote-control-topic" - type(mock_mqtt_client).remote_control_command_topic = PropertyMock( - return_value=topic - ) +# topic = "remote-control-topic" +# type(mock_mqtt_client).remote_control_command_topic = PropertyMock( +# return_value=topic +# ) - mocker.patch.object(WorkerManager, "get_device_metadata", return_value={}) +# mocker.patch.object(WorkerManager, "get_device_metadata", return_value={}) - mock_start_monitoring = mocker.patch.object(WorkerManager, "start_monitoring") +# mock_start_monitoring = mocker.patch.object(WorkerManager, "start_monitoring") - manager = WorkerManager(plugin) - manager._remote_control_event_handlers = { - "octoprint_nanny_plugin_monitoring_start": manager.start_monitoring - } - - command = { - "message": { - "octoprint_event_type": "octoprint_nanny_plugin_monitoring_start", - "command": "MonitoringStart", - "remote_control_command_id": 1, - }, - "topic": topic, - } - manager.remote_control_queue.put_nowait(command) - - await manager._remote_control_receive_loop() - - mock_remote_control_snapshot.assert_called_once_with( - command["message"]["remote_control_command_id"] - ) +# manager = WorkerManager(plugin) +# manager._remote_control_event_handlers = { +# "octoprint_nanny_plugin_monitoring_start": manager.start_monitoring +# } - mock_rest_client.update_remote_control_command.assert_has_calls( - [ - mocker.call( - command["message"]["remote_control_command_id"], - received=True, - metadata={}, - ), - mocker.call( - command["message"]["remote_control_command_id"], - success=True, - metadata={}, - ), - ] - ) - mock_start_monitoring.assert_called_once_with( - event=command["message"], event_type=command["message"]["octoprint_event_type"] - ) +# command = { +# "message": { +# "octoprint_event_type": "octoprint_nanny_plugin_monitoring_start", +# "command": "MonitoringStart", +# "remote_control_command_id": 1, +# }, +# "topic": topic, +# } +# manager.remote_control_queue.put_nowait(command) + +# await manager._remote_control_receive_loop() + +# mock_remote_control_snapshot.assert_called_once_with( +# command["message"]["remote_control_command_id"] +# ) + +# mock_rest_client.update_remote_control_command.assert_has_calls( +# [ +# mocker.call( +# command["message"]["remote_control_command_id"], +# received=True, +# metadata={}, +# ), +# mocker.call( +# command["message"]["remote_control_command_id"], +# success=True, +# metadata={}, +# ), +# ] +# ) +# mock_start_monitoring.assert_called_once_with( +# event=command["message"], event_type=command["message"]["octoprint_event_type"] +# ) From f6abdd8e5cdc900c6efb2872cec00e90b3b25a83 Mon Sep 17 00:00:00 2001 From: Leigh Johnson Date: Mon, 8 Feb 2021 03:56:08 +0000 Subject: [PATCH 09/16] test passing for test_telemetry_queue_send_loop_bounding_box_predict --- octoprint_nanny/workers/mqtt.py | 2 +- tests/test_manager.py | 37 +++++++++++++++++++-------------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/octoprint_nanny/workers/mqtt.py b/octoprint_nanny/workers/mqtt.py index b1ae009e..1077aacc 100644 --- a/octoprint_nanny/workers/mqtt.py +++ b/octoprint_nanny/workers/mqtt.py @@ -371,7 +371,7 @@ async def _loop(self): ) try: - event = self.queue.get_nowait() + payload = self.queue.get_nowait() except queue.Empty: return self._honeycomb_tracer.add_context(dict(event=payload)) diff --git a/tests/test_manager.py b/tests/test_manager.py index a4478d7b..937bac9f 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -68,29 +68,34 @@ async def test_mqtt_send_queue_valid_octoprint_event(mocker): ) -# @pytest.mark.asyncio -# async def test_telemetry_queue_send_loop_bounding_box_predict(mocker): -# plugin = octoprint_nanny.plugins.OctoPrintNannyPlugin() -# plugin.get_setting = get_default_setting +@pytest.mark.asyncio +async def test_telemetry_queue_send_loop_bounding_box_predict(mocker): + plugin = mocker.Mock() + plugin.get_setting = get_default_setting -# mocker.patch.object(WorkerManager, "test_mqtt_settings") + mocker.patch("octoprint_nanny.settings.PluginSettingsMemoize.test_mqtt_settings") -# mocker.patch.object(WorkerManager, "telemetry_events") -# mocker.patch.object(WorkerManager, "event_in_tracked_telemetry", return_value=True) + mocker.patch("octoprint_nanny.settings.PluginSettingsMemoize.telemetry_events") + mocker.patch( + "octoprint_nanny.settings.PluginSettingsMemoize.event_in_tracked_telemetry", + return_value=True, + ) -# manager = WorkerManager(plugin) + manager = WorkerManager(plugin) -# event = {"event_type": BOUNDING_BOX_PREDICT_EVENT, "event_data": {}} -# manager.telemetry_queue.put_nowait(event) + event = {"event_type": BOUNDING_BOX_PREDICT_EVENT, "event_data": {}} + manager.mqtt_send_queue.put_nowait(event) -# mock_fn = mocker.patch.object( -# manager, "_publish_bounding_box_telemetry", return_value=asyncio.Future() -# ) -# mock_fn.return_value.set_result("foo") + mock_fn = mocker.patch.object( + manager.mqtt_manager.publisher_worker, + "_publish_bounding_box_telemetry", + return_value=asyncio.Future(), + ) + mock_fn.return_value.set_result("foo") -# await manager._telemetry_queue_send_loop() + await manager.mqtt_manager.publisher_worker._loop() -# mock_fn.assert_called_once_with(event) + mock_fn.assert_called_once_with(event) # @pytest.mark.asyncio From 5cd82a64cb44c31c2698302ea5887eb2a0771921 Mon Sep 17 00:00:00 2001 From: Leigh Johnson Date: Mon, 8 Feb 2021 04:30:12 +0000 Subject: [PATCH 10/16] test passing test_mqtt_receive_queue_valid_octoprint_event --- octoprint_nanny/manager.py | 32 ++++---- octoprint_nanny/settings.py | 17 ++--- tests/test_manager.py | 148 ++++++++++++++++++++---------------- 3 files changed, 105 insertions(+), 92 deletions(-) diff --git a/octoprint_nanny/manager.py b/octoprint_nanny/manager.py index ea3b508b..d3b2d01e 100644 --- a/octoprint_nanny/manager.py +++ b/octoprint_nanny/manager.py @@ -108,24 +108,26 @@ def _register_plugin_event_handlers(self): """ Events.PLUGIN_OCTOPRINT_NANNY* events are not available on Events until plugin is fully initialized """ - pass - # self._local_event_handlers.update( - # { - # Events.PLUGIN_OCTOPRINT_NANNY_MONITORING_START: self.start_monitoring, - # Events.PLUGIN_OCTOPRINT_NANNY_MONITORING_STOP: self.stop_monitoring, - # } - # ) - # self._remote_control_event_handlers.update( - # { - # Events.PLUGIN_OCTOPRINT_NANNY_MONITORING_START: self.start_monitoring, - # Events.PLUGIN_OCTOPRINT_NANNY_MONITORING_STOP: self.stop_monitoring, - # } - # ) + self.mqtt_manager.publisher_worker.register_callbacks( + { + Events.PLUGIN_OCTOPRINT_NANNY_MONITORING_START: self.monitoring_manager.start, + Events.PLUGIN_OCTOPRINT_NANNY_MONITORING_STOP: self.monitoring_manager.stop, + } + ) + + self.mqtt_manager.subscriber_worker.register_callbacks( + { + Events.PLUGIN_OCTOPRINT_NANNY_MONITORING_START: self.monitoring_manager.start, + Events.PLUGIN_OCTOPRINT_NANNY_MONITORING_STOP: self.monitoring_manager.stop, + } + ) @beeline.traced def on_settings_initialized(self): - self._honeycomb_tracer.add_global_context(self.get_device_metadata()) - # self._register_plugin_event_handlers() + self._honeycomb_tracer.add_global_context( + self.plugin_settings.get_device_metadata() + ) + self._register_plugin_event_handlers() self.mqtt_manager.start() @beeline.traced diff --git a/octoprint_nanny/settings.py b/octoprint_nanny/settings.py index b03edf69..350f22a4 100644 --- a/octoprint_nanny/settings.py +++ b/octoprint_nanny/settings.py @@ -32,24 +32,21 @@ def __init__(self, plugin, mqtt_receive_queue): self.environment = {} - @beeline.traced + @beeline.traced("PluginSettingsMemoize.reset_monitoring_settings") def reset_monitoring_settings(self): self._calibration = None self._monitoring_frames_per_minute = None - @beeline.traced - @beeline.traced_thread + @beeline.traced("PluginSettingsMemoize.reset_device_settings_state") def reset_device_settings_state(self): self._mqtt_client = None self._device_info = None - @beeline.traced - @beeline.traced_thread + @beeline.traced("PluginSettingsMemoize.reset_rest_client_state") def reset_rest_client_state(self): self._rest_client = None - @beeline.traced - @beeline.traced_thread + @beeline.traced("PluginSettingsMemoize.get_device_metadata") def get_device_metadata(self): metadata = dict( created_dt=datetime.now(pytz.timezone("UTC")), @@ -58,8 +55,7 @@ def get_device_metadata(self): metadata.update(self.device_info) return metadata - @beeline.traced - @beeline.traced_thread + @beeline.traced("PluginSettingsMemoize.get_print_job_metadata") def get_print_job_metadata(self): return dict( printer_data=self.plugin._printer.get_current_data(), @@ -67,8 +63,7 @@ def get_print_job_metadata(self): temperatures=self.plugin._printer.get_current_temperatures(), ) - @beeline.traced - @beeline.traced_thread + @beeline.traced("PluginSettingsMemoize.on_environment_detected") def on_environment_detected(self, environment): self.environment = environment diff --git a/tests/test_manager.py b/tests/test_manager.py index 937bac9f..6e277789 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -69,7 +69,7 @@ async def test_mqtt_send_queue_valid_octoprint_event(mocker): @pytest.mark.asyncio -async def test_telemetry_queue_send_loop_bounding_box_predict(mocker): +async def test_mqtt_send_queue_bounding_box_predict(mocker): plugin = mocker.Mock() plugin.get_setting = get_default_setting @@ -98,68 +98,84 @@ async def test_telemetry_queue_send_loop_bounding_box_predict(mocker): mock_fn.assert_called_once_with(event) -# @pytest.mark.asyncio -# async def test_remote_control_receive_loop_valid_octoprint_event(mocker): -# plugin = octoprint_nanny.plugins.OctoPrintNannyPlugin() -# plugin.get_setting = get_default_setting - -# mocker.patch.object(WorkerManager, "test_mqtt_settings") - -# mock_remote_control_snapshot = mocker.patch.object( -# WorkerManager, "_remote_control_snapshot", return_value=asyncio.Future() -# ) -# mock_remote_control_snapshot.return_value.set_result("foo") - -# mock_rest_client = mocker.patch.object(WorkerManager, "rest_client") -# mock_rest_client.update_remote_control_command.return_value = asyncio.Future() -# mock_rest_client.update_remote_control_command.return_value.set_result("foo") - -# mock_mqtt_client = mocker.patch.object(WorkerManager, "mqtt_client") - -# topic = "remote-control-topic" -# type(mock_mqtt_client).remote_control_command_topic = PropertyMock( -# return_value=topic -# ) - -# mocker.patch.object(WorkerManager, "get_device_metadata", return_value={}) - -# mock_start_monitoring = mocker.patch.object(WorkerManager, "start_monitoring") - -# manager = WorkerManager(plugin) -# manager._remote_control_event_handlers = { -# "octoprint_nanny_plugin_monitoring_start": manager.start_monitoring -# } - -# command = { -# "message": { -# "octoprint_event_type": "octoprint_nanny_plugin_monitoring_start", -# "command": "MonitoringStart", -# "remote_control_command_id": 1, -# }, -# "topic": topic, -# } -# manager.remote_control_queue.put_nowait(command) - -# await manager._remote_control_receive_loop() - -# mock_remote_control_snapshot.assert_called_once_with( -# command["message"]["remote_control_command_id"] -# ) - -# mock_rest_client.update_remote_control_command.assert_has_calls( -# [ -# mocker.call( -# command["message"]["remote_control_command_id"], -# received=True, -# metadata={}, -# ), -# mocker.call( -# command["message"]["remote_control_command_id"], -# success=True, -# metadata={}, -# ), -# ] -# ) -# mock_start_monitoring.assert_called_once_with( -# event=command["message"], event_type=command["message"]["octoprint_event_type"] -# ) +@pytest.mark.asyncio +async def test_mqtt_receive_queue_valid_octoprint_event(mocker): + plugin = mocker.Mock() + plugin.get_setting = get_default_setting + + mocker.patch("octoprint_nanny.settings.PluginSettingsMemoize.test_mqtt_settings") + mocker.patch("octoprint_nanny.settings.PluginSettingsMemoize.telemetry_events") + mocker.patch( + "octoprint_nanny.settings.PluginSettingsMemoize.event_in_tracked_telemetry", + return_value=True, + ) + + manager = WorkerManager(plugin) + + mock_remote_control_snapshot = mocker.patch( + "octoprint_nanny.workers.mqtt.MQTTSubscriberWorker._remote_control_snapshot", + return_value=asyncio.Future(), + ) + mock_remote_control_snapshot.return_value.set_result("foo") + + mock_rest_client = mocker.patch( + "octoprint_nanny.settings.PluginSettingsMemoize.rest_client" + ) + mock_rest_client.create_snapshot.return_value = asyncio.Future() + mock_rest_client.create_snapshot.return_value.set_result("foo") + mock_rest_client.update_remote_control_command.return_value = asyncio.Future() + mock_rest_client.update_remote_control_command.return_value.set_result("foo") + + mock_mqtt_client = mocker.patch( + "octoprint_nanny.settings.PluginSettingsMemoize.mqtt_client" + ) + + topic = "remote-control-topic" + type(mock_mqtt_client).remote_control_command_topic = PropertyMock( + return_value=topic + ) + mocker.patch( + "octoprint_nanny.settings.PluginSettingsMemoize.get_device_metadata", + return_value={}, + ) + + mock_start_monitoring = mocker.patch.object(manager.monitoring_manager, "start") + + manager = WorkerManager(plugin) + manager.mqtt_manager.subscriber_worker.register_callbacks( + {"octoprint_nanny_plugin_monitoring_start": mock_start_monitoring} + ) + + command = { + "message": { + "octoprint_event_type": "octoprint_nanny_plugin_monitoring_start", + "command": "MonitoringStart", + "remote_control_command_id": 1, + }, + "topic": topic, + } + manager.mqtt_manager.mqtt_receive_queue.put_nowait(command) + + await manager.mqtt_manager.subscriber_worker._loop() + + mock_remote_control_snapshot.assert_called_once_with( + command["message"]["remote_control_command_id"] + ) + + mock_rest_client.update_remote_control_command.assert_has_calls( + [ + mocker.call( + command["message"]["remote_control_command_id"], + received=True, + metadata={}, + ), + mocker.call( + command["message"]["remote_control_command_id"], + success=True, + metadata={}, + ), + ] + ) + mock_start_monitoring.assert_called_once_with( + event=command["message"], event_type=command["message"]["octoprint_event_type"] + ) From cc147cf3838452d96f534924c0aa6f7a346ff6ce Mon Sep 17 00:00:00 2001 From: Leigh Johnson Date: Mon, 8 Feb 2021 04:35:55 +0000 Subject: [PATCH 11/16] fix trace decorators in WorkerManager class --- octoprint_nanny/manager.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/octoprint_nanny/manager.py b/octoprint_nanny/manager.py index d3b2d01e..4bbf2db0 100644 --- a/octoprint_nanny/manager.py +++ b/octoprint_nanny/manager.py @@ -103,7 +103,7 @@ def _event_loop_worker(self): self.loop = loop return loop.run_forever() - @beeline.traced + @beeline.traced("WorkerManager._register_plugin_event_handlers") def _register_plugin_event_handlers(self): """ Events.PLUGIN_OCTOPRINT_NANNY* events are not available on Events until plugin is fully initialized @@ -122,7 +122,7 @@ def _register_plugin_event_handlers(self): } ) - @beeline.traced + @beeline.traced("WorkerManager.on_settings_initialized") def on_settings_initialized(self): self._honeycomb_tracer.add_global_context( self.plugin_settings.get_device_metadata() @@ -130,37 +130,37 @@ def on_settings_initialized(self): self._register_plugin_event_handlers() self.mqtt_manager.start() - @beeline.traced + @beeline.traced("WorkerManager.apply_device_registration") def apply_device_registration(self): self.mqtt_manager.stop() logger.info("Resetting WorkerManager device registration state") self.plugin_settings.reset_device_settings_state() self.mqtt_manager.start() - @beeline.traced + @beeline.traced("WorkerManager.apply_auth") def apply_auth(self): logger.info("Resetting WorkerManager user auth state") self.mqtt_manager.stop() self.plugin_settings.reset_rest_client_state() self.mqtt_manager.start() - @beeline.traced + @beeline.traced("WorkerManager.apply_monitoring_settings") def apply_monitoring_settings(self): - self.reset_monitoring_settings() + self.plugin_settings.reset_monitoring_settings() logger.info( "Stopping any existing monitoring processes to apply new calibration" ) - self.stop_monitoring() + self.monitoring_manager.stop() if self.monitoring_active: logger.info( "Monitoring was active when new calibration was applied. Re-initializing monitoring processes" ) - self.start_monitoring() + self.monitoring_manager.start() - @beeline.traced + @beeline.traced("WorkerManager.shutdown") def shutdown(self): - self.stop_monitoring() + self.monitoring_manager.stop() asyncio.run_coroutine_threadsafe( self.rest_client.update_octoprint_device( @@ -169,10 +169,10 @@ def shutdown(self): self.loop, ).result() - self.stop_worker_threads() + self.mqtt_manager.stop() self._honeycomb_tracer.on_shutdown() - @beeline.traced + @beeline.traced("WorkerManager.on_print_start") async def on_print_start(self, event_type, event_data, **kwargs): logger.info(f"on_print_start called for {event_type} with data {event_data}") try: From 3ca388cdf74d05a533e2bf628f1f018781b96a78 Mon Sep 17 00:00:00 2001 From: Leigh Johnson Date: Mon, 8 Feb 2021 04:37:03 +0000 Subject: [PATCH 12/16] fix beeline.traced decorators in plugins module --- octoprint_nanny/plugins.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octoprint_nanny/plugins.py b/octoprint_nanny/plugins.py index 96d1dbe3..ec68eb52 100644 --- a/octoprint_nanny/plugins.py +++ b/octoprint_nanny/plugins.py @@ -113,7 +113,7 @@ def __init__(self, *args, **kwargs): def get_setting(self, key): return self._settings.get([key]) - @beeline.traced + @beeline.traced("OctoPrintNannyPlugin._test_api_auth") @beeline.traced_thread async def _test_api_auth(self, auth_token, api_url): rest_client = RestAPIClient(auth_token=auth_token, api_url=api_url) @@ -126,7 +126,7 @@ async def _test_api_auth(self, auth_token, api_url): self._logger.error(f"_test_api_auth API call failed {e}") self._settings.set(["auth_valid"], False) - @beeline.traced + @beeline.traced("OctoPrintNannyPlugin._cpuinfo") def _cpuinfo(self) -> dict: """ Dict from /proc/cpu From e8cef51c2f0225fdb52c53c70272429427e9b255 Mon Sep 17 00:00:00 2001 From: Leigh Johnson Date: Mon, 8 Feb 2021 05:03:29 +0000 Subject: [PATCH 13/16] server boots --- octoprint_nanny/manager.py | 42 +++++++------ octoprint_nanny/plugins.py | 34 +++++------ octoprint_nanny/predictor.py | 12 ++-- octoprint_nanny/workers/monitoring.py | 32 +++++----- octoprint_nanny/workers/mqtt.py | 85 +++++++++++++-------------- tests/test_manager.py | 12 ++-- 6 files changed, 109 insertions(+), 108 deletions(-) diff --git a/octoprint_nanny/manager.py b/octoprint_nanny/manager.py index 4bbf2db0..282bcf26 100644 --- a/octoprint_nanny/manager.py +++ b/octoprint_nanny/manager.py @@ -68,18 +68,19 @@ def __init__(self, plugin): self.mqtt_receive_queue = mqtt_receive_queue plugin_settings = PluginSettingsMemoize(plugin, mqtt_receive_queue) - self.plugin_settings = plugin_settings + self.plugin.settings = plugin_settings self.monitoring_manager = MonitoringManager( octo_ws_queue, pn_ws_queue, mqtt_send_queue, - plugin_settings, - self.plugin._event_bus, + plugin, ) self.mqtt_manager = MQTTManager( - mqtt_send_queue, mqtt_receive_queue, plugin_settings, plugin + mqtt_send_queue=mqtt_send_queue, + mqtt_receive_queue=mqtt_receive_queue, + plugin=plugin, ) # local callback/handler functions for events published via telemetry queue self._mqtt_send_queue_callbacks = { @@ -125,7 +126,7 @@ def _register_plugin_event_handlers(self): @beeline.traced("WorkerManager.on_settings_initialized") def on_settings_initialized(self): self._honeycomb_tracer.add_global_context( - self.plugin_settings.get_device_metadata() + self.plugin.settings.get_device_metadata() ) self._register_plugin_event_handlers() self.mqtt_manager.start() @@ -134,20 +135,20 @@ def on_settings_initialized(self): def apply_device_registration(self): self.mqtt_manager.stop() logger.info("Resetting WorkerManager device registration state") - self.plugin_settings.reset_device_settings_state() + self.plugin.settings.reset_device_settings_state() self.mqtt_manager.start() @beeline.traced("WorkerManager.apply_auth") def apply_auth(self): logger.info("Resetting WorkerManager user auth state") self.mqtt_manager.stop() - self.plugin_settings.reset_rest_client_state() + self.plugin.settings.reset_rest_client_state() self.mqtt_manager.start() @beeline.traced("WorkerManager.apply_monitoring_settings") def apply_monitoring_settings(self): - self.plugin_settings.reset_monitoring_settings() + self.plugin.settings.reset_monitoring_settings() logger.info( "Stopping any existing monitoring processes to apply new calibration" ) @@ -163,8 +164,8 @@ def shutdown(self): self.monitoring_manager.stop() asyncio.run_coroutine_threadsafe( - self.rest_client.update_octoprint_device( - self.device_id, monitoring_active=False + self.plugin.settings.rest_client.update_octoprint_device( + self.plugin.settings.device_id, monitoring_active=False ), self.loop, ).result() @@ -179,8 +180,10 @@ async def on_print_start(self, event_type, event_data, **kwargs): current_profile = ( self.plugin._printer_profile_manager.get_current_or_default() ) - printer_profile = await self.rest_client.update_or_create_printer_profile( - current_profile, self.device_id + printer_profile = ( + await self.plugin.settings.rest_client.update_or_create_printer_profile( + current_profile, self.plugin.settings.device_id + ) ) self.shared.printer_profile_id = printer_profile.id @@ -188,12 +191,17 @@ async def on_print_start(self, event_type, event_data, **kwargs): gcode_file_path = self.plugin._file_manager.path_on_disk( octoprint.filemanager.FileDestinations.LOCAL, event_data["path"] ) - gcode_file = await self.rest_client.update_or_create_gcode_file( - event_data, gcode_file_path, self.device_id + gcode_file = ( + await self.plugin.settings.rest_client.update_or_create_gcode_file( + event_data, gcode_file_path, self.plugin.settings.device_id + ) ) - print_job = await self.rest_client.create_print_job( - event_data, gcode_file.id, printer_profile.id, self.device_id + print_job = await self.plugin.settings.rest_client.create_print_job( + event_data, + gcode_file.id, + printer_profile.id, + self.plugin.settings.device_id, ) self.shared.print_job_id = print_job.id @@ -204,4 +212,4 @@ async def on_print_start(self, event_type, event_data, **kwargs): if self.plugin.get_setting("auto_start"): logger.info("Print Nanny monitoring is set to auto-start") - self.start_monitoring() + self.monitoring_manager.start() diff --git a/octoprint_nanny/plugins.py b/octoprint_nanny/plugins.py index ec68eb52..e5bbd422 100644 --- a/octoprint_nanny/plugins.py +++ b/octoprint_nanny/plugins.py @@ -107,7 +107,7 @@ def __init__(self, *args, **kwargs): self._log_path = None self._environment = {} - self._worker_manager = WorkerManager(plugin=self) + self.worker_manager = WorkerManager(plugin=self) self._honeycomb_tracer = HoneycombTracer(service_name="octoprint_plugin") def get_setting(self, key): @@ -205,7 +205,7 @@ async def _sync_printer_profiles(self, device_id): for profile_id, profile in printer_profiles.items(): self._logger.info("Syncing profile") created_profile = ( - await self._worker_manager.rest_client.update_or_create_printer_profile( + await self.worker_manager.rest_client.update_or_create_printer_profile( profile, device_id ) ) @@ -279,7 +279,7 @@ async def _register_device(self, device_name): context={"name": "update_or_create_octoprint_device"} ) try: - device = await self._worker_manager.rest_client.update_or_create_octoprint_device( + device = await self.worker_manager.plugin.settings.rest_client.update_or_create_octoprint_device( name=device_name, **device_info ) self._honeycomb_tracer.add_context(dict(device_upserted=device)) @@ -379,13 +379,13 @@ def register_device(self): ) future = asyncio.run_coroutine_threadsafe( - self._register_device(device_name), self._worker_manager.loop + self._register_device(device_name), self.worker_manager.loop ) result = future.result() if isinstance(result, Exception): raise result self._settings.save() - self._worker_manager.apply_device_registration() + self.worker_manager.apply_device_registration() return flask.jsonify(result) @beeline.traced(name="OctoPrintNannyPlugin.test_snapshot_url") @@ -394,7 +394,7 @@ def test_snapshot_url(self): snapshot_url = flask.request.json.get("snapshot_url") image = asyncio.run_coroutine_threadsafe( - self._test_snapshot_url(snapshot_url), self._worker_manager.loop + self._test_snapshot_url(snapshot_url), self.worker_manager.loop ).result() return flask.jsonify({"image": base64.b64encode(image)}) @@ -408,7 +408,7 @@ def test_auth_token(self): self._logger.info("Testing auth_token in event loop") response = asyncio.run_coroutine_threadsafe( - self._test_api_auth(auth_token, api_url), self._worker_manager.loop + self._test_api_auth(auth_token, api_url), self.worker_manager.loop ) response = response.result() @@ -452,7 +452,7 @@ def register_custom_events(self): @beeline.traced(name="OctoPrintNannyPlugin.on_after_startup") def on_shutdown(self): self._logger.info("Processing shutdown event") - self._worker_manager.shutdown() + self.worker_manager.shutdown() @beeline.traced(name="OctoPrintNannyPlugin.on_after_startup") def on_after_startup(self): @@ -471,7 +471,7 @@ def on_event(self, event_type, event_data): # shutdown event is handled in .on_shutdown if event_type == Events.SHUTDOWN: return - self._worker_manager.telemetry_queue.put_nowait( + self.worker_manager.mqtt_send_queue.put_nowait( {"event_type": event_type, "event_data": event_data} ) @@ -482,12 +482,12 @@ def on_settings_initialized(self): """ self._honeycomb_tracer.add_global_context(self.get_device_info()) self._log_path = self._settings.get_plugin_logfile_path() - self._worker_manager.on_settings_initialized() + self.worker_manager.on_settings_initialized() ## Progress plugin def on_print_progress(self, storage, path, progress): - self._worker_manager.telemetry_queue.put_nowait( + self.worker_manager.mqtt_send_queue.put_nowait( {"event_type": Events.PRINT_PROGRESS, "event_data": {"progress": progress}} ) @@ -495,7 +495,7 @@ def on_print_progress(self, storage, path, progress): @beeline.traced(name="OctoPrintNannyPlugin.on_environment_detected") def on_environment_detected(self, environment, *args, **kwargs): self._environment = environment - self._worker_manager.on_environment_detected(environment) + self.worker_manager.plugin.settings.on_environment_detected(environment) ## SettingsPlugin mixin def get_settings_defaults(self): @@ -541,7 +541,7 @@ def on_settings_save(self, data): if prev_mqtt_bridge_certificate_url != new_mqtt_bridge_certificate_url: asyncio.run_coroutine_threadsafe( - self._download_root_certificates(), self._worker_manager.loop + self._download_root_certificates(), self.worker_manager.loop ) if ( prev_monitoring_fpm != new_monitoring_fpm @@ -551,11 +551,11 @@ def on_settings_save(self, data): "Change in frames per minute or calibration detected, applying new settings" ) self._event_bus.fire(Events.PLUGIN_OCTOPRINT_NANNY_PREDICT_OFFLINE) - self._worker_manager.apply_monitoring_settings() + self.worker_manager.apply_monitoring_settings() if prev_auth_token != new_auth_token: self._logger.info("Change in auth detected, applying new settings") - self._worker_manager.apply_auth() + self.worker_manager.apply_auth() if ( prev_device_fingerprint != new_device_fingerprint @@ -566,7 +566,7 @@ def on_settings_save(self, data): self._logger.info( "Change in device identity detected (did you re-register?), applying new settings" ) - self._worker_manager.apply_device_registration() + self.worker_manager.apply_device_registration() ## Template plugin @@ -578,7 +578,7 @@ def get_template_vars(self): key: self._settings.get([key]) for key in self.get_settings_defaults().keys() }, - "active": self._worker_manager.monitoring_active, + "active": self.worker_manager.monitoring_active, } ## Wizard plugin mixin diff --git a/octoprint_nanny/predictor.py b/octoprint_nanny/predictor.py index b637ba61..c7f4b040 100644 --- a/octoprint_nanny/predictor.py +++ b/octoprint_nanny/predictor.py @@ -264,10 +264,10 @@ def __init__( calibration: dict, octoprint_ws_queue, pn_ws_queue, - telemetry_queue, + mqtt_send_queue, fpm, halt, - plugin_event_bus, + plugin, trace_context={}, ): """ @@ -278,7 +278,7 @@ def __init__( fpm - approximate frame per minute sample rate, depends on asyncio.sleep() halt - threading.Event() """ - self._plugin_event_bus = plugin_event_bus + self._plugin = plugin self._calibration = calibration self._fpm = fpm self._sleep_interval = 60 / int(fpm) @@ -286,7 +286,7 @@ def __init__( self._octoprint_ws_queue = octoprint_ws_queue self._pn_ws_queue = pn_ws_queue - self._telemetry_queue = telemetry_queue + self._mqtt_send_queue = mqtt_send_queue self._honeycomb_tracer = HoneycombTracer(service_name="octoprint_plugin") self._honeycomb_tracer.add_global_context(trace_context) @@ -404,12 +404,12 @@ async def _producer(self): pool, _get_predict_bytes, msg ) ws_msg, mqtt_msg = self._create_msgs(msg, viz_buffer, prediction) - self._plugin_event_bus.fire( + self._plugin._event_bus.fire( Events.PLUGIN_OCTOPRINT_NANNY_PREDICT_DONE, payload={"image": base64.b64encode(viz_buffer.getvalue())}, ) self._pn_ws_queue.put_nowait(ws_msg) - self._telemetry_queue.put_nowait(mqtt_msg) + self._mqtt_send_queue.put_nowait(mqtt_msg) self._honeycomb_tracer.finish_trace(trace) diff --git a/octoprint_nanny/workers/monitoring.py b/octoprint_nanny/workers/monitoring.py index f8c6d30f..033a7b14 100644 --- a/octoprint_nanny/workers/monitoring.py +++ b/octoprint_nanny/workers/monitoring.py @@ -19,16 +19,14 @@ def __init__( octo_ws_queue, pn_ws_queue, mqtt_send_queue, - plugin_settings, - plugin_event_bus, + plugin, ): self.halt = threading.Event() self.octo_ws_queue = octo_ws_queue self.pn_ws_queue = pn_ws_queue self.mqtt_send_queue = mqtt_send_queue - self.plugin_settings = plugin_settings - self.plugin_event_bus = plugin_event_bus + self.plugin = plugin @beeline.traced("MonitoringManager._drain") def _drain(self): @@ -42,23 +40,23 @@ def _drain(self): def _reset(self): self.halt = threading.Event() self._predict_worker = PredictWorker( - self.plugin_settings.snapshot_url, - self.plugin_settings.calibration, + self.plugin.settings.snapshot_url, + self.plugin.settings.calibration, self.octo_ws_queue, self.pn_ws_queue, self.mqtt_send_queue, - self.plugin_settings.monitoring_frames_per_minute, + self.plugin.settings.monitoring_frames_per_minute, self.halt, - self.plugin_event_bus, - trace_context=self.plugin_settings.get_device_metadata(), + self.plugin, + trace_context=self.plugin.settings.get_device_metadata(), ) self._websocket_worker = WebSocketWorker( - self.plugin_settings.ws_url, - self.plugin_settings.auth_token, + self.plugin.settings.ws_url, + self.plugin.settings.auth_token, self.pn_ws_queue, - self.plugin_settings.device_id, + self.plugin.settings.device_id, self.halt, - trace_context=self.plugin_settings.get_device_metadata(), + trace_context=self.plugin.settings.get_device_metadata(), ) self._workers = [self._predict_worker, self._websocket_worker] self._worker_threads = [] @@ -72,13 +70,13 @@ async def start(self): thread.daemon = True self._worker_threads.append(thread) thread.start() - await self.plugin_settings.rest_client.update_octoprint_device( - self.plugin_settings.device_id, active=True + await self.plugin.settings.rest_client.update_octoprint_device( + self.plugin.settings.device_id, active=True ) @beeline.traced("MonitoringManager.stop") async def stop(self): self._drain() - await self.plugin_settings.rest_client.update_octoprint_device( - self.plugin_settings.device_id, active=False + await self.plugin.settings.rest_client.update_octoprint_device( + self.plugin.settings.device_id, active=False ) diff --git a/octoprint_nanny/workers/mqtt.py b/octoprint_nanny/workers/mqtt.py index 1077aacc..53806ebf 100644 --- a/octoprint_nanny/workers/mqtt.py +++ b/octoprint_nanny/workers/mqtt.py @@ -3,12 +3,13 @@ import aioprocessing import asyncio import concurrent -import io import inspect +import io +import json import logging import os -import threading import queue +import threading import logging @@ -33,11 +34,9 @@ def __init__( self, mqtt_send_queue: aioprocessing.Queue, mqtt_receive_queue: aioprocessing.Queue, - plugin_settings: PluginSettingsMemoize, plugin, ): - self.plugin_settings = plugin_settings self.mqtt_send_queue = mqtt_send_queue self.mqtt_receive_queue = mqtt_receive_queue halt = threading.Event() @@ -45,12 +44,12 @@ def __init__( self.plugin = plugin self.publisher_worker = MQTTPublisherWorker( - self.halt, self.mqtt_send_queue, self.plugin_settings + self.halt, self.mqtt_send_queue, self.plugin ) self.subscriber_worker = MQTTSubscriberWorker( - self.halt, self.mqtt_receive_queue, self.plugin_settings, self.plugin + self.halt, self.mqtt_receive_queue, self.plugin ) - self.client_worker = MQTTClientWorker(self.halt, self.plugin_settings) + self.client_worker = MQTTClientWorker(self.halt, self.plugin) self._workers = [] self._worker_threads = [] @@ -63,10 +62,10 @@ def _drain(self): try: logger.info("Waiting for MQTTManager.mqtt_client network loop to finish") - while self.client_worker.plugin_settings.mqtt_client.client.is_connected(): - self.plugin_settings.plugin_settings.mqtt_client.client.disconnect() + while self.plugin.settings.mqtt_client.client.is_connected(): + self.plugin.settings.plugin.settings.mqtt_client.client.disconnect() logger.info("Stopping MQTTManager.mqtt_client network loop") - self.plugin_settings.mqtt_client.client.loop_stop() + self.plugin.settings.mqtt_client.client.loop_stop() except PluginSettingsRequired: pass @@ -78,14 +77,12 @@ def _drain(self): def _reset(self): self.halt = threading.Event() self.publisher_worker = MQTTPublisherWorker( - self.halt, self.mqtt_send_queue, self.plugin_settings + self.halt, self.mqtt_send_queue, self.plugin ) self.subscriber_worker = MQTTSubscriberWorker( - self.halt, self.mqtt_receive_queue, self.plugin_settings, self.plugin - ) - self.client_worker = MQTTClientWorker( - self.halt, self.plugin_settings.mqtt_client + self.halt, self.mqtt_receive_queue, self.plugin ) + self.client_worker = MQTTClientWorker(self.halt, self.plugin) self._workers = [ self.client_worker, @@ -119,15 +116,16 @@ class MQTTClientWorker: https://www.eclipse.org/paho/index.php?page=clients/python/docs/index.php#network-loop """ - def __init__(self, halt, plugin_settings): + def __init__(self, halt, plugin): - self.plugin_settings = plugin_settings + self.plugin = plugin self.halt = halt @beeline.traced("MQTTClientWorker.run") def run(self): + try: - return self.plugin_settings.mqtt_client.run(self.halt) + return self.plugin.settings.mqtt_client.run(self.halt) except PluginSettingsRequired: pass logger.warning("MQTTClientWorker will exit soon") @@ -151,11 +149,11 @@ class MQTTPublisherWorker: # do not warn when the following events are skipped on telemetry update MUTED_EVENTS = [Events.Z_CHANGE, "plugin_octoprint_nanny_predict_done"] - def __init__(self, halt, queue, plugin_settings): + def __init__(self, halt, queue, plugin): self.halt = halt self.queue = queue - self.plugin_settings = plugin_settings + self.plugin = plugin self._callbacks = {} self._honeycomb_tracer = HoneycombTracer(service_name="octoprint_plugin") @@ -187,27 +185,27 @@ async def _publish_octoprint_event_telemetry(self, event): logger.info(f"_publish_octoprint_event_telemetry {event}") event.update( dict( - user_id=self.plugin_settings.user_id, - device_id=self.plugin_settings.device_id, - device_cloudiot_name=self.plugin_settings.device_cloudiot_name, + user_id=self.plugin.settings.user_id, + device_id=self.plugin.settings.device_id, + device_cloudiot_name=self.plugin.settings.device_cloudiot_name, ) ) - event.update(self.plugin_settings.get_device_metadata()) + event.update(self.plugin.settings.get_device_metadata()) if event_type in self.PRINT_JOB_EVENTS: - event.update(self.plugin_settings.get_print_job_metadata()) - self.plugin_settings.mqtt_client.publish_octoprint_event(event) + event.update(self.plugin.settings.get_print_job_metadata()) + self.plugin.settings.mqtt_client.publish_octoprint_event(event) @beeline.traced("MQTTPublisherWorker._publish_bounding_box_telemetry") async def _publish_bounding_box_telemetry(self, event): event.update( dict( - user_id=self.plugin_settings.user_id, - device_id=self.plugin_settings.device_id, - device_cloudiot_name=self.plugin_settings.device_cloudiot_name, + user_id=self.plugin.settings.user_id, + device_id=self.plugin.settings.device_id, + device_cloudiot_name=self.plugin.settings.device_cloudiot_name, ) ) - self.plugin_settings.mqtt_client.publish_bounding_boxes(event) + self.plugin.settings.mqtt_client.publish_bounding_boxes(event) @beeline.traced("MQTTPublisherWorker._loop") async def _loop(self): @@ -237,7 +235,7 @@ async def _loop(self): await self._publish_bounding_box_telemetry(event) return - if self.plugin_settings.event_in_tracked_telemetry(event_type): + if self.plugin.settings.event_in_tracked_telemetry(event_type): await self._publish_octoprint_event_telemetry(event) else: if event_type not in self.MUTED_EVENTS: @@ -268,11 +266,10 @@ async def loop_forever(self): class MQTTSubscriberWorker: - def __init__(self, halt, queue, plugin_settings, plugin): + def __init__(self, halt, queue, plugin): self.halt = halt self.queue = queue - self.plugin_settings = plugin_settings self.plugin = plugin self._callbacks = {} self._honeycomb_tracer = HoneycombTracer(service_name="octoprint_plugin") @@ -309,10 +306,10 @@ async def loop_forever(self): @beeline.traced("MQTTSubscriberWorker._remote_control_snapshot") async def _remote_control_snapshot(self, command_id): async with aiohttp.ClientSession() as session: - res = await session.get(self.plugin_settings.snapshot_url) + res = await session.get(self.plugin.settings.snapshot_url) snapshot_io = io.BytesIO(await res.read()) - return await self.plugin_settings.rest_client.create_snapshot( + return await self.plugin.settings.rest_client.create_snapshot( image=snapshot_io, command=command_id ) @@ -328,8 +325,8 @@ async def _handle_remote_control_command(self, topic, message): await self._remote_control_snapshot(command_id) - metadata = self.plugin_settings.get_device_metadata() - await self.plugin_settings.rest_client.update_remote_control_command( + metadata = self.plugin.settings.get_device_metadata() + await self.plugin.settings.rest_client.update_remote_control_command( command_id, received=True, metadata=metadata ) @@ -346,17 +343,17 @@ async def _handle_remote_control_command(self, topic, message): else: handler_fn(event=message, event_type=event_type) - metadata = self.plugin_settings.get_device_metadata() + metadata = self.plugin.settings.get_device_metadata() # set success state - await self.plugin_settings.rest_client.update_remote_control_command( + await self.plugin.settings.rest_client.update_remote_control_command( command_id, success=True, metadata=metadata, ) except Exception as e: logger.error(f"Error calling handler_fn {handler_fn} \n {e}") - metadata = self.plugin_settings.get_device_metadata() - await self.plugin_settings.rest_client.update_remote_control_command( + metadata = self.plugin.settings.get_device_metadata() + await self.plugin.settings.rest_client.update_remote_control_command( command_id, success=False, metadata=metadata, @@ -382,10 +379,10 @@ async def _loop(self): if topic is None: logger.warning("Ignoring received message where topic=None") - elif topic == self.plugin_settings.mqtt_client.remote_control_command_topic: + elif topic == self.plugin.settings.mqtt_client.commands_topic: await self._handle_remote_control_command(**payload) - elif topic == self.plugin_settings.mqtt_client.config_topic: + elif topic == self.plugin.settings.mqtt_client.config_topic: await self._handle_config_update(**payload) else: @@ -428,6 +425,6 @@ async def _data_file(content, filename): os.path.join(self.plugin.get_plugin_data_folder(), "version.txt"), ) await _data_file( - metadata, + json.dumps(metadata), os.path.join(self.plugin.get_plugin_data_folder(), "metadata.json"), ) diff --git a/tests/test_manager.py b/tests/test_manager.py index 6e277789..27cdf39f 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -25,13 +25,13 @@ def test_default_settings_client_states(mocker): plugin.get_setting = get_default_setting manager = WorkerManager(plugin) - assert manager.plugin_settings.auth_token is None - assert manager.plugin_settings.device_id is None + assert manager.plugin.settings.auth_token is None + assert manager.plugin.settings.device_id is None with pytest.raises(PluginSettingsRequired): - dir(manager.plugin_settings.mqtt_client) + dir(manager.plugin.settings.mqtt_client) with pytest.raises(PluginSettingsRequired): - dir(manager.plugin_settings.rest_client) + dir(manager.plugin.settings.rest_client) @pytest.mark.asyncio @@ -131,9 +131,7 @@ async def test_mqtt_receive_queue_valid_octoprint_event(mocker): ) topic = "remote-control-topic" - type(mock_mqtt_client).remote_control_command_topic = PropertyMock( - return_value=topic - ) + type(mock_mqtt_client).commands_topic = PropertyMock(return_value=topic) mocker.patch( "octoprint_nanny.settings.PluginSettingsMemoize.get_device_metadata", return_value={}, From 879ff3b885ff6abccdcb5bef0aa0d5f8edca07c3 Mon Sep 17 00:00:00 2001 From: Leigh Johnson Date: Mon, 8 Feb 2021 06:14:37 +0000 Subject: [PATCH 14/16] add passing test_handle_config_update --- dev-requirements.txt | 4 +- octoprint_nanny/workers/mqtt.py | 1 + tests/workers/test_mqtt.py | 72 +++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 tests/workers/test_mqtt.py diff --git a/dev-requirements.txt b/dev-requirements.txt index 5bf756c6..280e7829 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -4,4 +4,6 @@ pytest-mock pytest-asyncio bump2version pytest-automock -pytest-cov \ No newline at end of file +pytest-cov +asyncmock +asynctest \ No newline at end of file diff --git a/octoprint_nanny/workers/mqtt.py b/octoprint_nanny/workers/mqtt.py index 53806ebf..f371d730 100644 --- a/octoprint_nanny/workers/mqtt.py +++ b/octoprint_nanny/workers/mqtt.py @@ -392,6 +392,7 @@ async def _loop(self): @beeline.traced("MQTTSubscriberWorker._handle_config_updat") async def _handle_config_update(self, topic, message): + device_config = print_nanny_client.ExperimentDeviceConfig(**message) labels = device_config.artifact.get("labels") diff --git a/tests/workers/test_mqtt.py b/tests/workers/test_mqtt.py new file mode 100644 index 00000000..f867d9d9 --- /dev/null +++ b/tests/workers/test_mqtt.py @@ -0,0 +1,72 @@ +import asyncio +import os +import pytest +from asynctest import CoroutineMock +from asynctest import MagicMock +from datetime import datetime +from octoprint_nanny.workers.mqtt import ( + MQTTClientWorker, + MQTTManager, + MQTTSubscriberWorker, + MQTTPublisherWorker, +) + + +@pytest.mark.asyncio +async def test_handle_config_update(mocker): + plugin = mocker.Mock() + halt = mocker.Mock() + queue = mocker.Mock() + + data_folder = "/path/to/data" + plugin.get_plugin_data_folder.return_value = data_folder + + subscriber_worker = MQTTSubscriberWorker(halt=halt, queue=queue, plugin=plugin) + + mock_res = MagicMock() + mock_res.__aenter__.return_value.get.return_value.__aenter__.return_value.text.return_value = ( + asyncio.Future() + ) + mock_res.__aenter__.return_value.get.return_value.__aenter__.return_value.text.return_value.set_result( + MagicMock() + ) + mocker.patch( + "octoprint_nanny.workers.mqtt.aiohttp.ClientSession", return_value=mock_res + ) + + writer_mock = MagicMock() + writer_mock.write.return_value = asyncio.Future() + writer_mock.write.return_value.set_result(MagicMock()) + open_mock = MagicMock() + open_mock.__aenter__.return_value = writer_mock + mock_aiofiles = mocker.patch( + "octoprint_nanny.workers.mqtt.aiofiles.open", return_value=open_mock + ) + + topic = "fake-topic" + + labels_url = "https://labels.com/labels.txt" + artifacts_url = "https://artifacts.com/artifacts.tflite" + version = "0.0.0" + metadata = {"python_version": "3.7.3"} + message = { + "id": 1, + "created_dt": datetime.now(), + "experiment": {}, + "artifact": { + "labels": labels_url, + "artifacts": artifacts_url, + "version": version, + "metadata": metadata, + }, + } + await subscriber_worker._handle_config_update(topic, message) + + mock_aiofiles.assert_has_calls( + [ + mocker.call(os.path.join(data_folder, "labels.txt"), "w+"), + mocker.call(os.path.join(data_folder, "model.tflite"), "w+"), + mocker.call(os.path.join(data_folder, "version.txt"), "w+"), + mocker.call(os.path.join(data_folder, "metadata.json"), "w+"), + ] + ) From 35c17402ce30f0656c7a1cbd733e9299cfcb7389 Mon Sep 17 00:00:00 2001 From: Leigh Johnson Date: Mon, 8 Feb 2021 07:40:08 +0000 Subject: [PATCH 15/16] use 15m keepalive in mqtt client --- octoprint_nanny/clients/mqtt.py | 27 ++++++++++++++++++--------- octoprint_nanny/plugins.py | 6 ++---- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/octoprint_nanny/clients/mqtt.py b/octoprint_nanny/clients/mqtt.py index 4a69874f..9984ad48 100644 --- a/octoprint_nanny/clients/mqtt.py +++ b/octoprint_nanny/clients/mqtt.py @@ -69,7 +69,8 @@ def __init__( project_id=GCP_PROJECT_ID, region=IOT_DEVICE_REGISTRY_REGION, registry_id=IOT_DEVICE_REGISTRY, - tls_version=ssl.PROTOCOL_TLSv1_2, + tls_version=ssl.PROTOCOL_TLS, + keepalive=900, # 15 minutes trace_context={}, message_callbacks=[], # see message_callback_add() https://www.eclipse.org/paho/index.php?page=clients/python/docs/index.php#subscribe-unsubscribe ): @@ -86,6 +87,7 @@ def __init__( self.ca_certs = ca_certs self.region = region self.registry_id = registry_id + self.keepalive = keepalive self.tls_version = tls_version self.region = region @@ -147,13 +149,18 @@ def _on_message(self, client, userdata, message): ) # callback to api to indicate command was received elif message.topic == self.config_topic: - parsed_message = json.loads(message.payload.decode("utf-8")) - logger.info( - f"Received config update on topic={message.topic} payload={parsed_message}" - ) - self.mqtt_receive_queue.put_nowait( - {"topic": self.config_topic, "message": parsed_message} - ) + try: + parsed_message = json.loads(message.payload.decode("utf-8")) + logger.info( + f"Received config update on topic={message.topic} payload={parsed_message}" + ) + self.mqtt_receive_queue.put_nowait( + {"topic": self.config_topic, "message": parsed_message} + ) + except json.decoder.JSONDecodeError as e: + logger.error( + f"Failed to decode message on topic={message.topic} payload={payload} message={payload.message}" + ) else: logger.info( f"MQTTClient._on_message called with userdata={userdata} topic={message.topic} payload={message}" @@ -241,7 +248,9 @@ def connect(self): username="unused", password=create_jwt(self.project_id, self.private_key_file, self.algorithm), ) - self.client.connect(self.mqtt_bridge_hostname, self.mqtt_bridge_port) + self.client.connect( + self.mqtt_bridge_hostname, self.mqtt_bridge_port, keepalive=self.keepalive + ) logger.info(f"MQTT client connected to {self.mqtt_bridge_hostname}") def publish(self, payload, topic=None, retain=False, qos=1): diff --git a/octoprint_nanny/plugins.py b/octoprint_nanny/plugins.py index e5bbd422..57747433 100644 --- a/octoprint_nanny/plugins.py +++ b/octoprint_nanny/plugins.py @@ -204,10 +204,8 @@ async def _sync_printer_profiles(self, device_id): id_map = {"octoprint": {}, "octoprint_nanny": {}} for profile_id, profile in printer_profiles.items(): self._logger.info("Syncing profile") - created_profile = ( - await self.worker_manager.rest_client.update_or_create_printer_profile( - profile, device_id - ) + created_profile = await self.worker_manager.plugin.settings.rest_client.update_or_create_printer_profile( + profile, device_id ) id_map["octoprint"][profile_id] = created_profile.id id_map["octoprint_nanny"][created_profile.id] = profile_id From 28488a8576c9d878897e8444a6719a379968e310 Mon Sep 17 00:00:00 2001 From: Leigh Johnson Date: Mon, 8 Feb 2021 07:45:29 +0000 Subject: [PATCH 16/16] UAT device registration --- octoprint_nanny/clients/mqtt.py | 7 ++++--- octoprint_nanny/workers/monitoring.py | 2 +- octoprint_nanny/workers/mqtt.py | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/octoprint_nanny/clients/mqtt.py b/octoprint_nanny/clients/mqtt.py index 9984ad48..ec7d5b66 100644 --- a/octoprint_nanny/clients/mqtt.py +++ b/octoprint_nanny/clients/mqtt.py @@ -150,16 +150,17 @@ def _on_message(self, client, userdata, message): # callback to api to indicate command was received elif message.topic == self.config_topic: try: - parsed_message = json.loads(message.payload.decode("utf-8")) + parsed_message = message.payload.decode("utf-8") + parsed_message = json.loads(parsed_message) logger.info( f"Received config update on topic={message.topic} payload={parsed_message}" ) self.mqtt_receive_queue.put_nowait( {"topic": self.config_topic, "message": parsed_message} ) - except json.decoder.JSONDecodeError as e: + except json.decoder.JSONDecodeError: logger.error( - f"Failed to decode message on topic={message.topic} payload={payload} message={payload.message}" + f"Failed to decode message on topic={message.topic} payload={message.payload} message={parsed_message}" ) else: logger.info( diff --git a/octoprint_nanny/workers/monitoring.py b/octoprint_nanny/workers/monitoring.py index 033a7b14..3e8f4238 100644 --- a/octoprint_nanny/workers/monitoring.py +++ b/octoprint_nanny/workers/monitoring.py @@ -34,7 +34,7 @@ def _drain(self): for worker in self._worker_threads: logger.info(f"Waiting for worker={worker} thread to drain") - worker.join() + worker.join(10) @beeline.traced("MonitoringManager._reset") def _reset(self): diff --git a/octoprint_nanny/workers/mqtt.py b/octoprint_nanny/workers/mqtt.py index f371d730..15b016d2 100644 --- a/octoprint_nanny/workers/mqtt.py +++ b/octoprint_nanny/workers/mqtt.py @@ -71,7 +71,7 @@ def _drain(self): for worker in self._worker_threads: logger.info(f"Waiting for worker={worker} thread to drain") - worker.join() + worker.join(10) @beeline.traced("MQTTManager._reset") def _reset(self):