From 6c3765fb67e0bdded776783fb3a177e4447fd140 Mon Sep 17 00:00:00 2001 From: ATATC Date: Tue, 2 Jul 2024 19:01:39 +0800 Subject: [PATCH 1/6] Supported invalid camera stream as `None`. (#250) --- leads_vec/devices_jarvis.py | 2 +- leads_video/camera.py | 7 ++++--- leads_video/utils.py | 21 +++++++++++++-------- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/leads_vec/devices_jarvis.py b/leads_vec/devices_jarvis.py index 0597d58f..5294fdf2 100644 --- a/leads_vec/devices_jarvis.py +++ b/leads_vec/devices_jarvis.py @@ -39,7 +39,7 @@ def initialize(self, *parent_tags: str) -> None: super().initialize(*parent_tags) @override - def read(self) -> _ndarray: + def read(self) -> _ndarray | None: return super().read() diff --git a/leads_video/camera.py b/leads_video/camera.py index 569ff353..fd9543b5 100644 --- a/leads_video/camera.py +++ b/leads_video/camera.py @@ -41,9 +41,10 @@ def transform(self, x: _ndarray) -> _ndarray: return _array(_fromarray(x).resize(self._resolution)) @_override - def read(self) -> _ndarray: - _, frame = self._video_capture.read() - return _cvtColor(self.transform(frame) if self._resolution else frame, _COLOR_BGR2RGB).transpose(2, 0, 1) + def read(self) -> _ndarray | None: + ret, frame = self._video_capture.read() + return _cvtColor(self.transform(frame) if self._resolution else frame, _COLOR_BGR2RGB + ).transpose(2, 0, 1) if ret else None @_override def close(self) -> None: diff --git a/leads_video/utils.py b/leads_video/utils.py index ce559f71..c0029e61 100644 --- a/leads_video/utils.py +++ b/leads_video/utils.py @@ -2,20 +2,25 @@ from io import BytesIO as _BytesIO from typing import Literal as _Literal -from PIL.Image import Image as _Image, fromarray as _fromarray, open as _open +from PIL.Image import Image as _Image, fromarray as _fromarray, open as _open, \ + UnidentifiedImageError as _UnidentifiedImageError from numpy import ndarray as _ndarray, array as _array -def base64_encode(x: _ndarray, mode: _Literal["L", "RGB"] | None = None) -> str: - img = _fromarray(x.transpose(1, 2, 0), mode) +def base64_encode(x: _ndarray | None, mode: _Literal["L", "RGB"] | None = None) -> str: + if x is None: + return "" buffer = _BytesIO() - img.save(buffer, "PNG") + _fromarray(x.transpose(1, 2, 0), mode).save(buffer, "PNG") return _b64encode(buffer.getvalue()).decode() -def base64_decode_image(x_base64: str) -> _Image: - return _open(_BytesIO(_b64decode(x_base64))) +def base64_decode_image(x_base64: str) -> _Image | None: + try: + return _open(_BytesIO(_b64decode(x_base64))) if x_base64 else None + except (ValueError, TypeError, _UnidentifiedImageError): + return None -def base64_decode(x_base64: str) -> _ndarray: - return _array(base64_decode_image(x_base64)).transpose(2, 0, 1) +def base64_decode(x_base64: str) -> _ndarray | None: + return _array(image).transpose(2, 0, 1) if (image := base64_decode_image(x_base64)) else None From 4b8242bc002b68e3300286bac590c9c675c5b4ea Mon Sep 17 00:00:00 2001 From: ATATC Date: Tue, 2 Jul 2024 20:48:04 +0800 Subject: [PATCH 2/6] Added `encode_image()`. (#250) --- leads_video/utils.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/leads_video/utils.py b/leads_video/utils.py index c0029e61..015329e0 100644 --- a/leads_video/utils.py +++ b/leads_video/utils.py @@ -7,11 +7,14 @@ from numpy import ndarray as _ndarray, array as _array +def encode_image(x: _ndarray | None, mode: _Literal["L", "RGB"] | None = None) -> _Image | None: + return None if x is None else _fromarray(x.transpose(1, 2, 0), mode) + + def base64_encode(x: _ndarray | None, mode: _Literal["L", "RGB"] | None = None) -> str: - if x is None: + if not (img := encode_image(x, mode)): return "" - buffer = _BytesIO() - _fromarray(x.transpose(1, 2, 0), mode).save(buffer, "PNG") + img.save(buffer := _BytesIO(), "PNG") return _b64encode(buffer.getvalue()).decode() From 1f1fc38e21f13ce4dab69b1c33e8360ce8cd8b49 Mon Sep 17 00:00:00 2001 From: ATATC Date: Tue, 2 Jul 2024 21:09:47 +0800 Subject: [PATCH 3/6] Increased the minimum interframe interval to 1ms. (#250) --- leads_gui/performance_checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/leads_gui/performance_checker.py b/leads_gui/performance_checker.py index 56b49d9b..451426f6 100644 --- a/leads_gui/performance_checker.py +++ b/leads_gui/performance_checker.py @@ -27,7 +27,7 @@ def record_frame(self, interval: float) -> None: self._net_delay_seq.append(delay - interval) mark = len(self._net_delay_seq) self._predicted_offset = max(min(_poly1d(_polyfit(range(mark), self._net_delay_seq, 5))(mark + 1) - if mark > self._refresh_rate else 0, self._interval), 0) + if mark > self._refresh_rate else 0, self._interval - .001), 0) self._last_frame = t def next_interval(self) -> float: From 4c6b9509a3b2e4cf4e1ceae2e16fe62032ce2335 Mon Sep 17 00:00:00 2001 From: ATATC Date: Tue, 2 Jul 2024 21:32:49 +0800 Subject: [PATCH 4/6] Increased the minimum interframe interval to 1ms. (#250) --- leads_gui/prototype.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/leads_gui/prototype.py b/leads_gui/prototype.py index 28ce863d..427ba3b3 100644 --- a/leads_gui/prototype.py +++ b/leads_gui/prototype.py @@ -307,7 +307,7 @@ def wrapper() -> None: self._root.after(int((ni := self._performance_checker.next_interval()) * 1000), wrapper) self._last_interval = ni - self._root.after(0, wrapper) + self._root.after(1, wrapper) self._root.mainloop() def kill(self) -> None: From f91f4c3f72ee538cea3c575774285faa26afeb9c Mon Sep 17 00:00:00 2001 From: ATATC Date: Tue, 2 Jul 2024 22:32:49 +0800 Subject: [PATCH 5/6] Using JPEG to increase the efficiency. (#250) --- leads_video/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/leads_video/utils.py b/leads_video/utils.py index 015329e0..b52ffb6e 100644 --- a/leads_video/utils.py +++ b/leads_video/utils.py @@ -14,7 +14,7 @@ def encode_image(x: _ndarray | None, mode: _Literal["L", "RGB"] | None = None) - def base64_encode(x: _ndarray | None, mode: _Literal["L", "RGB"] | None = None) -> str: if not (img := encode_image(x, mode)): return "" - img.save(buffer := _BytesIO(), "PNG") + img.save(buffer := _BytesIO(), "JPEG") return _b64encode(buffer.getvalue()).decode() From b7b2460c516f78480d053178b4fd1f4982edf17e Mon Sep 17 00:00:00 2001 From: ATATC Date: Tue, 2 Jul 2024 23:18:32 +0800 Subject: [PATCH 6/6] Added a new shadow device to encode images into Base64 in a separate thread. (#250) --- leads_vec/devices.py | 9 ++++----- leads_vec/devices_jarvis.py | 10 ++-------- leads_video/camera.py | 23 ++++++++++++++++++++--- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/leads_vec/devices.py b/leads_vec/devices.py index 082c62b4..00b4e51d 100644 --- a/leads_vec/devices.py +++ b/leads_vec/devices.py @@ -7,7 +7,6 @@ from leads_arduino import ArduinoMicro, WheelSpeedSensor, VoltageSensor from leads_gui import Config from leads_raspberry_pi import NMEAGPSReceiver, LEDGroup, LED, LEDGroupCommand, LEDCommand, Entire -from leads_video import base64_encode config: Config = require_config() GPS_ONLY: int = config.get("gps_only", False) @@ -38,13 +37,13 @@ def read(self) -> DataContainer: wsc = {"speed": gps[0]} if GPS_ONLY else self.device("wsc").read() visual = {} if has_device(FRONT_VIEW_CAMERA): - visual["front_view_base64"] = base64_encode(get_device(FRONT_VIEW_CAMERA).read()) + visual["front_view_base64"] = get_device(FRONT_VIEW_CAMERA).read() if has_device(LEFT_VIEW_CAMERA): - visual["left_view_base64"] = base64_encode(get_device(LEFT_VIEW_CAMERA).read()) + visual["left_view_base64"] = get_device(LEFT_VIEW_CAMERA).read() if has_device(RIGHT_VIEW_CAMERA): - visual["right_view_base64"] = base64_encode(get_device(RIGHT_VIEW_CAMERA).read()) + visual["right_view_base64"] = get_device(RIGHT_VIEW_CAMERA).read() if has_device(REAR_VIEW_CAMERA): - visual["rear_view_base64"] = base64_encode(get_device(REAR_VIEW_CAMERA).read()) + visual["rear_view_base64"] = get_device(REAR_VIEW_CAMERA).read() return DataContainer(**wsc, **general) if len(visual) < 1 else VisualDataContainer(**visual, **wsc, **general) diff --git a/leads_vec/devices_jarvis.py b/leads_vec/devices_jarvis.py index 5294fdf2..c0a9b1e3 100644 --- a/leads_vec/devices_jarvis.py +++ b/leads_vec/devices_jarvis.py @@ -1,11 +1,9 @@ from typing import override -from numpy import ndarray as _ndarray - from leads import device, MAIN_CONTROLLER, mark_device, FRONT_VIEW_CAMERA, LEFT_VIEW_CAMERA, RIGHT_VIEW_CAMERA, \ REAR_VIEW_CAMERA, require_config from leads_gui import Config -from leads_video import Camera +from leads_video import Base64Camera import_error: ImportError | None = None try: @@ -32,16 +30,12 @@ @device(CAMERA_TAGS, MAIN_CONTROLLER, CAMERA_ARGS) -class Cameras(Camera): +class Cameras(Base64Camera): @override def initialize(self, *parent_tags: str) -> None: mark_device(self, "Jarvis") super().initialize(*parent_tags) - @override - def read(self) -> _ndarray | None: - return super().read() - if import_error: raise import_error diff --git a/leads_video/camera.py b/leads_video/camera.py index fd9543b5..c12e3a36 100644 --- a/leads_video/camera.py +++ b/leads_video/camera.py @@ -4,7 +4,8 @@ from cv2 import VideoCapture as _VideoCapture, cvtColor as _cvtColor, COLOR_BGR2RGB as _COLOR_BGR2RGB from numpy import ndarray as _ndarray, pad as _pad, array as _array -from leads import Device as _Device +from leads import Device as _Device, ShadowDevice as _ShadowDevice +from leads_video.utils import base64_encode class Camera(_Device): @@ -43,9 +44,25 @@ def transform(self, x: _ndarray) -> _ndarray: @_override def read(self) -> _ndarray | None: ret, frame = self._video_capture.read() - return _cvtColor(self.transform(frame) if self._resolution else frame, _COLOR_BGR2RGB - ).transpose(2, 0, 1) if ret else None + return _cvtColor(self.transform(frame) if self._resolution else frame, _COLOR_BGR2RGB).transpose( + 2, 0, 1) if ret else None @_override def close(self) -> None: self._video_capture.release() + + +class Base64Camera(Camera, _ShadowDevice): + def __init__(self, port: int, resolution: tuple[int, int] | None = None) -> None: + Camera.__init__(self, port, resolution) + _ShadowDevice.__init__(self, port) + self._base64: str = "" + + @_override + def loop(self) -> None: + if self._video_capture: + self._base64 = base64_encode(super().read()) + + @_override + def read(self) -> str: + return self._base64