Merge pull request #54 from WaverleySoftware/GH-47-add-missing-ros-ev…

…ents

By @sskorol:
- Added ROS events for tracking sensor states and poses.
- Added 1080P resolution for RGB camera stream.
- Added dynamic camera type switching.
- Fixed unselected default simulation option.
- Fixed most unreleased system hotkeys issue (virtual keyboard bug).
- Fixed fullscreen padding issue, which caused a broken video player layout on a specific Chrome profile.
- Fixed video overlay z-index.
- Added BE and FE settings API.
- Added automatic playback and camera type switching if the data stream is not yet started.

By @alex-waverley:
- Fixed fullscreen positioning issues.
- Updated icons based on recent designs.
- Updated react-simple-keyboard version, which addressed issues with keys highlighting and releasing.
- Set physicalKeyboardHighlightPreventDefault prop in the Keyboard component depending on the current tab.

backend/mini_pupper_webrtc/dto/robot_type.py

@@ -0,0 +1,7 @@
+from pydantic import BaseModel
+from ..model.robot_type import RobotType
+
+
+class RobotTypeDTO(BaseModel):
+    id: int
+    label: RobotType

backend/mini_pupper_webrtc/main.py

@@ -1,21 +1,24 @@
 import asyncio
 
-from typing import Set
+from typing import Dict, List
 
 from fastapi import FastAPI, Request
 from fastapi.middleware.cors import CORSMiddleware
 
-from mini_pupper_webrtc.dto.offer_payload import OfferPayload
+from .dto.offer_payload import OfferPayload
+from .dto.webrtc_response import WebRTCResponse
+from .dto.robot_type import RobotTypeDTO
 
 from .model.ros_bridge import RosBridge
 from .model.camera_type import CameraType
-
-from mini_pupper_webrtc.model.peer_connection import PeerConnection
-from mini_pupper_webrtc.model.uvicorn_logger import logger
-from mini_pupper_webrtc.dto.webrtc_response import WebRTCResponse
+from .model.peer_connection import PeerConnection
+from .model.uvicorn_logger import logger
+from .model.robot_type import RobotType
+from .model.robot_settings import RobotSettings
+from .model.widget import Widget
 
 ros_bridge: RosBridge = RosBridge()
-peer_connections: Set[PeerConnection] = set()
+peer_connections: Dict[str, PeerConnection] = {}
 app = FastAPI()
 app.add_middleware(
     CORSMiddleware,
@@ -26,15 +29,50 @@
 )
 
 
+# ToDo: add DB support and replace hardcoded values
+@app.get('/robot_settings')
+def get_robots() -> List[RobotSettings]:
+    return [
+        RobotSettings(id=1, name='MiniPupper', type=1),
+        RobotSettings(id=2, name='Husky', type=2, speed_step=20)
+    ]
+
+
+@app.get('/robot_types')
+def get_robot_types() -> List[RobotTypeDTO]:
+    return [RobotTypeDTO(id=id + 1, label=robot_type.value) for id, robot_type in enumerate(RobotType)]
+
+
+@app.get('/widgets')
+def get_widgets() -> List[Widget]:
+    return [
+        Widget(id=1, label='Screen', name='screen'),
+        Widget(id=2, label='Battery', name='battery'),
+        Widget(id=3, label='Robot\'s Speed', name='speed'),
+        Widget(id=4, label='Additional actions', name='actions')
+    ]
+
+
 @app.post('/offer', description='Establish WebRTC connection', response_model=WebRTCResponse)
 async def offer(offer_payload: OfferPayload, request: Request):
     if offer_payload.options.camera_type == CameraType.SIMULATOR:
         ros_bridge.connect()
 
-    peer_connection = PeerConnection(ros_bridge, offer_payload, on_close=on_connection_close)
-    peer_connections.add(peer_connection)
+    host = request.client.host
+    peer_connection = peer_connections.get(host, None)
+
+    if peer_connection:
+        await peer_connection.cleanup()
+
+    peer_connection = PeerConnection(
+        ros_bridge,
+        offer_payload,
+        on_close=lambda: on_connection_close(host)
+    )
+    peer_connections[host] = peer_connection
+
     await peer_connection.offer()
-    logger.info(f"Created offer for {request.client.host}")
+    logger.info(f"Created offer for {host}")
     return await peer_connection.answer()
 
 
@@ -45,11 +83,10 @@ def startup_event():
 
 @app.on_event("shutdown")
 async def on_shutdown():
-    coroutines = [pc.close() for pc in peer_connections]
+    coroutines = {pc.cleanup() for _, pc in peer_connections.items()}
     await asyncio.gather(*coroutines)
-    peer_connections.clear()
     ros_bridge.shutdown()
 
 
-def on_connection_close(connection: PeerConnection):
-    peer_connections.discard(connection)
+def on_connection_close(host):
+    del peer_connections[host]

backend/mini_pupper_webrtc/model/camera_resolution.py

@@ -6,3 +6,4 @@ class CameraResolution(str, Enum):
     THE_480_P = 'THE_480_P'
     THE_720_P = 'THE_720_P'
     THE_800_P = 'THE_800_P'
+    THE_1080_P = 'THE_1080_P'

backend/mini_pupper_webrtc/model/peer_connection.py

@@ -88,7 +88,7 @@ async def cleanup(self):
         if self.track:
             self.track.stop()
         await self.close()
-        self.on_close(self)
+        self.on_close()
 
     @property
     def sdp(self):

backend/mini_pupper_webrtc/model/robot_settings.py

@@ -0,0 +1,10 @@
+from pydantic import BaseModel
+
+
+class RobotSettings(BaseModel):
+    id: int
+    name: str
+    type: int
+    speed_step: int = 1
+    speed_min: int = 0
+    speed_max: int = 100

backend/mini_pupper_webrtc/model/robot_type.py

@@ -0,0 +1,6 @@
+from enum import Enum
+
+
+class RobotType(str, Enum):
+    LEGGED = 'Legged'
+    WHEELED = 'Wheeled'

backend/mini_pupper_webrtc/model/transformers.py

@@ -3,6 +3,7 @@
 import depthai as dai
 import blobconverter
 import numpy as np
+import time
 
 from aiortc import VideoStreamTrack
 from av import VideoFrame
@@ -14,6 +15,18 @@
 from .uvicorn_logger import logger
 
 
+class FPSHandler:
+    def __init__(self):
+        self.timestamp = time.time() + 1
+        self.start = time.time()
+        self.frame_cnt = 0
+    def next_iter(self):
+        self.timestamp = time.time()
+        self.frame_cnt += 1
+    def fps(self):
+        return self.frame_cnt / (self.timestamp - self.start)
+
+
 class VideoTransformTrack(VideoStreamTrack):
 
     def __init__(self, options: Options):
@@ -142,10 +155,12 @@ def __init__(self, options: Options):
         # Properties
         self.camRgb.setPreviewSize(self.options.cam_width, self.options.cam_height)
         self.camRgb.setInterleaved(False)
+        self.camRgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)
+        self.camRgb.setFps(40)
         self.camRgb.setColorOrder(dai.ColorCameraProperties.ColorOrder.RGB)
 
         # Linking
-        self.camRgb.preview.link(self.xoutRgb.input)
+        self.camRgb.video.link(self.xoutRgb.input)
         self.nn = None
 
         if options.nn_model != "":
@@ -159,7 +174,7 @@ def __init__(self, options: Options):
             self.camRgb.preview.link(self.nn.input)
             self.nn.out.link(self.nnOut.input)
         self.device = dai.Device(self.pipeline)
-        self.qRgb = self.device.getOutputQueue(name="rgb", maxSize=1, blocking=False)
+        self.qRgb = self.device.getOutputQueue(name="rgb", maxSize=4, blocking=False)
 
         if self.nn is not None:
             self.qDet = self.device.getOutputQueue(name="nn", maxSize=4, blocking=False)
@@ -259,6 +274,10 @@ def create_pipeline(self):
             self.monoLeft.setResolution(dai.MonoCameraProperties.SensorResolution.THE_800_P)
             self.monoRight.setResolution(dai.MonoCameraProperties.SensorResolution.THE_800_P)
             self.frame = np.zeros((800, 1280, 3), np.uint8)
+        elif mono_camera_resolution == CameraResolution.THE_1080_P:
+            self.monoLeft.setResolution(dai.MonoCameraProperties.SensorResolution.THE_1080_P)
+            self.monoRight.setResolution(dai.MonoCameraProperties.SensorResolution.THE_1080_P)
+            self.frame = np.zeros((1080, 1920, 3), np.uint8)
         self.monoLeft.setBoardSocket(dai.CameraBoardSocket.LEFT)
         self.monoRight.setBoardSocket(dai.CameraBoardSocket.RIGHT)
 

backend/mini_pupper_webrtc/model/widget.py

@@ -0,0 +1,8 @@
+from pydantic import BaseModel
+
+
+class Widget(BaseModel):
+    id: int
+    label: str
+    name: str
+    selected: bool = True

frontend/.env.example

@@ -2,3 +2,9 @@ REACT_APP_ROSBRIDGE_SERVER_IP=
 REACT_APP_ROSBRIDGE_SERVER_PORT=
 REACT_APP_RECONNECTION_TIMER=
 REACT_APP_BE_URL=
+REACT_APP_BATTERY_TOPIC=
+REACT_APP_MEMORY_TOPIC=
+REACT_APP_CPU_TOPIC=
+REACT_APP_POSE_CHANGE_TOPIC=
+REACT_APP_POSE_STATE_TOPIC=
+REACT_APP_IS_SIMULATING=