GH-47: 1. added ros events for tracking sensor states and poses; 2. a…

…dded 1080P resolution for rgb camera stream; 3. minor UI improvements

backend/mini_pupper_webrtc/model/camera_resolution.py

@@ -6,3 +6,4 @@ class CameraResolution(str, Enum):
     THE_480_P = 'THE_480_P'
     THE_720_P = 'THE_720_P'
     THE_800_P = 'THE_800_P'
+    THE_1080_P = 'THE_1080_P'

backend/mini_pupper_webrtc/model/transformers.py

@@ -3,6 +3,7 @@
 import depthai as dai
 import blobconverter
 import numpy as np
+import time
 
 from aiortc import VideoStreamTrack
 from av import VideoFrame
@@ -14,6 +15,18 @@
 from .uvicorn_logger import logger
 
 
+class FPSHandler:
+    def __init__(self):
+        self.timestamp = time.time() + 1
+        self.start = time.time()
+        self.frame_cnt = 0
+    def next_iter(self):
+        self.timestamp = time.time()
+        self.frame_cnt += 1
+    def fps(self):
+        return self.frame_cnt / (self.timestamp - self.start)
+
+
 class VideoTransformTrack(VideoStreamTrack):
 
     def __init__(self, options: Options):
@@ -142,10 +155,12 @@ def __init__(self, options: Options):
         # Properties
         self.camRgb.setPreviewSize(self.options.cam_width, self.options.cam_height)
         self.camRgb.setInterleaved(False)
+        self.camRgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)
+        self.camRgb.setFps(40)
         self.camRgb.setColorOrder(dai.ColorCameraProperties.ColorOrder.RGB)
 
         # Linking
-        self.camRgb.preview.link(self.xoutRgb.input)
+        self.camRgb.video.link(self.xoutRgb.input)
         self.nn = None
 
         if options.nn_model != "":
@@ -159,7 +174,7 @@ def __init__(self, options: Options):
             self.camRgb.preview.link(self.nn.input)
             self.nn.out.link(self.nnOut.input)
         self.device = dai.Device(self.pipeline)
-        self.qRgb = self.device.getOutputQueue(name="rgb", maxSize=1, blocking=False)
+        self.qRgb = self.device.getOutputQueue(name="rgb", maxSize=4, blocking=False)
 
         if self.nn is not None:
             self.qDet = self.device.getOutputQueue(name="nn", maxSize=4, blocking=False)
@@ -259,6 +274,10 @@ def create_pipeline(self):
             self.monoLeft.setResolution(dai.MonoCameraProperties.SensorResolution.THE_800_P)
             self.monoRight.setResolution(dai.MonoCameraProperties.SensorResolution.THE_800_P)
             self.frame = np.zeros((800, 1280, 3), np.uint8)
+        elif mono_camera_resolution == CameraResolution.THE_1080_P:
+            self.monoLeft.setResolution(dai.MonoCameraProperties.SensorResolution.THE_1080_P)
+            self.monoRight.setResolution(dai.MonoCameraProperties.SensorResolution.THE_1080_P)
+            self.frame = np.zeros((1080, 1920, 3), np.uint8)
         self.monoLeft.setBoardSocket(dai.CameraBoardSocket.LEFT)
         self.monoRight.setBoardSocket(dai.CameraBoardSocket.RIGHT)
 

frontend/.env.example

@@ -2,3 +2,9 @@ REACT_APP_ROSBRIDGE_SERVER_IP=
 REACT_APP_ROSBRIDGE_SERVER_PORT=
 REACT_APP_RECONNECTION_TIMER=
 REACT_APP_BE_URL=
+REACT_APP_BATTERY_TOPIC=
+REACT_APP_MEMORY_TOPIC=
+REACT_APP_CPU_TOPIC=
+REACT_APP_POSE_CHANGE_TOPIC=
+REACT_APP_POSE_STATE_TOPIC=
+REACT_APP_IS_SIMULATING=

frontend/src/components/additionalActions/index.js

@@ -5,7 +5,6 @@ import { observer } from 'mobx-react'
 import Widget, { WidgetTitle } from '../widget'
 import { ReactComponent as RobotStandIcon } from './robot_stand.svg'
 import { ReactComponent as RobotSitIcon } from './robot_sit.svg'
-import { robotPosition } from '../../store/RosController'
 import { useStore } from '../../store'
 
 const RobotPositionButton = observer(({heading = '', position, currentPosition, children, handleClick}) => {
@@ -46,36 +45,49 @@ const RobotPositionButton = observer(({heading = '', position, currentPosition,
 
 const AdditionalActions = observer(() => {
   const theme = useTheme()
-  const { rosStore: { currentPosition, setCurrentPosition }} = useStore()
+  const { rosStore: { isStanding, changePose } } = useStore()
+
+  const RobotPose = {
+    SIT: 'sit',
+    STAND: 'stand'
+  }
+
+  const pose = () => {
+    return isStanding ? RobotPose.STAND : RobotPose.SIT
+  }
+
+  const colorOf = (expectedPose) => {
+    return pose() === expectedPose ? theme.palette.common.white : theme.palette.secondary.main
+  }
 
   return (
     <Widget widgetName='actions'>
       <WidgetTitle>Additional actions</WidgetTitle>
       <Grid container justifyContent='center'>
         <RobotPositionButton
           heading='Stand'
-          position={robotPosition.stand}
-          currentPosition={currentPosition}
-          handleClick={setCurrentPosition}
+          position={RobotPose.STAND}
+          currentPosition={pose()}
+          handleClick={() => changePose(RobotPose.STAND)}
         >
           <SvgIcon
             component={RobotStandIcon}
             inheritViewBox
             sx={{
-              fill: currentPosition === robotPosition.stand ? theme.palette.common.white : theme.palette.secondary.main
+              fill: colorOf(RobotPose.STAND)
             }} />
         </RobotPositionButton>
         <RobotPositionButton
           heading='Sit'
-          position={robotPosition.sit}
-          currentPosition={currentPosition}
-          handleClick={setCurrentPosition}
+          position={RobotPose.SIT}
+          currentPosition={pose()}
+          handleClick={() => changePose(RobotPose.SIT)}
         >
           <SvgIcon
             component={RobotSitIcon}
             inheritViewBox
             sx={{
-              fill: currentPosition === robotPosition.sit ? theme.palette.common.white : theme.palette.secondary.main
+              fill: colorOf(RobotPose.SIT)
             }}
           />
         </RobotPositionButton>

frontend/src/components/keyboard/index.js

@@ -8,7 +8,7 @@ import './style.css'
 import { display, layout, buttonTheme, buttonAttributes, useStyles, syntethicKeysMap } from './keyboardSettings'
 
 const Keyboard = observer(() => {
-    const {rosStore: { handleKeyboardShortcuts }} = useStore()
+    const {rosStore: { publishKey }} = useStore()
     const [layoutName, setLayoutName] = useState('default')
     const { classes } = useStyles();
 
@@ -17,7 +17,7 @@ const Keyboard = observer(() => {
             setLayoutName(layoutName === 'default' ? 'shift' : 'default')
         }
         const key = syntethicKeysMap[button] || button
-        handleKeyboardShortcuts(key)
+        publishKey(key)
     }
 
     const theme = useTheme()

frontend/src/components/keyboard/keyboardSettings.js

@@ -55,9 +55,13 @@ export const buttonTheme = [
     buttons: 'g h',
   },
   {
-    class: 'incline-buttons',
+    class: 'incline-side-buttons',
     buttons: 'a s',
   },
+  {
+    class: 'incline-straight-buttons',
+    buttons: 'd f',
+  },
   {
     class: 'math-buttons',
     buttons: '- =',
@@ -183,6 +187,16 @@ export const buttonAttributes = [
     value: 'incline-right',
     buttons: 's',
   },
+  {
+    attribute: 'data-img',
+    value: 'incline-front',
+    buttons: 'd',
+  },
+  {
+    attribute: 'data-img',
+    value: 'incline-back',
+    buttons: 'f',
+  },
   {
     attribute: 'data-img',
     value: 'underscore',
@@ -294,11 +308,16 @@ export const useStyles = makeStyles()((theme) => ({
           transform: 'translate(-50%, 0)',
         }
       },
-      '&.incline-buttons': {
+      '&.incline-straight-buttons': {
         '&:before': {
-          left: '50%',
-          bottom: '4px',
-          transform: 'translate(-50%, 0)',
+          bottom: '0',
+          right: '1px',
+        }
+      },
+      '&.incline-side-buttons': {
+        '&:before': {
+          right: '7px',
+          bottom: '2px',
         }
       },
       '&.brackets-buttons': {
@@ -498,6 +517,22 @@ export const useStyles = makeStyles()((theme) => ({
         },
         '&:hover:before': {
           content: keyboardIcon.getIcon('inclineRight', theme.palette.common.white),
+        }
+      },
+      '&[data-img="incline-front"]': {
+        '&:before': {
+          content: keyboardIcon.getIcon('inclineFront'),
+        },
+        '&:hover:before': {
+          content: keyboardIcon.getIcon('inclineFront', theme.palette.common.white),
+        },
+      },
+      '&[data-img="incline-back"]': {
+        '&:before': {
+          content: keyboardIcon.getIcon('inclineBack'),
+        },
+        '&:hover:before': {
+          content: keyboardIcon.getIcon('inclineBack', theme.palette.common.white),
         },
       },
       '&[data-img="underscore"]': {

frontend/src/components/robotHardware/index.js

@@ -2,6 +2,7 @@
 import { Divider, Grid, SvgIcon, Typography } from '@mui/material'
 import { useTheme } from '@mui/material/styles'
 import { observer } from 'mobx-react'
+import { useStore } from '../../store'
 import Widget, { WidgetTitle } from '../widget'
 import { ReactComponent as LightningIcon } from './lightning.svg'
 import {
@@ -31,28 +32,30 @@ const BatteryLevelDivider = observer(({left = '0'}) => {
 
 const Battery = observer(() => {
   const theme = useTheme()
-  // TODO: define logic for percentage in a store (data, requests to DB, etc.)
-  const percentage = 80
-
-  let color = theme.palette.success.main
-  let capacity = 'High'
-
-  if (percentage < 20) {
-    color = theme.palette.error.main
-    capacity = 'Low'
-  } else if (percentage < 40) {
-    color = theme.palette.warning.main
-    capacity = 'Middle'
-  } else if (percentage < 80) {
-    color = theme.palette.info.main
-    capacity = 'Average'
+  const {rosStore: { batteryState }} = useStore()
+  const Level = {
+    HIGH: 'High',
+    AVG: 'Average',
+    LOW: 'Low'
+  }
+
+  const paletteColorOf = (level) => {
+    return (level === Level.LOW ? theme.palette.error : level === Level.AVG ? theme.palette.warning : theme.palette.success).main
+  }
+
+  const getCapacity = () => {
+    return batteryState < 20 ? Level.LOW : batteryState < 60 ? Level.AVG : Level.HIGH
+  }
+
+  const getColor = () => {
+    return paletteColorOf(getCapacity())
   }
 
   return (
     <Grid>
-      <Grid container justifyContent='space-between' >
-        <WidgetTitle styles={{marginBottom: '14px', fontWeight: theme.typography.fontWeightMedium}}>{capacity}</WidgetTitle>
-        <WidgetTitle styles={{marginBottom: '14px', fontWeight: theme.typography.fontWeightMedium}}>{percentage}%</WidgetTitle>
+      <Grid container justifyContent='space-between'>
+        <WidgetTitle styles={{marginBottom: '14px', fontWeight: theme.typography.fontWeightMedium}}>{getCapacity()}</WidgetTitle>
+        <WidgetTitle styles={{marginBottom: '14px', fontWeight: theme.typography.fontWeightMedium}}>{batteryState}%</WidgetTitle>
       </Grid>
       <Grid
         sx={{
@@ -73,24 +76,24 @@ const Battery = observer(() => {
           top: 0,
           left: 0,
           height: '100%',
-          width: `${percentage}%`,
-          backgroundColor: color,
+          width: `${batteryState}%`,
+          backgroundColor: getColor(),
           borderRadius: '6px',
         }}/>
       </Grid>
     </Grid>
   )
 })
 
-const CircleProgress = observer(({percentage, title}) => {
+const CircleProgress = observer(({percentage, title, color=''}) => {
   const theme = useTheme()
 
   return (
     <CircularProgressbarWithChildren
       value={percentage}
       strokeWidth={10}
       styles={buildStyles({
-        pathColor: theme.palette.info.main,
+        pathColor: color || theme.palette.info.main,
         trailColor: theme.palette.background.circleProgressTrail,
       })}
     >
@@ -116,9 +119,26 @@ const CircleProgress = observer(({percentage, title}) => {
 })
 
 const RobotHardware = observer(() => {
-  // TODO: define logic for memoryUsagePercentage and cpuUsagePercentage in a store (data, requests to DB, etc.)
-  const memoryUsagePercentage = 35
-  const cpuUsagePercentage = 60
+  const theme = useTheme()
+  const {rosStore: { cpuState, memoryState }} = useStore()
+
+  const getCpuColor = () => {
+    return getColor(cpuState)
+  }
+
+  const getMemoryColor = () => {
+    return getColor(memoryState)
+  }
+
+  const getColor = (state) => {
+    if (state >= 0 && state <= 40) {
+      return theme.palette.success.main
+    } else if (state > 40 && state <= 80) {
+      return theme.palette.warning.main
+    } else if (state > 80) {
+      return theme.palette.error.main
+    }
+  }
 
   return (
     <Widget widgetName='battery' styles={{height: '100%'}}>
@@ -136,11 +156,11 @@ const RobotHardware = observer(() => {
       <Grid container alignItems='center' justifyContent='center'>
         <Grid item sx={{width: '100px', marginRight: '11px'}}>
           <WidgetTitle styles={{marginBottom: '16px'}}>Memory</WidgetTitle>
-          <CircleProgress percentage={memoryUsagePercentage} title='MEM'/>
+          <CircleProgress percentage={memoryState} title='MEM' color={getMemoryColor()}/>
         </Grid>
-        <Grid item sx={{width: '100px', marginLeft: '11px' }}>
+        <Grid item sx={{width: '100px', marginLeft: '11px'}}>
           <WidgetTitle styles={{marginBottom: '16px'}}>CPU</WidgetTitle>
-          <CircleProgress percentage={cpuUsagePercentage} title='CPU'/>
+          <CircleProgress percentage={cpuState} title='CPU' color={getCpuColor()}/>
         </Grid>
       </Grid>
     </Widget>

frontend/src/components/videoControls/components/CameraControls/index.js

@@ -1,8 +1,9 @@
-import React from 'react'
+import React, { Fragment } from 'react'
 import { observer } from 'mobx-react'
 import { Radio, RadioGroup, FormControlLabel, FormControl } from '@mui/material'
 import { useTheme } from '@mui/material/styles'
 import { useStore } from '../../../../store'
+import { IS_SIMULATING } from '../../../../constants'
 
 const CameraLabel = observer(({name, marginLeft = 0, connected}) => {
   const theme = useTheme()
@@ -69,9 +70,13 @@ const CameraControls = observer(({ connected }) => {
         value={selectedMode}
         onChange={handleCameraModeChange}
       >
-        <CameraLabel name='rgb' connected={connected} />
-        <CameraLabel name='depth' marginLeft='16px' connected={connected} />
-        <CameraLabel name='sim' marginLeft='16px' connected={connected} />
+        {
+          IS_SIMULATING ? <CameraLabel name='sim' marginLeft='16px' connected={connected} /> :
+          <Fragment>
+            <CameraLabel name='rgb' connected={connected} />
+            <CameraLabel name='depth' marginLeft='16px' connected={connected} />
+          </Fragment>
+        }
       </RadioGroup>
     </FormControl>
   )