EDGEAI-364

src/config/webui.html

@@ -269,6 +269,12 @@ <h2>WebUI Configuration</h2>
                 Configure the settings for the WebUI Service, including bin widths, thresholds, and topics.
             </div>
 
+            <!-- Mirror Inputs Checkbox -->
+            <div class="form-group">
+                <label for="mirror_inputs">Mirror Inputs</label>
+                <input type="checkbox" id="mirror_inputs" name="mirror_inputs">
+            </div>
+
             <!-- Angle Bin Configuration -->
             <div class="form-group">
                 <label for="angle_bin_width">Angle Bin Width:</label>
@@ -393,7 +399,11 @@ <h2>WebUI Configuration</h2>
                 <label for="draw_box_text">Draw Box Text:</label>
                 <input type="checkbox" id="draw_box_text" name="draw_box_text">
             </div>
-
+            <!-- Show Stats Checkbox -->
+            <div class="form-group">
+                <label for="show_stats">Show Stats</label>
+                <input type="checkbox" id="show_stats" name="show_stats">
+            </div>
             <button id="saveConfig">Save Configuration</button>
         </div>
     </main>
@@ -408,6 +418,7 @@ <h2>WebUI Configuration</h2>
             })
                 .then(response => response.json())
                 .then(data => {
+                    document.getElementById('mirror_inputs').checked = data.MIRROR === 'true' || data.MIRROR === true;
                     document.getElementById('angle_bin_width').value = data.ANGLE_BIN_WIDTH || 10;
                     document.getElementById('angle_bin_limits_min').value = data.ANGLE_BIN_LIMITS_MIN || -70;
                     document.getElementById('angle_bin_limits_max').value = data.ANGLE_BIN_LIMITS_MAX || 70;
@@ -429,6 +440,7 @@ <h2>WebUI Configuration</h2>
                     document.getElementById('draw_unknown_cells').checked = data.DRAW_UNKNOWN_CELLS === 'true' || data.DRAW_UNKNOWN_CELLS === true;
                     document.getElementById('draw_box').checked = data.DRAW_BOX === 'true' || data.DRAW_BOX === true;
                     document.getElementById('draw_box_text').checked = data.DRAW_BOX_TEXT === 'true' || data.DRAW_BOX_TEXT === true;
+                    document.getElementById('show_stats').checked = data.SHOW_STATS === 'true' || data.SHOW_STATS === true;
 
                     const deviceToggle = document.getElementById('device_type');
                     const deviceTypeText = document.getElementById('device_type_text');
@@ -445,6 +457,7 @@ <h2>WebUI Configuration</h2>
         document.getElementById('saveConfig').addEventListener('click', function () {
             const configData = {
                 fileName: "webui",
+                MIRROR: document.getElementById('mirror_inputs').checked.toString(),
                 ANGLE_BIN_WIDTH: document.getElementById('angle_bin_width').value,
                 ANGLE_BIN_LIMITS_MIN: document.getElementById('angle_bin_limits_min').value,
                 ANGLE_BIN_LIMITS_MAX: document.getElementById('angle_bin_limits_max').value,
@@ -467,6 +480,7 @@ <h2>WebUI Configuration</h2>
                 DRAW_UNKNOWN_CELLS: document.getElementById('draw_unknown_cells').checked.toString(),
                 DRAW_BOX: document.getElementById('draw_box').checked.toString(),
                 DRAW_BOX_TEXT: document.getElementById('draw_box_text').checked.toString(),
+                SHOW_STATS: document.getElementById('show_stats').checked.toString(),
             };
 
             fetch('/config/webui', {

src/js/ProjectedMask.js

@@ -2,7 +2,7 @@ import * as THREE from './three.js'
 
 // Based on https://tympanus.net/codrops/2020/01/07/playing-with-texture-projection-in-three-js/
 export default class ProjectedMask extends THREE.ShaderMaterial {
-    constructor({ camera, texture, colors, alphas, default_alpha = 0.7, ...options } = {}) {
+    constructor({ camera, texture, colors, alphas, default_alpha = 0.7, flip = false, ...options } = {}) {
         if (!texture || !texture.isTexture) {
             throw new Error('Invalid texture passed to the ProjectedMask')
         }
@@ -56,6 +56,8 @@ export default class ProjectedMask extends THREE.ShaderMaterial {
         const modelMatrixCamera = camera.matrixWorld.clone()
 
         const projPosition = camera.position.clone()
+
+        const flip_shader = flip ? `uv.x = 1.0 - uv.x;` : ``
         super({
             ...options,
             uniforms: {
@@ -119,6 +121,7 @@ export default class ProjectedMask extends THREE.ShaderMaterial {
                 void main() {
                     float w = max(vTexCoords.w, 0.0);
                     vec2 uv = (vTexCoords.xy / w) * 0.5 + 0.5;
+                    ${flip_shader}
                     if (!(0.0 <= uv.x && uv.x <= 1.0 && 0.0 <= uv.y && uv.y <= 1.0)) {
                         pc_fragColor = vec4(0.0,0.0,0.0,0.0);
                         return;

src/js/ProjectedMaterial.js

@@ -2,7 +2,7 @@ import * as THREE from './three.js'
 
 // Based on https://tympanus.net/codrops/2020/01/07/playing-with-texture-projection-in-three-js/
 export default class ProjectedMaterial extends THREE.ShaderMaterial {
-    constructor({ camera, texture, color = 0xffffff, ...options } = {}) {
+    constructor({ camera, texture, color = 0xffffff, flip = false, ...options } = {}) {
         if (!texture || !texture.isTexture) {
             throw new Error('Invalid texture passed to the ProjectedMaterial')
         }
@@ -23,6 +23,7 @@ export default class ProjectedMaterial extends THREE.ShaderMaterial {
 
         const projPosition = camera.position.clone()
 
+        const flip_shader = flip ? `uv.x = 1.0 - uv.x;` : ``
         super({
             ...options,
             uniforms: {
@@ -64,7 +65,7 @@ export default class ProjectedMaterial extends THREE.ShaderMaterial {
                 void main() {
                     float w = max(vTexCoords.w, 0.0);
                     vec2 uv = (vTexCoords.xy / w) * 0.5 + 0.5;
-
+                    ${flip_shader}            
                     vec4 outColor = texture(tex, uv);
                     if (!(0.0 <= uv.x && uv.x <= 1.0 && 0.0 <= uv.y && uv.y <= 1.0)) {
                         outColor.a = 0.0;

src/js/Stats.js

@@ -168,4 +168,38 @@ Stats.Panel = function (name, fg, bg) {
 
 };
 
+export function fpsUpdate(panel, max) {
+    if (!max) {
+        max = 40
+    }
+    let fpsInd = 0
+    let timeBetweenUpdates = []
+    let lastUpdateTime = 0
+    let stablized = false
+    let firstUpdate = 0
+    return () => {
+        if (!lastUpdateTime) {
+            lastUpdateTime = performance.now()
+            firstUpdate = lastUpdateTime
+            return
+        }
+        const curr = performance.now()
+        if (timeBetweenUpdates.length < 10) {
+            timeBetweenUpdates.push(curr - lastUpdateTime)
+            lastUpdateTime = curr
+            return
+        }
+        timeBetweenUpdates[fpsInd] = curr - lastUpdateTime
+        fpsInd = (fpsInd + 1) % timeBetweenUpdates.length
+        if (stablized) {
+            const avg_fps = 1000 / timeBetweenUpdates.reduce((a, b) => a + b, 0) * timeBetweenUpdates.length
+            panel.update(avg_fps, max)
+        } else if (curr - firstUpdate > 2000) {
+            // has been 2s since first update
+            stablized = true
+        }
+        lastUpdateTime = curr
+    }
+}
+
 export { Stats as default };

src/js/classify.js

@@ -250,7 +250,8 @@ export function project_points_onto_box(points, boxes) {
             z: p.z,
             range: p.range,
             angle: p.angle,
-            speed: p.speed
+            speed: p.speed,
+            class: p.class
         }
 
         pos.applyMatrix3(cam_mtx)
@@ -269,36 +270,59 @@ export function project_points_onto_box(points, boxes) {
 
     points_cpy.sort(dynamicSortMultiple("range"))
 
-
+    let extra_points = []
     for (let p of points_cpy)  {
         if (p.class == 0) {
             continue
         }
         let i = p.i;
         let j = p.j;
+
+        let point_marked = false
         for (let box of boxes) {
             if (!point_in_box(j, i, box)) {
                 continue
             }
+            point_marked = true
             if (box.text) {
                 continue
             }
             box.text = `${p.range.toFixed(1).padStart(5, " ")}m\n${p.speed.toFixed(1).padStart(5, " ")}m/s`
+
         }
+
+        if (point_marked) {
+           continue
+        } 
+        extra_points.push(p)
+    }
+    for (let p of extra_points) {
+        let box = {}
+        box.center_x = p.j
+        box.center_y = p.i
+        box.width = Math.atan(0.7 / p.x) / 1.43117 // maybe get this from projection
+        box.height = Math.atan(1.7 / p.x) / 1.43117 // maybe get this from projection
+        box.label = 1.0
+        box.score = 0.7
+        box.distance = p.range
+        box.speed = p.speed
+        box.track = "NA"
+        box.text = `${p.range.toFixed(1).padStart(5, " ")}m\n${p.speed.toFixed(1).padStart(5, " ")}m/s`
+        boxes.push(box)
     }
 }
 
 function point_in_box(x,y, box) {
-    if (x < box.center_x - box.width / 2 - 0.05) { // pad 0.15 left of the box
+    if (x < box.center_x - box.width / 2 - 0.1) { // pad 0.1 left of the box
         return false
     }
-    if (x > box.center_x + box.width / 2 + 0.05) { // pad 0.15 right of the box
+    if (x > box.center_x + box.width / 2 + 0.1) { // pad 0.1 right of the box
         return false
     }
-    if (y < box.center_y - box.height / 2) {
+    if (y < box.center_y - box.height / 2 - 0.1) {
         return false
     }
-    if (y > box.center_y + box.height / 2) {
+    if (y > box.center_y + box.height / 2 + 0.1) {
         return false
     }
     return true

src/js/combined.js

@@ -3,10 +3,10 @@ import ProjectedMaterial from './ProjectedMaterial.js'
 import ProjectedMask from './ProjectedMask.js'
 import segstream, { get_shape } from './mask.js'
 import h264Stream from './stream.js'
-import pcdStream from './pcd.js'
+import pcdStream, { preprocessPoints } from './pcd.js'
 import { project_points_onto_box } from './classify.js'
 import boxesstream from './boxes.js'
-import Stats from "./Stats.js"
+import Stats, { fpsUpdate } from "./Stats.js"
 import droppedframes from './droppedframes.js'
 import { parseNumbersInObject } from './parseNumbersInObject.js';
 import { OrbitControls } from './OrbitControls.js'
@@ -21,44 +21,10 @@ const cameraPanel = stats.addPanel(new Stats.Panel('cameraFPS', '#fff', '#222'))
 // const renderPanel = stats.addPanel(new Stats.Panel('renderFPS', '#4ff', '#022'));
 const radarPanel = stats.addPanel(new Stats.Panel('radarFPS', '#ff4', '#220'));
 const modelPanel = stats.addPanel(new Stats.Panel('modelFPS', '#f4f', '#210'));
+stats.showPanel([])
 stats.dom.style.cssText = "position: absolute; top: 0px; right: 0px; opacity: 0.9; z-index: 10000;";
-stats.showPanel([3, 4, 5])
-document.querySelector('main').appendChild(stats.dom);
-
-function fpsUpdate(panel, max) {
-    if (!max) {
-        max = 40
-    }
-    let fpsInd = 0
-    let timeBetweenUpdates = []
-    let lastUpdateTime = 0
-    let stablized = false
-    let firstUpdate = 0
-    return () => {
-        if (!lastUpdateTime) {
-            lastUpdateTime = performance.now()
-            firstUpdate = lastUpdateTime
-            return
-        }
-        const curr = performance.now()
-        if (timeBetweenUpdates.length < 10) {
-            timeBetweenUpdates.push(curr - lastUpdateTime)
-            lastUpdateTime = curr
-            return
-        }
-        timeBetweenUpdates[fpsInd] = curr - lastUpdateTime
-        fpsInd = (fpsInd + 1) % timeBetweenUpdates.length
-        if (stablized) {
-            const avg_fps = 1000 / timeBetweenUpdates.reduce((a, b) => a + b, 0) * timeBetweenUpdates.length
-            panel.update(avg_fps, max)
-        } else if (curr - firstUpdate > 2000) {
-            // has been 2s since first update
-            stablized = true
-        }
-        lastUpdateTime = curr
-    }
-}
 
+document.querySelector('main').appendChild(stats.dom);
 
 const grid_scene = new THREE.Scene()
 grid_scene.background = new THREE.Color(0xa0a0a0)
@@ -109,7 +75,8 @@ let socketUrlDetect = '/rt/detect/boxes2d/'
 let socketUrlMask = '/rt/detect/mask/'
 let socketUrlErrors = '/ws/dropped'
 let RANGE_BIN_LIMITS = [0, 20]
-
+let mirror = false
+let show_stats = false
 
 
 droppedframes(socketUrlErrors, playerCanvas)
@@ -188,21 +155,23 @@ loader.load(
         console.log(config)
 
         init_config(config)
-
+        if (show_stats) {
+            stats.showPanel([3, 4, 5])
+        } 
         config.GRID_DRAW_PCD = config.COMBINED_GRID_DRAW_PCD
         init_grid(grid_scene, renderer_grid, camera_grid, config)
 
         const quad = new THREE.PlaneGeometry(width / height * 500, 500);
         const cameraUpdate = fpsUpdate(cameraPanel)
-        h264Stream(socketUrlH264, 1920, 1080, 30, (timing) => {
+        h264Stream(socketUrlH264, 1920, 1080, 30, () => {
             cameraUpdate(); resetTimeout(); cameraNeedsUpdate = true;
-            // cameraMSPanel.update(timing.decode_time, 33) 
         }).then((tex) => {
             texture_camera = tex;
             material_proj = new ProjectedMaterial({
                 camera: camera, // the camera that acts as a projector
                 texture: texture_camera, // the texture being projected
                 color: '#000', // the color of the object if it's not projected on
+                flip: mirror,
                 transparent: true,
             })
             const mesh_cam = new THREE.Mesh(quad, material_proj);
@@ -219,14 +188,14 @@ loader.load(
         // const maskMSPanel = stats.addPanel(new Stats.Panel('mask decode ms', '#A2A', '#420'));
         get_shape(socketUrlMask, (height, width, length, mask) => {
             const classes = Math.round(mask.length / height / width)
-            segstream(socketUrlMask, height, width, classes, (timing) => {
+            segstream(socketUrlMask, height, width, classes, () => {
                 modelFPSUpdate();
-                // maskMSPanel.update(timing.decode_time, 33) 
             }).then((texture_mask) => {
                 material_mask = new ProjectedMask({
                     camera: camera, // the camera that acts as a projector
                     texture: texture_mask, // the texture being projected
                     transparent: true,
+                    flip: mirror,
                     colors: mask_colors,
                 })
                 const mesh_mask = new THREE.Mesh(quad, material_mask);
@@ -240,6 +209,12 @@ loader.load(
         let boxes;
         boxesstream(socketUrlDetect, null, () => {
             if (boxes && radar_points) {
+                if (mirror) {
+                    for (let box of boxes.msg.boxes) {
+                        box.center_x = 1.0 - box.center_x
+                    }
+                }
+
                 drawBoxesSpeedDistance(boxCanvas, boxes.msg.boxes, radar_points.points)
             }
         }).then((b) => {
@@ -249,15 +224,7 @@ loader.load(
         let radarFpsFn = fpsUpdate(radarPanel);
         pcdStream(socketUrlPcd, () => {
             radarFpsFn();
-            let filteredPoints = []
-            for (let p of radar_points.points) {
-                const range = p.range
-                if (range < RANGE_BIN_LIMITS[0] || RANGE_BIN_LIMITS[1] < range) {
-                   continue
-                }
-                filteredPoints.push(JSON.parse(JSON.stringify(p))) // deepclone the point
-            }
-            radar_points.points = filteredPoints
+            radar_points.points = preprocessPoints(RANGE_BIN_LIMITS[0], RANGE_BIN_LIMITS[1], mirror, radar_points.points)
         }).then((pcd) => {
             radar_points = pcd;
             grid_set_radarpoints(radar_points)
@@ -309,6 +276,15 @@ function init_config(config) {
     if (typeof config.DRAW_BOX_TEXT == "boolean") {
         DRAW_BOX_TEXT = config.DRAW_BOX_TEXT
     }
+
+    if (typeof config.MIRROR == "boolean") {
+        mirror = config.MIRROR
+    }
+
+    if (typeof config.SHOW_STATS == "boolean") {
+        show_stats = config.SHOW_STATS
+    }
+
 }