Use multi-pass rendering for sensors (#585)

* Add name

* new depth cam

* set lens in image buffer

* depth camera

* depth cam works

* use create camera

* make it better

* try another way

* compute shader example

* fix bug

* depth compute shader

* depth camera

* depth cam!

* restore depth value

* move factor to outside

* format

* optimize for CPU

* format

* use float depth

* turn off show interface when main camera is off

* add doc string

* sensor non-scale

* fix dashboard

* format

* multi-pass rendering

* cancel baseblock semantic settings

* fix bug

* multi-passing works

* fix

* fix

* Use multi-pass rendering

* format

* test instance cam

* fix instance cam

* instance cam

* use 8 bit

* put reset to close in test cases

* fix crosswalk contact

* remove screenshot cam

* fix pssm shadow lag

* run on ubuntu large

* restore

* setup opengl

* setup opengl

* setup opengl

* setup opengl

* disable depth test

.github/workflows/main.yml

@@ -186,7 +186,15 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: 3.9
-      - name: Prepare OpenGL
+      - name: Install dependencies
+        run: sudo apt-get install -y libglu1-mesa-dev freeglut3-dev mesa-common-dev
+      - name: Setup Mesa OpenGL
+        run: |
+          sudo add-apt-repository ppa:kisak/kisak-mesa
+          sudo apt-get update
+          sudo apt-get install libgl1-mesa-glx libgl1-mesa-dri
+          export LIBGL_ALWAYS_SOFTWARE=1
+      - name: open-window
         run: |
           sudo apt-get -y install xvfb
           sudo /usr/bin/Xvfb :0 -screen 0 1280x1024x24 &

metadrive/base_class/base_object.py

@@ -11,7 +11,7 @@
 import seaborn as sns
 from panda3d.bullet import BulletWorld, BulletBodyNode, BulletVehicle
 from panda3d.core import LVector3, NodePath, PandaNode
-from metadrive.constants import Semantics
+from metadrive.constants import Semantics, CameraTagStateKey
 from metadrive.base_class.base_runnable import BaseRunnable
 from metadrive.constants import ObjectState
 from metadrive.engine.asset_loader import AssetLoader
@@ -133,7 +133,7 @@ def __init__(self, name=None, random_seed=None, config=None, escape_random_seed_
         self.origin = NodePath(self.name)
 
         # semantic color
-        self.origin.setTag("type", self.SEMANTIC_LABEL)
+        self.origin.setTag(CameraTagStateKey.Semantic, self.SEMANTIC_LABEL)
 
         # Temporally store bullet nodes that have to place in bullet world (not NodePath)
         self.dynamic_nodes = PhysicsNodeList()
@@ -194,7 +194,7 @@ def add_body(self, physics_body, add_to_static_world=False):
 
             self._node_path_list.append(self.origin)
             self.origin = new_origin
-            self.origin.setTag("type", self.SEMANTIC_LABEL)
+            self.origin.setTag(CameraTagStateKey.Semantic, self.SEMANTIC_LABEL)
             if add_to_static_world:
                 self.static_nodes.append(physics_body)
             else:

metadrive/component/block/base_block.py

@@ -1,33 +1,33 @@
 import logging
+from metadrive.constants import Semantics, CameraTagStateKey
 import math
-import time
+import warnings
 from abc import ABC
 from typing import Dict
-from panda3d.core import LVecBase4, TextureStage
-from metadrive.utils.utils import create_rectangle_from_midpoints
+
 import numpy as np
 from panda3d.bullet import BulletBoxShape
 from panda3d.bullet import BulletConvexHullShape
 from panda3d.bullet import BulletTriangleMeshShape, BulletTriangleMesh
 from panda3d.core import LPoint3f, Material
+from panda3d.core import TextureStage
 from panda3d.core import Vec3, LQuaternionf, RigidBodyCombiner, \
     SamplerState, NodePath, Texture
 from panda3d.core import Vec4
-from metadrive.constants import TerrainProperty
+
 from metadrive.base_class.base_object import BaseObject
 from metadrive.component.road_network.node_road_network import NodeRoadNetwork
 from metadrive.component.road_network.road import Road
 from metadrive.constants import CollisionGroup
 from metadrive.constants import MetaDriveType, CamMask, PGLineType, PGLineColor, PGDrivableAreaProperty
-from metadrive.constants import Semantics
+from metadrive.constants import TerrainProperty
 from metadrive.engine.asset_loader import AssetLoader
 from metadrive.engine.core.physics_world import PhysicsWorld
+from metadrive.engine.logger import get_logger
 from metadrive.engine.physics_node import BaseRigidBodyNode, BaseGhostBodyNode
 from metadrive.utils.coordinates_shift import panda_vector, panda_heading
 from metadrive.utils.math import norm
 from metadrive.utils.vertex import make_polygon_model
-from metadrive.engine.logger import get_logger
-import warnings
 
 warnings.filterwarnings('ignore', 'invalid value encountered in intersection')
 
@@ -219,12 +219,6 @@ def _create_in_world(self, skip=False):
         self.sidewalk_node_path = NodePath(RigidBodyCombiner(self.name + "_sidewalk"))
         self.crosswalk_node_path = NodePath(RigidBodyCombiner(self.name + "_crosswalk"))
         self.lane_node_path = NodePath(RigidBodyCombiner(self.name + "_lane"))
-        self.lane_vis_node_path = NodePath(RigidBodyCombiner(self.name + "_lane_vis"))
-
-        self.sidewalk_node_path.setTag("type", Semantics.SIDEWALK.label)
-        self.crosswalk_node_path.setTag("type", Semantics.CROSSWALK.label)
-        self.lane_vis_node_path.setTag("type", Semantics.ROAD.label)
-        self.lane_line_node_path.setTag("type", Semantics.LANE_LINE.label)
 
         if skip:  # for debug
             pass
@@ -257,17 +251,16 @@ def _create_in_world(self, skip=False):
         self.lane_node_path.flattenStrong()
         self.lane_node_path.node().collect()
 
-        self.lane_vis_node_path.flattenStrong()
-        self.lane_vis_node_path.node().collect()
-        self.lane_vis_node_path.hide(CamMask.DepthCam | CamMask.ScreenshotCam | CamMask.SemanticCam)
-
         self.origin.hide(CamMask.Shadow)
 
         self.sidewalk_node_path.reparentTo(self.origin)
         self.crosswalk_node_path.reparentTo(self.origin)
         self.lane_line_node_path.reparentTo(self.origin)
         self.lane_node_path.reparentTo(self.origin)
 
+        # semantics
+        self.sidewalk_node_path.setTag(CameraTagStateKey.Semantic, Semantics.SIDEWALK.label)
+
         try:
             self._bounding_box = self.block_network.get_bounding_box()
         except:
@@ -279,7 +272,6 @@ def _create_in_world(self, skip=False):
         self._node_path_list.append(self.crosswalk_node_path)
         self._node_path_list.append(self.lane_line_node_path)
         self._node_path_list.append(self.lane_node_path)
-        self._node_path_list.append(self.lane_vis_node_path)
 
     def create_in_world(self):
         """
@@ -416,16 +408,14 @@ def _construct_crosswalk(self):
                 if polygons is None:
                     continue
                 for polygon in polygons:
-                    np = make_polygon_model(polygon, 0.0)
-                    np.reparentTo(self.crosswalk_node_path)
-                    np.setPos(0, 0, 0.005)
+                    np = make_polygon_model(polygon, 1.5)
 
                     body_node = BaseGhostBodyNode(cross_id, MetaDriveType.CROSSWALK)
                     body_node.setKinematic(False)
                     body_node.setStatic(True)
                     body_np = self.crosswalk_node_path.attachNewNode(body_node)
                     # A trick allowing collision with sidewalk
-                    body_np.setPos(0, 0, 0.5)
+                    body_np.setPos(0, 0, 1.5)
                     self._node_path_list.append(body_np)
 
                     geom = np.node().getGeom(0)
@@ -436,7 +426,7 @@ def _construct_crosswalk(self):
                     body_node.addShape(shape)
                     self.static_nodes.append(body_node)
                     body_node.setIntoCollideMask(CollisionGroup.Crosswalk)
-                    self._node_path_list.append(np)
+                    np.removeNode()
 
     def _construct_lane(self, lane, lane_index):
         """
@@ -525,22 +515,5 @@ def _construct_lane_line_segment(self, start_point, end_point, line_color: Vec4,
         # theta = -numpy.arctan2(direction_v[1], direction_v[0])
         theta = panda_heading(math.atan2(direction_v[1], direction_v[0]))
         body_np.setQuat(LQuaternionf(math.cos(theta / 2), 0, 0, math.sin(theta / 2)))
-        #
-        # if self.render and not self.use_render_pipeline:
-        #     # For visualization in semantic camera
-        #     polygon = create_rectangle_from_midpoints(start_point, end_point, PGDrivableAreaProperty.LANE_LINE_WIDTH)
-        #     lane_line = make_polygon_model(polygon, height=0)
-        #     lane_line.reparentTo(parent_np)
-        #     lane_line.setPos(0, 0, 0.005)
-        if self.render and not self.use_render_pipeline \
-                and self.engine and self.engine.global_config["build_lane_line_for_semantic_cam"]:
-            # For visualization
-            lane_line = NodePath("line_seg")
-            self.line_seg.instanceTo(lane_line)
-            lane_line.setScale(
-                length, PGDrivableAreaProperty.LANE_LINE_WIDTH, PGDrivableAreaProperty.LANE_LINE_THICKNESS
-            )
-            lane_line.setQuat(LQuaternionf(math.cos(theta / 2), 0, 0, math.sin(theta / 2)))
-            lane_line.setPos(panda_vector(middle, 0.1))
-            lane_line.reparentTo(parent_np)
+
         return node_path_list

metadrive/component/sensors/base_camera.py

@@ -65,8 +65,7 @@ def __init__(self, engine, need_cuda=False, frame_buffer_property=None):
             self._cuda_buffer = None
 
             # make texture
-            self.cuda_texture = Texture()
-            self.buffer.addRenderTexture(self.cuda_texture, GraphicsOutput.RTMBindOrCopy)
+            self._make_cuda_texture()
 
             def _callback_func(cbdata: DisplayRegionDrawCallbackData):
                 # print("DRAW CALLBACK!!!!!!!!!!!!!!!11")
@@ -88,6 +87,13 @@ def _callback_func(cbdata: DisplayRegionDrawCallbackData):
             self.new_cuda_mem_ptr = None
             self.cuda_rendered_result = None
 
+    def _make_cuda_texture(self):
+        """
+        Make a texture for cuda access
+        """
+        self.cuda_texture = Texture()
+        self.buffer.addRenderTexture(self.cuda_texture, GraphicsOutput.RTMBindOrCopy)
+
     @property
     def enable_cuda(self):
         return self is not None and self._enable_cuda
@@ -109,11 +115,9 @@ def perceive(self, base_object, clip=True) -> np.ndarray:
         self.track(base_object)
         if self.enable_cuda:
             assert self.cuda_rendered_result is not None
-            ret = self.cuda_rendered_result[..., :-1][..., ::-1][::-1][..., :self.num_channels]
+            ret = self.cuda_rendered_result[..., :self.num_channels][..., ::-1][::-1]
         else:
             ret = self.get_rgb_array_cpu()
-        # if self.engine.global_config["rgb_to_grayscale"]:
-        #     ret = np.dot(ret[..., :3], [0.299, 0.587, 0.114])
         if not clip:
             return ret.astype(np.uint8, copy=False, order="C")
         else:

metadrive/component/sensors/dashboard.py

@@ -1,4 +1,4 @@
-from panda3d.core import NodePath, PGTop, TextNode, CardMaker, Vec3
+from panda3d.core import NodePath, PGTop, TextNode, CardMaker, Vec3, OrthographicLens
 
 from metadrive.component.sensors.base_sensor import BaseSensor
 from metadrive.constants import CamMask
@@ -9,6 +9,12 @@ class DashBoard(ImageBuffer, BaseSensor):
     """
     Dashboard for showing the speed and brake/throttle/steering
     """
+    def perceive(self, *args, **kwargs):
+        """
+        This is only used for GUI and won't provide any observation result
+        """
+        raise NotImplementedError
+
     PARA_VIS_LENGTH = 12
     PARA_VIS_HEIGHT = 1
     MAX_SPEED = 120
@@ -86,7 +92,20 @@ def __init__(self, engine, *, cuda):
         )
         self._node_path_list.extend(tmp_node_path_list)
 
+    def _create_camera(self, pos, bkg_color):
+        """
+        Create orthogonal camera for the buffer
+        """
+        self.cam = cam = self.engine.makeCamera(self.buffer, clearColor=bkg_color)
+        cam.node().setCameraMask(self.CAM_MASK)
+
+        self.cam.reparentTo(self.origin)
+        self.cam.setPos(pos)
+
     def update_vehicle_state(self, vehicle):
+        """
+        Update the dashboard result given a vehicle
+        """
         steering, throttle_brake, speed = vehicle.steering, vehicle.throttle_brake, vehicle.speed_km_h
         if throttle_brake < 0:
             self.para_vis_np["Throttle"].setScale(0, 1, 1)
@@ -115,8 +134,8 @@ def remove_display_region(self):
         self.buffer.set_active(False)
         super(DashBoard, self).remove_display_region()
 
-    def add_display_region(self, display_region):
-        super(DashBoard, self).add_display_region(display_region)
+    def add_display_region(self, display_region, keep_height=False):
+        super(DashBoard, self).add_display_region(display_region, False)
         self.buffer.set_active(True)
         self.origin.reparentTo(self.aspect2d_np)