Remake terrain (#531)

* new shader terrain

* to pbr

* fix bug

* hide

* fix arrow bug

* disable model compression

* remove shadow related feature

* pssm class

* finish shader

* pssm flicker

* shading light!

* add shaders to git track

* shadow

* finish pssm

* grass update

* restore

* fix

* fix bug

* naive draw map

* test map visualizer

* small film size

* small shadow

* add docstring

* fix rgb

* fix shader bug

* rename

* fix bug introduced from buffer bits

* format

* fix terrain

* allow force pull

* fix test

* drive in waymo env

* remove lock file

* update file lock

* downgrade sensor test server to ubuntu-20.04

* use small uffer

* split test

* normal mapping

* tex_ratio_to_shader_input

* rough mapping

* use 128

* update docstring

* fix

* use dialate conv for extening driving area

* fix vis carla town

* Auto update the asset

* update

* update shader & improve efficiency

* move lock to outside

* fix asset loader

* terrain test

* format

* sync render pipe

* fix & add test

* format

---------

Co-authored-by: pengzhenghao <pzh@cs.ucla.edu>

.github/workflows/main.yml

@@ -200,7 +200,7 @@ jobs:
           pip install pytest-cov
           pip install ray
           cd metadrive/
-          pytest --cov=./ --cov-config=.coveragerc --cov-report=xml -sv tests/test_sensors
+          pytest --cov=./ --cov-config=.coveragerc --cov-report=xml -sv tests/test_sensors/
 
   test_examples:
     runs-on: ubuntu-latest

metadrive/base_class/base_object.py

@@ -526,4 +526,9 @@ def bounding_box(self):
 
     @property
     def use_render_pipeline(self):
+        """
+        Return if we are using render_pipeline
+        Returns: Boolean
+
+        """
         return self.engine is not None and self.engine.use_render_pipeline

metadrive/component/block/base_block.py

@@ -57,17 +57,18 @@ def __init__(
             self.ts_normal.setMode(TextureStage.M_normal)
 
             # Only maintain one copy of asset
-            self.road_texture = self.loader.loadTexture(AssetLoader.file_path("textures", "sci", "new_color.png"))
-            self.road_normal = self.loader.loadTexture(AssetLoader.file_path("textures", "sci", "normal.jpg"))
-            self.road_texture.set_format(Texture.F_srgb)
-            self.road_normal.set_format(Texture.F_srgb)
-            self.road_texture.setMinfilter(SamplerState.FT_linear_mipmap_linear)
-            self.road_texture.setAnisotropicDegree(8)
-
-            # # continuous line
-            # self.lane_line_model = self.loader.loadModel(AssetLoader.file_path("models", "box.bam"))
-            # self.lane_line_model.setPos(0, 0, -DrivableAreaProperty.LANE_LINE_GHOST_HEIGHT / 2)
-            self.lane_line_texture = self.loader.loadTexture(AssetLoader.file_path("textures", "sci", "floor.jpg"))
+            if self.naive_draw_map:
+                self.road_texture = self.loader.loadTexture(AssetLoader.file_path("textures", "sci", "new_color.png"))
+                self.road_normal = self.loader.loadTexture(AssetLoader.file_path("textures", "sci", "normal.jpg"))
+                self.road_texture.set_format(Texture.F_srgb)
+                self.road_normal.set_format(Texture.F_srgb)
+                self.road_texture.setMinfilter(SamplerState.FT_linear_mipmap_linear)
+                self.road_texture.setAnisotropicDegree(8)
+
+                # # continuous line
+                # self.lane_line_model = self.loader.loadModel(AssetLoader.file_path("models", "box.bam"))
+                # self.lane_line_model.setPos(0, 0, -DrivableAreaProperty.LANE_LINE_GHOST_HEIGHT / 2)
+                self.lane_line_texture = self.loader.loadTexture(AssetLoader.file_path("textures", "sci", "floor.jpg"))
             # self.lane_line_model.setScale(DrivableAreaProperty.STRIPE_LENGTH*4,
             #                                    DrivableAreaProperty.LANE_LINE_WIDTH,
             #                                    DrivableAreaProperty.LANE_LINE_THICKNESS)
@@ -231,7 +232,10 @@ def _create_in_world(self, skip=False):
         self.sidewalk_node_path.reparentTo(self.origin)
         self.lane_line_node_path.reparentTo(self.origin)
         self.lane_node_path.reparentTo(self.origin)
-        self.lane_vis_node_path.reparentTo(self.origin)
+
+        if self.naive_draw_map:
+            self.lane_vis_node_path.reparentTo(self.origin)
+
         try:
             self._bounding_box = self.block_network.get_bounding_box()
         except:
@@ -345,3 +349,12 @@ def destroy(self):
     @property
     def bounding_box(self):
         return self._bounding_box
+
+    @property
+    def naive_draw_map(self):
+        """
+        Use the most naive way to draw map
+        Returns: Boolean
+
+        """
+        return self.engine is None and self.render

metadrive/component/lane/abs_lane.py

@@ -240,7 +240,8 @@ def construct_lane_segment(self, block, position, width, length, theta, lane_ind
         segment_np.setH(theta / np.pi * 180)
         segment_np.setP(-90)
         segment_np.reparentTo(block.lane_node_path)
-        if block.render and not block.use_render_pipeline:
+
+        if block.naive_draw_map:
             cm = CardMaker('card')
             cm.setFrame(-length / 2, length / 2, -width / 2, width / 2)
             cm.setHasNormals(True)
@@ -302,7 +303,7 @@ def construct_lane_line_segment(block, start_point, end_point, line_color: Vec4,
         theta = panda_heading(math.atan2(direction_v[1], direction_v[0]))
         body_np.setQuat(LQuaternionf(math.cos(theta / 2), 0, 0, math.sin(theta / 2)))
 
-        if block.render and not block.use_render_pipeline:
+        if block.naive_draw_map:
             # For visualization
             lane_line = block.loader.loadModel(AssetLoader.file_path("models", "box.bam"))
             lane_line.setTwoSided(False)
@@ -387,7 +388,7 @@ def _construct_lane_only_vis_segment(self, block, position, width, length, theta
         Only create visual part for this lane, usually used with _construct_lane_only_physics_polygon()
         """
         # The lane surface is created with terrain now
-        if block.render and not block.use_render_pipeline:
+        if block.naive_draw_map:
             length += 0.1
             theta = panda_heading(theta)
             cm = CardMaker('card')

metadrive/component/lane/circular_lane.py

@@ -90,12 +90,13 @@ def construct_lane_in_block(self, block, lane_index):
             self.construct_lane_segment(block, middle, width, self.length, theta, lane_index)
             return
 
-        for i in range(segment_num):
-            middle = self.position(self.length * (i + .5) / segment_num, 0)
-            theta = self.heading_theta_at(self.length * (i + .5) / segment_num)
-            width = self.width_at(0) + DrivableAreaProperty.SIDEWALK_LINE_DIST * 2
-            length = self.length
-            self._construct_lane_only_vis_segment(block, middle, width, length * 1.3 / segment_num, theta)
+        if block.naive_draw_map:
+            for i in range(segment_num):
+                middle = self.position(self.length * (i + .5) / segment_num, 0)
+                theta = self.heading_theta_at(self.length * (i + .5) / segment_num)
+                width = self.width_at(0) + DrivableAreaProperty.SIDEWALK_LINE_DIST * 2
+                length = self.length
+                self._construct_lane_only_vis_segment(block, middle, width, length * 1.3 / segment_num, theta)
 
         self._construct_lane_only_physics_polygon(block, self.polygon)
 

metadrive/component/lane/point_lane.py

@@ -162,20 +162,23 @@ def construct_lane_in_block(self, block, lane_index):
             self.index = lane_index
         assert self._polygon is not None, "Polygon is required for building lane"
         # build visualization
-        segment_num = int(self.length / DrivableAreaProperty.LANE_SEGMENT_LENGTH)
-        if segment_num == 0:
-            middle = self.position(self.length / 2, 0)
-            end = self.position(self.length, 0)
-            theta = self.heading_theta_at(self.length / 2)
-            self._construct_lane_only_vis_segment(block, middle, self.VIS_LANE_WIDTH, self.length, theta)
-
-        for i in range(segment_num):
-            middle = self.position(self.length * (i + .5) / segment_num, 0)
-            end = self.position(self.length * (i + 1) / segment_num, 0)
-            direction_v = end - middle
-            theta = math.atan2(direction_v[1], direction_v[0])
-            length = self.length
-            self._construct_lane_only_vis_segment(block, middle, self.VIS_LANE_WIDTH, length * 1.3 / segment_num, theta)
+        if block.naive_draw_map:
+            segment_num = int(self.length / DrivableAreaProperty.LANE_SEGMENT_LENGTH)
+            if segment_num == 0:
+                middle = self.position(self.length / 2, 0)
+                end = self.position(self.length, 0)
+                theta = self.heading_theta_at(self.length / 2)
+                self._construct_lane_only_vis_segment(block, middle, self.VIS_LANE_WIDTH, self.length, theta)
+
+            for i in range(segment_num):
+                middle = self.position(self.length * (i + .5) / segment_num, 0)
+                end = self.position(self.length * (i + 1) / segment_num, 0)
+                direction_v = end - middle
+                theta = math.atan2(direction_v[1], direction_v[0])
+                length = self.length
+                self._construct_lane_only_vis_segment(
+                    block, middle, self.VIS_LANE_WIDTH, length * 1.3 / segment_num, theta
+                )
 
         # build physics contact
         if self.need_lane_localization:

metadrive/component/map/base_map.py

@@ -1,4 +1,7 @@
 import logging
+import geopandas as gpd
+from shapely.ops import unary_union
+from metadrive.utils.utils import time_me
 import math
 
 import cv2
@@ -153,14 +156,14 @@ def get_semantic_map(
         layer=("lane_line", "lane")
     ):
         """
-        Get semantics of the map
+        Get semantics of the map for terrain generation
         :param size: [m] length and width
         :param pixels_per_meter: the returned map will be in (size*pixels_per_meter * size*pixels_per_meter) size
         :param color_setting: color palette for different attribute. When generating terrain, make sure using
         :param line_sample_interval: [m] It determines the resolution of sampled points.
         :param layer: layer to get
         MapTerrainAttribute
-        :return: heightfield image
+        :return: semantic map
         """
         if self._semantic_map is None:
             all_lanes = self.get_map_features(interval=line_sample_interval)
@@ -186,8 +189,8 @@ def get_semantic_map(
                         polylines.append((obj["polyline"], MapTerrainSemanticColor.get_color(obj["type"])))
 
             size = int(size * pixels_per_meter)
-            mask = np.zeros([size, size, 4], dtype=np.float32)
-            mask[..., 0:] = color_setting.get_color(MetaDriveType.GROUND)
+            mask = np.zeros([size, size, 1], dtype=np.float32)
+            mask[..., 0] = color_setting.get_color(MetaDriveType.GROUND)
             # create an example bounding box polygon
             # for idx in range(len(polygons)):
             center_p = self.get_center_point()
@@ -210,6 +213,7 @@ def get_semantic_map(
             self._semantic_map = mask
         return self._semantic_map
 
+    # @time_me
     def get_height_map(
         self,
         size=2048,
@@ -218,7 +222,7 @@ def get_height_map(
         height=1,
     ):
         """
-        Get height of the map
+        Get height of the map for terrain generation
         :param size: [m] length and width
         :param pixels_per_meter: the returned map will be in (size*pixels_per_meter * size*pixels_per_meter) size
         :param extension: If > 1, the returned height map's drivable region will be enlarged.
@@ -240,30 +244,19 @@ def get_height_map(
             center_p = self.get_center_point()
             need_scale = abs(extension - 1) > 1e-1
             for polygon in polygons:
-                if need_scale:
-                    scaled_polygon = Polygon(polygon).buffer(extension, join_style=2)
-                    if isinstance(scaled_polygon, MultiPolygon):
-                        scaled_polygons = scaled_polygon.geoms
-                    else:
-                        scaled_polygons = [scaled_polygon]
-                    for scaled_polygon in scaled_polygons:
-                        points = [
-                            [
-                                int(
-                                    (scaled_polygon.exterior.coords.xy[0][index] - center_p[0]) * pixels_per_meter +
-                                    size / 2
-                                ),
-                                int((scaled_polygon.exterior.coords.xy[1][index] - center_p[1]) * pixels_per_meter) +
-                                size / 2
-                            ] for index in range(len(scaled_polygon.exterior.coords.xy[0]))
-                        ]
-                else:
-                    points = [
-                        [
-                            int((x - center_p[0]) * pixels_per_meter + size / 2),
-                            int((y - center_p[1]) * pixels_per_meter) + size / 2
-                        ] for x, y in polygon
-                    ]
+                points = [
+                    [
+                        int((x - center_p[0]) * pixels_per_meter + size / 2),
+                        int((y - center_p[1]) * pixels_per_meter) + size / 2
+                    ] for x, y in polygon
+                ]
                 cv2.fillPoly(mask, np.asarray([points]).astype(np.int32), color=[height])
+            if need_scale:
+                # Define a kernel. A 3x3 rectangle kernel
+                kernel = np.ones((extension * pixels_per_meter + 1, extension * pixels_per_meter + 1), np.uint8)
+
+                # Apply dilation
+                mask = cv2.dilate(mask, kernel, iterations=1)
+                mask = np.expand_dims(mask, axis=-1)
             self._height_map = mask
         return self._height_map

metadrive/component/sensors/depth_camera.py

@@ -38,7 +38,7 @@ def __init__(self, width, height, engine, *, cuda=False):
         #     frag_path = AssetLoader.file_path("shaders", "depth_cam_gles.frag.glsl")
         # else:
         if self.VIEW_GROUND:
-            ground = PNMImage(513, 513, 4)
+            ground = PNMImage(257, 257, 4)
             ground.fill(1., 1., 1.)
 
             self.GROUND = GeoMipTerrain("mySimpleTerrain")
@@ -71,11 +71,11 @@ def _setup_effect(self):
         """
         from metadrive.utils import is_mac
         if is_mac():
-            vert_path = AssetLoader.file_path("shaders", "{}_mac.vert.glsl".format(self.shader_name))
-            frag_path = AssetLoader.file_path("shaders", "{}_mac.frag.glsl".format(self.shader_name))
+            vert_path = AssetLoader.file_path("../shaders", "{}_mac.vert.glsl".format(self.shader_name))
+            frag_path = AssetLoader.file_path("../shaders", "{}_mac.frag.glsl".format(self.shader_name))
         else:
-            vert_path = AssetLoader.file_path("shaders", "{}.vert.glsl".format(self.shader_name))
-            frag_path = AssetLoader.file_path("shaders", "{}.frag.glsl".format(self.shader_name))
+            vert_path = AssetLoader.file_path("../shaders", "{}.vert.glsl".format(self.shader_name))
+            frag_path = AssetLoader.file_path("../shaders", "{}.frag.glsl".format(self.shader_name))
         custom_shader = Shader.load(Shader.SL_GLSL, vertex=vert_path, fragment=frag_path)
         self.get_cam().node().setInitialState(
             RenderState.make(

metadrive/component/sensors/distance_detector.py

@@ -1,5 +1,5 @@
 from collections import namedtuple
-from metadrive.engine.core.line import MyLineNodePath
+from metadrive.engine.core.draw_line import ColorLineNodePath
 
 import numpy as np
 from panda3d.core import NodePath, LVecBase4
@@ -108,7 +108,9 @@ def __init__(self, engine):
         self.mask = CollisionGroup.BrokenLaneLine
         # visualization
         self.origin.hide(CamMask.RgbCam | CamMask.Shadow | CamMask.Shadow | CamMask.DepthCam | CamMask.SemanticCam)
-        self.cloud_points_vis = MyLineNodePath(self.origin, thickness=3.0) if AssetLoader.loader is not None else None
+        self.cloud_points_vis = ColorLineNodePath(
+            self.origin, thickness=3.0
+        ) if AssetLoader.loader is not None else None
         self.logger.debug("Load Vehicle Module: {}".format(self.__class__.__name__))
         self._current_frame = None
 

metadrive/component/sensors/rgb_camera.py

@@ -1,7 +1,7 @@
 import panda3d.core as p3d
 from direct.filter.FilterManager import FilterManager
 from simplepbr import _load_shader_str
-
+from panda3d.core import FrameBufferProperties
 from metadrive.component.sensors.base_camera import BaseCamera
 from metadrive.constants import CamMask
 
@@ -52,3 +52,19 @@ def _setup_effect(self):
         self.tonemap_quad.set_shader(tonemap_shader)
         self.tonemap_quad.set_shader_input('tex', self.scene_tex)
         self.tonemap_quad.set_shader_input('exposure', 1.0)
+
+    def _create_buffer(self, width, height, frame_buffer_property):
+        """
+        Create the buffer object to render the scene into it. Use 3 channels speed up the data retrieval for RGB Camera.
+        Args:
+            width: image width
+            height: image height
+            frame_buffer_property: panda3d.core.FrameBufferProperties
+
+        Returns: buffer object
+
+        """
+        if frame_buffer_property is None:
+            frame_buffer_property = FrameBufferProperties()
+        frame_buffer_property.set_rgba_bits(8, 8, 8, 0)  # disable alpha for RGB camera
+        return self.engine.win.makeTextureBuffer("camera", width, height, fbp=frame_buffer_property)

metadrive/component/sensors/semantic_camera.py

@@ -79,16 +79,3 @@ def _setup_effect(self):
         for t in [v for v, m in vars(Semantics).items() if not (v.startswith('_') or callable(m))]:
             label, c = getattr(Semantics, t)
             cam.setTagState(label, RenderState.make(ColorAttrib.makeFlat((c[0] / 255, c[1] / 255, c[2] / 255, 1)), 1))
-
-    def _create_buffer(self, width, height, frame_buffer_property):
-        """
-        The buffer should be created without frame_buffer_property
-        Args:
-            width: Image width
-            height: Image height
-            frame_buffer_property: disabled in Semantic Camera
-
-        Returns: Buffer object
-
-        """
-        return self.engine.win.makeTextureBuffer("camera", width, height)

metadrive/component/vehicle_navigation_module/trajectory_navigation.py

@@ -1,4 +1,5 @@
 from collections import deque
+from metadrive.constants import CamMask
 
 import numpy as np
 from panda3d.core import NodePath, Material
@@ -41,6 +42,9 @@ def __init__(
             name=name,
             vehicle_config=vehicle_config
         )
+        if self.origin is not None:
+            self.origin.hide(CamMask.RgbCam | CamMask.Shadow | CamMask.DepthCam | CamMask.SemanticCam)
+
         self._route_completion = 0
         self.checkpoints = None  # All check points
 
@@ -56,12 +60,12 @@ def __init__(
                 if self._navi_point_model is None:
                     self._navi_point_model = AssetLoader.loader.loadModel(AssetLoader.file_path("models", "box.bam"))
                     self._navi_point_model.setScale(0.5)
-                    if self.engine.use_render_pipeline:
-                        material = Material()
-                        material.setBaseColor((19 / 255, 212 / 255, 237 / 255, 1))
-                        material.setShininess(16)
-                        material.setEmission((0.2, 0.2, 0.2, 0.2))
-                        self._navi_point_model.setMaterial(material, True)
+                    # if self.engine.use_render_pipeline:
+                    material = Material()
+                    material.setBaseColor((19 / 255, 212 / 255, 237 / 255, 1))
+                    material.setShininess(16)
+                    material.setEmission((0.2, 0.2, 0.2, 0.2))
+                    self._navi_point_model.setMaterial(material, True)
                 self._navi_point_model.instanceTo(model)
                 model.reparentTo(self.origin)