Remaking the SimplePBR pipeline with PSSM Shadow (#577)

* remove redundant code

* line color

* don't import strings

* begin merge

* fix terrain

* fix

* render pipeline with sidewalk

* green

* simple pbr to third party

* include simple pbr

* develop shader

* simplify pssm

* add debug code

* use pssm shadow

* 3 splits by default

* use version 330

* delet argu

* shadow filtering

* :small shadow buffer

* 2 split

* light

* fix camera

* merge pbr render and render

* line thickness

* shader auto

* color

* format

* fixbug

* add anotation

* t navi

* fix bug

LICENSE.txt

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright [2020] [MetaDrive Authors]
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

metadrive/component/block/base_block.py

@@ -3,12 +3,13 @@
 import time
 from abc import ABC
 from typing import Dict
+from panda3d.core import LVecBase4, TextureStage
 from metadrive.utils.utils import create_rectangle_from_midpoints
 import numpy as np
 from panda3d.bullet import BulletBoxShape
 from panda3d.bullet import BulletConvexHullShape
 from panda3d.bullet import BulletTriangleMeshShape, BulletTriangleMesh
-from panda3d.core import LPoint3f
+from panda3d.core import LPoint3f, Material
 from panda3d.core import Vec3, LQuaternionf, RigidBodyCombiner, \
     SamplerState, NodePath, Texture
 from panda3d.core import Vec4
@@ -71,7 +72,7 @@ def __init__(
         self.crosswalks = {}
         self.sidewalks = {}
 
-        if self.render and not self.use_render_pipeline:
+        if self.render:
             # side
             self.side_texture = self.loader.loadTexture(AssetLoader.file_path("textures", "sidewalk", "color.png"))
             # self.side_texture.set_format(Texture.F_srgb)
@@ -83,11 +84,6 @@ def __init__(
             # self.side_normal.set_format(Texture.F_srgb)
             self.side_normal.setWrapU(Texture.WM_repeat)
             self.side_normal.setWrapV(Texture.WM_repeat)
-
-            self.sidewalk = self.loader.loadModel(AssetLoader.file_path("models", "box.bam"))
-            self.sidewalk.setTwoSided(False)
-            self.sidewalk.setTexture(self.side_texture)
-
             self.line_seg = make_polygon_model([(-0.5, 0.5), (-0.5, -0.5), (0.5, -0.5), (0.5, 0.5)], 0)
 
     def _sample_topology(self) -> bool:
@@ -159,7 +155,7 @@ def attach_to_world(self, parent_node_path: NodePath, physics_world: PhysicsWorl
         """
         if not self.is_attached():
             for obj in self._block_objects:
-                obj.attach_to_world(self.engine.pbr_worldNP, physics_world)
+                obj.attach_to_world(self.engine.worldNP, physics_world)
         super(BaseBlock, self).attach_to_world(parent_node_path, physics_world)
 
     def destruct_block(self, physics_world: PhysicsWorld):
@@ -242,7 +238,15 @@ def _create_in_world(self, skip=False):
 
         self.sidewalk_node_path.flattenStrong()
         self.sidewalk_node_path.node().collect()
-        self.sidewalk_node_path.hide(CamMask.ScreenshotCam | CamMask.Shadow)
+        if self.render:
+            # np.setShaderInput("p3d_TextureBaseColor", self.side_texture)
+            # np.setShaderInput("p3d_TextureNormal", self.side_normal)
+            self.sidewalk_node_path.setTexture(self.side_texture)
+            ts = TextureStage("normal")
+            ts.setMode(TextureStage.MNormal)
+            self.sidewalk_node_path.setTexture(ts, self.side_normal)
+            material = Material()
+            self.sidewalk_node_path.setMaterial(material, True)
 
         self.crosswalk_node_path.flattenStrong()
         self.crosswalk_node_path.node().collect()
@@ -367,7 +371,7 @@ def _construct_sidewalk(self):
         """
         Construct the sidewalk with collision shape
         """
-        if self.engine is None or (self.engine.global_config["show_sidewalk"] and not self.engine.use_render_pipeline):
+        if self.engine is None or (self.engine.global_config["show_sidewalk"]):
             for sidewalk_id, sidewalk in self.sidewalks.items():
                 if len(sidewalk["polygon"]) == 0:
                     continue
@@ -382,9 +386,6 @@ def _construct_sidewalk(self):
                     np = make_polygon_model(polygon, height)
                     np.reparentTo(self.sidewalk_node_path)
                     np.setPos(0, 0, z_pos)
-                    if self.render:
-                        np.setTexture(self.side_texture)
-                    # np.setTexture(self.ts_normal, self.side_normal)
 
                     body_node = BaseRigidBodyNode(None, MetaDriveType.BOUNDARY_SIDEWALK)
                     body_node.setKinematic(False)

metadrive/component/map/base_map.py

@@ -153,9 +153,9 @@ def _show_coordinates(self, lanes):
             lateral_end = lane.position(0, 2)
 
             long_end = long_start + lane.heading_at(0) * 4
-            np_y = self.engine._draw_line_3d(Vec3(*long_start, 0), Vec3(*long_end, 0), color=[0, 1, 0, 1], thickness=2)
+            np_y = self.engine._draw_line_3d(Vec3(*long_start, 0), Vec3(*long_end, 0), color=[0, 1, 0, 1], thickness=3)
             np_x = self.engine._draw_line_3d(
-                Vec3(*lateral_start, 0), Vec3(*lateral_end, 0), color=[1, 0, 0, 1], thickness=2
+                Vec3(*lateral_start, 0), Vec3(*lateral_end, 0), color=[1, 0, 0, 1], thickness=3
             )
             np_x.reparentTo(self.lane_coordinates_debug_node)
             np_y.reparentTo(self.lane_coordinates_debug_node)

metadrive/component/navigation_module/base_navigation.py

@@ -23,6 +23,7 @@ class BaseNavigation:
     CKPT_UPDATE_RANGE = 5
     FORCE_CALCULATE = False
     LINE_TO_DEST_HEIGHT = 0.6
+    MARK_HEIGHT = 1.2
 
     def __init__(
         self,
@@ -55,7 +56,12 @@ def __init__(
         self._show_navi_info = (
             self.engine.mode == RENDER_MODE_ONSCREEN and not self.engine.global_config["debug_physics_world"]
         )
-        self.origin = NodePath("navigation_sign") if self._show_navi_info else None
+        if self._show_navi_info:
+            self.origin = NodePath("navigation_sign")
+            self.origin.clearShader()
+            self.origin.setShaderAuto()
+        else:
+            self.origin = None
         if panda_color is not None:
             assert len(panda_color) == 3 and 0 <= panda_color[0] <= 1
             self.navi_mark_color = tuple(panda_color)

metadrive/component/navigation_module/edge_network_navigation.py

@@ -80,8 +80,7 @@ def set_route(self, current_lane_index: str, destination: str):
             ref_lane = self.final_lane
             later_middle = (float(self.get_current_lane_num()) / 2 - 0.5) * self.get_current_lane_width()
             check_point = ref_lane.position(ref_lane.length, later_middle)
-            # self._dest_node_path.setPos(check_point[0], -check_point[1], 1.8)
-            self._dest_node_path.setPos(panda_vector(check_point[0], check_point[1], 1.8))
+            self._dest_node_path.setPos(panda_vector(check_point[0], check_point[1], self.MARK_HEIGHT))
 
     def update_localization(self, ego_vehicle):
         position = ego_vehicle.position
@@ -115,7 +114,7 @@ def update_localization(self, ego_vehicle):
 
         if self._show_navi_info:  # Whether to visualize little boxes in the scene denoting the checkpoints
             pos_of_goal = checkpoint
-            self._goal_node_path.setPos(panda_vector(pos_of_goal[0], pos_of_goal[1], 1.8))
+            self._goal_node_path.setPos(panda_vector(pos_of_goal[0], pos_of_goal[1], self.MARK_HEIGHT))
             self._goal_node_path.setH(self._goal_node_path.getH() + 3)
             self.navi_arrow_dir = [lanes_heading1, lanes_heading2]
             dest_pos = self._dest_node_path.getPos()

metadrive/component/navigation_module/node_network_navigation.py

@@ -125,7 +125,7 @@ def set_route(self, current_lane_index: str, destination: str):
             ref_lane = final_lanes[0]
             later_middle = (float(self.get_current_lane_num()) / 2 - 0.5) * self.get_current_lane_width()
             check_point = ref_lane.position(ref_lane.length, later_middle)
-            self._dest_node_path.setPos(panda_vector(check_point[0], check_point[1], 1.8))
+            self._dest_node_path.setPos(panda_vector(check_point[0], check_point[1], self.MARK_HEIGHT))
 
     def update_localization(self, ego_vehicle):
         position = ego_vehicle.position
@@ -170,7 +170,7 @@ def update_localization(self, ego_vehicle):
         if self._show_navi_info:
             # Whether to visualize little boxes in the scene denoting the checkpoints
             pos_of_goal = checkpoint
-            self._goal_node_path.setPos(panda_vector(pos_of_goal[0], pos_of_goal[1], 1.8))
+            self._goal_node_path.setPos(panda_vector(pos_of_goal[0], pos_of_goal[1], self.MARK_HEIGHT))
             self._goal_node_path.setH(self._goal_node_path.getH() + 3)
             self.navi_arrow_dir = [lanes_heading1, lanes_heading2]
             dest_pos = self._dest_node_path.getPos()

metadrive/component/navigation_module/trajectory_navigation.py

@@ -128,13 +128,13 @@ def update_localization(self, ego_vehicle):
 
         # target_road_1 is the road segment the vehicle is driving on.
         self._navi_info.fill(0.0)
-        for k, ckpt in enumerate(ckpts):
+        for k, ckpt in enumerate(ckpts[1:]):
             start = k * self.CHECK_POINT_INFO_DIM
             end = (k + 1) * self.CHECK_POINT_INFO_DIM
             self._navi_info[start:end], lanes_heading = self._get_info_for_checkpoint(ckpt, ego_vehicle)
             if self._show_navi_info and self._ckpt_vis_models is not None:
                 pos_of_goal = ckpt
-                self._ckpt_vis_models[k].setPos(panda_vector(pos_of_goal[0], pos_of_goal[1], 1.8))
+                self._ckpt_vis_models[k].setPos(panda_vector(pos_of_goal[0], pos_of_goal[1], self.MARK_HEIGHT))
                 self._ckpt_vis_models[k].setH(self._goal_node_path.getH() + 3)
 
         self._navi_info[end] = clip((lat / self.engine.global_config["max_lateral_dist"] + 1) / 2, 0.0, 1.0)

metadrive/component/road_network/base_road_network.py

@@ -74,7 +74,7 @@ def show_bounding_box(self, engine, color=None):
         points = [(x, y) for x in bound_box[:2] for y in bound_box[2:]]
         for k, p in enumerate(points[:-1]):
             for p_ in points[k + 1:]:
-                line = engine._draw_line_3d((*p, 2), (*p_, 2), color, 2)
+                line = engine._draw_line_3d((*p, 2), (*p_, 2), color, 3)
                 line.reparentTo(engine.render)
                 self._lines_np.append(line)
 

metadrive/component/sensors/base_camera.py

@@ -96,7 +96,6 @@ def get_image(self, base_object):
         """
         Borrow the camera to get observations
         """
-        self.sync_light(base_object)
         self.origin.reparentTo(base_object.origin)
         img = self.get_rgb_array_cpu()
         self.track(self.attached_object)
@@ -107,7 +106,6 @@ def save_image(self, base_object, name="debug.png"):
         cv2.imwrite(name, img)
 
     def perceive(self, base_object, clip=True) -> np.ndarray:
-        self.sync_light(base_object)
         self.track(base_object)
         if self.enable_cuda:
             assert self.cuda_rendered_result is not None
@@ -144,10 +142,6 @@ def track(self, base_object):
             self.attached_object = base_object
             self.origin.reparentTo(base_object.origin)
 
-    def sync_light(self, obj):
-        if self.engine.world_light is not None:
-            self.engine.world_light.set_pos(obj.position)
-
     def __del__(self):
         if self.enable_cuda:
             self.unregister()

metadrive/component/sensors/depth_camera.py

@@ -54,14 +54,14 @@ def __init__(self, width, height, engine, *, cuda=False):
             self.GROUND_MODEL.show(CamMask.DepthCam)
             self.GROUND.generate()
 
-    def track(self, base_object):
-        if self.VIEW_GROUND:
-            pos = base_object.origin.getPos()
-            self.GROUND_MODEL.setPos(pos[0], pos[1], self.GROUND_HEIGHT - 0.5)
-            self.GROUND_MODEL.setH(base_object.origin.getH())
-            # self.GROUND_MODEL.setP(-base_object.origin.getR())
-            # self.GROUND_MODEL.setR(-base_object.origin.getR())
-        return super(DepthCamera, self).track(base_object)
+            def sync_pos(task):
+                if self.attached_object:
+                    pos = self.attached_object.origin.getPos()
+                    self.GROUND_MODEL.setPos(pos[0], pos[1], self.GROUND_HEIGHT - 0.5)
+                    self.GROUND_MODEL.setH(self.attached_object.origin.getH())
+                return task.cont
+
+            self.engine.taskMgr.add(sync_pos)
 
     def _setup_effect(self):
         """

metadrive/component/sensors/rgb_camera.py

@@ -1,6 +1,6 @@
 import panda3d.core as p3d
 from direct.filter.FilterManager import FilterManager
-from simplepbr import _load_shader_str
+from metadrive.third_party.simplepbr import _load_shader_str
 from panda3d.core import FrameBufferProperties
 from metadrive.component.sensors.base_camera import BaseCamera
 from metadrive.constants import CamMask

metadrive/component/sensors/semantic_camera.py

@@ -52,14 +52,14 @@ def __init__(self, width, height, engine, *, cuda=False):
             self.GROUND_MODEL.setTag("type", Semantics.ROAD.label)
             self.GROUND.generate()
 
-    def track(self, base_object):
-        if self.VIEW_GROUND and base_object is not None:
-            pos = base_object.origin.getPos()
-            self.GROUND_MODEL.setPos(pos[0], pos[1], self.GROUND_HEIGHT - 0.5)
-            self.GROUND_MODEL.setH(base_object.origin.getH())
-            # self.GROUND_MODEL.setP(-base_object.origin.getR())
-            # self.GROUND_MODEL.setR(-base_object.origin.getR())
-        return super(SemanticCamera, self).track(base_object)
+            def sync_pos(task):
+                if self.attached_object:
+                    pos = self.attached_object.origin.getPos()
+                    self.GROUND_MODEL.setPos(pos[0], pos[1], self.GROUND_HEIGHT - 0.5)
+                    self.GROUND_MODEL.setH(self.attached_object.origin.getH())
+                return task.cont
+
+            self.engine.taskMgr.add(sync_pos)
 
     def _setup_effect(self):
         """

metadrive/component/static_object/base_static_object.py

@@ -33,9 +33,9 @@ def show_coordinates(self):
         self.coordinates_debug_np = NodePath("debug coordinate")
         self.coordinates_debug_np.hide(CamMask.AllOn)
         self.coordinates_debug_np.show(CamMask.MainCam)
-        x = self.engine._draw_line_3d([0, 0, height], [2, 0, height], [1, 0, 0, 1], 1)
-        y = self.engine._draw_line_3d([0, 0, height], [0, 1, height], [1, 0, 0, 1], 1)
-        z = self.engine._draw_line_3d([0, 0, height], [0, 0, height + 0.5], [0, 0, 1, 1], 2)
+        x = self.engine._draw_line_3d([0, 0, height], [2, 0, height], [1, 0, 0, 1], 3)
+        y = self.engine._draw_line_3d([0, 0, height], [0, 1, height], [0, 1, 0, 1], 3)
+        z = self.engine._draw_line_3d([0, 0, height], [0, 0, height + 0.5], [0, 0, 1, 1], 3)
         x.reparentTo(self.coordinates_debug_np)
         y.reparentTo(self.coordinates_debug_np)
         z.reparentTo(self.coordinates_debug_np)

metadrive/component/traffic_participants/base_traffic_participant.py

@@ -75,9 +75,9 @@ def show_coordinates(self):
         self.coordinates_debug_np = NodePath("debug coordinate")
         self.coordinates_debug_np.hide(CamMask.AllOn)
         self.coordinates_debug_np.show(CamMask.MainCam)
-        x = self.engine._draw_line_3d([0, 0, height], [1, 0, height], [1, 0, 0, 1], 1)
-        y = self.engine._draw_line_3d([0, 0, height], [0, 0.5, height], [1, 0, 0, 1], 1)
-        z = self.engine._draw_line_3d([0, 0, height], [0, 0, height + 0.5], [0, 0, 1, 1], 1)
+        x = self.engine._draw_line_3d([0, 0, height], [1, 0, height], [1, 0, 0, 1], 3)
+        y = self.engine._draw_line_3d([0, 0, height], [0, 0.5, height], [0, 1, 0, 1], 3)
+        z = self.engine._draw_line_3d([0, 0, height], [0, 0, height + 0.5], [0, 0, 1, 1], 3)
         x.reparentTo(self.coordinates_debug_np)
         y.reparentTo(self.coordinates_debug_np)
         z.reparentTo(self.coordinates_debug_np)

metadrive/component/vehicle/base_vehicle.py

@@ -1,10 +1,8 @@
 import math
-from metadrive.constants import CamMask
-from metadrive.engine.logger import get_logger
 import os
 from collections import deque
 from typing import Union, Optional
-from metadrive.constants import Semantics
+
 import numpy as np
 import seaborn as sns
 from panda3d._rplight import RPSpotLight
@@ -17,13 +15,14 @@
 from metadrive.component.lane.circular_lane import CircularLane
 from metadrive.component.lane.point_lane import PointLane
 from metadrive.component.lane.straight_lane import StraightLane
-from metadrive.component.pg_space import VehicleParameterSpace, ParameterSpace
-from metadrive.component.road_network.node_road_network import NodeRoadNetwork
-from metadrive.component.navigation_module.edge_network_navigation import EdgeNetworkNavigation
 from metadrive.component.navigation_module.node_network_navigation import NodeNetworkNavigation
+from metadrive.component.pg_space import VehicleParameterSpace, ParameterSpace
+from metadrive.constants import CamMask
 from metadrive.constants import MetaDriveType, CollisionGroup
+from metadrive.constants import Semantics
 from metadrive.engine.asset_loader import AssetLoader
 from metadrive.engine.engine_utils import get_engine, engine_initialized
+from metadrive.engine.logger import get_logger
 from metadrive.engine.physics_node import BaseRigidBodyNode
 from metadrive.utils import Config, safe_clip_for_small_array
 from metadrive.utils.math import get_vertical_vector, norm, clip
@@ -1030,9 +1029,9 @@ def show_coordinates(self):
         self.coordinates_debug_np.hide(CamMask.AllOn)
         self.coordinates_debug_np.show(CamMask.MainCam)
         # 90 degrees offset
-        x = self.engine._draw_line_3d([0, 0, height], [0, 2, height], [1, 1, 1, 1], 2)
-        y = self.engine._draw_line_3d([0, 0, height], [-1, 0, height], [1, 1, 1, 1], 2)
-        z = self.engine._draw_line_3d([0, 0, height], [0, 0, height + 0.5], [1, 1, 1, 1], 2)
+        x = self.engine._draw_line_3d([0, 0, height], [0, 2, height], [1, 1, 1, 1], 3)
+        y = self.engine._draw_line_3d([0, 0, height], [-1, 0, height], [1, 1, 1, 1], 3)
+        z = self.engine._draw_line_3d([0, 0, height], [0, 0, height + 0.5], [1, 1, 1, 1], 3)
         x.reparentTo(self.coordinates_debug_np)
         y.reparentTo(self.coordinates_debug_np)
         z.reparentTo(self.coordinates_debug_np)

metadrive/engine/base_engine.py

@@ -120,13 +120,10 @@ def has_policy(self, object_id, policy_cls=None):
                 self._object_policies[object_id], policy_cls
             ) else False
 
-    def spawn_object(
-        self, object_class, pbr_model=True, force_spawn=False, auto_fill_random_seed=True, record=True, **kwargs
-    ):
+    def spawn_object(self, object_class, force_spawn=False, auto_fill_random_seed=True, record=True, **kwargs):
         """
         Call this func to spawn one object
         :param object_class: object class
-        :param pbr_model: if the visualization model is pbr model
         :param force_spawn: spawn a new object instead of fetching from _dying_objects list
         :param auto_fill_random_seed: whether to set random seed using purely random integer
         :param record: record the spawn information
@@ -157,7 +154,7 @@ def spawn_object(
             print("FK!~")
             exit()
 
-        obj.attach_to_world(self.pbr_worldNP if pbr_model else self.worldNP, self.physics_world)
+        obj.attach_to_world(self.worldNP, self.physics_world)
         return obj
 
     def _pick_color(self, id):
@@ -392,11 +389,6 @@ def process_memory():
         center_p = [0, 0]
         self.terrain.reset(center_p)
 
-        # init shadow if required
-        if hasattr(self, "pssm") and self.pssm.buffer is None and self.global_config["show_terrain"] \
-                and not self.global_config["debug_physics_world"]:
-            self.pssm.init()
-
         # move skybox
         if self.sky_box is not None:
             self.sky_box.set_position(center_p)
@@ -682,7 +674,7 @@ def _object_clean_check(self):
         assert len(filtered) == 0, "Physics Bodies should be cleaned before manager.reset() is called. " \
                                    "Uncleared bodies: {}".format(filtered)
 
-        children = self.pbr_worldNP.getChildren() + self.worldNP.getChildren()
+        children = self.worldNP.getChildren()
         assert len(children) == 0, "NodePath are not cleaned thoroughly. Remaining NodePath: {}".format(children)
 
     def update_manager(self, manager_name: str, manager: BaseManager, destroy_previous_manager=True):

metadrive/engine/core/draw.py

@@ -9,6 +9,8 @@ class ColorLineNodePath(LineNodePath):
     def __init__(self, parent=None, thickness=1.0):
         super(ColorLineNodePath, self).__init__(parent, name=None, thickness=thickness, colorVec=VBase4(1))
         self.hide(CamMask.Shadow)
+        self.clearShader()
+        self.setShaderAuto()
 
     def draw_lines(self, lineList, colorList=None):
         """