Fix doc & fix WorldLight (#492)

* fix doc * fix * fix doc * fix * fix typo * fix light bug * aligh camera API * format
metadriverse · Sep 6, 2023 · e2f1dc5 · e2f1dc5
1 parent 9b79a07
commit e2f1dc5
Show file tree

Hide file tree

Showing 13 changed files with 151 additions and 61 deletions.
diff --git a/.readthedocs.yaml b/.readthedocs.yaml
@@ -0,0 +1,35 @@
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+    # You can also specify other tool versions:
+    # nodejs: "20"
+    # rust: "1.70"
+    # golang: "1.20"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: documentation/source/conf.py
+  # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
+  # builder: "dirhtml"
+  # Fail on all warnings to avoid broken references
+  # fail_on_warning: true
+
+# Optionally build your docs in additional formats such as PDF and ePub
+# formats:
+#    - pdf
+#    - epub
+
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+    install:
+    - requirements: documentation//requirements.txt
diff --git a/README.md b/README.md
@@ -46,14 +46,12 @@ Install MetaDrive via:
 git clone https://github.com/metadriverse/metadrive.git
 cd metadrive
 pip install -e .
-
 ```
 
 or
 
 ```bash
 pip install metadrive-simulator
-
 ```
 *Note that the program is tested on both Linux and Windows. Some control and display issues in MacOS wait to be solved*
 
@@ -67,7 +65,7 @@ python -m metadrive.examples.profile_metadrive
 *Note that please do not run the above command in a folder that has a sub-folder called `./metadrive`.*
 
 ## 🚕 Examples
-We provide examples to demonstrate features and basic usages of MetaDrive after the local installation.
+We provide [examples](https://github.com/metadriverse/metadrive/tree/main/metadrive/examples) to demonstrate features and basic usages of MetaDrive after the local installation.
 Or you can run some examples directly in Colab. [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/metadriverse/metadrive/blob/main/metadrive/examples/Basic_MetaDrive_Usages.ipynb) 
 
 ### Single Agent Environment

diff --git a/documentation/requirements.txt b/documentation/requirements.txt
@@ -0,0 +1,59 @@
+#
+# This file is autogenerated by pip-compile with python 3.10
+# To update, run:
+#
+#    pip-compile docs/requirements.in
+#
+alabaster==0.7.12
+    # via sphinx
+babel==2.10.3
+    # via sphinx
+certifi==2022.6.15
+    # via requests
+charset-normalizer==2.1.0
+    # via requests
+docutils==0.17.1
+    # via
+    #   sphinx
+    #   sphinx-rtd-theme
+idna==3.3
+    # via requests
+imagesize==1.4.1
+    # via sphinx
+jinja2==3.1.2
+    # via sphinx
+markupsafe==2.1.1
+    # via jinja2
+packaging==21.3
+    # via sphinx
+pygments==2.12.0
+    # via sphinx
+pyparsing==3.0.9
+    # via packaging
+pytz==2022.1
+    # via babel
+requests==2.28.1
+    # via sphinx
+snowballstemmer==2.2.0
+    # via sphinx
+sphinx==5.0.2
+    # via
+    #   -r docs/requirements.in
+    #   sphinx-rtd-theme
+sphinx-rtd-theme==1.0.0
+    # via -r docs/requirements.in
+sphinxcontrib-applehelp==1.0.2
+    # via sphinx
+sphinxcontrib-devhelp==1.0.2
+    # via sphinx
+sphinxcontrib-htmlhelp==2.0.0
+    # via sphinx
+sphinxcontrib-jsmath==1.0.1
+    # via sphinx
+sphinxcontrib-qthelp==1.0.3
+    # via sphinx
+sphinxcontrib-serializinghtml==1.1.5
+    # via sphinx
+urllib3==1.26.9
+    # via requests
+sphinx_rtd_theme
diff --git a/documentation/source/install.rst b/documentation/source/install.rst
@@ -69,8 +69,10 @@ Installation:
 #. Install Torch: ``conda install pytorch==1.12.1 torchvision==0.13.1 torchaudio==0.12.1 cudatoolkit=11.6 -c pytorch -c conda-forge``
 #. Install CuPy: ``pip install cupy-cuda11x``
 #. Install Cuda-Python: ``conda install -c nvidia cuda-python``
-#. For verifying your installation, cd ``metadrive/examples`` and run ``python verify_image_on_cuda.py``
+#. For verifying your installation, cd ``metadrive/examples`` and run ``python verify_image_observation.py --cuda``
 
 
-After running the script, if no error messages, then congratulations! It works. you can also use ``python verify_image_on_cuda.py --render`` to visualize the image observations.
-Besides, a ``--native`` flag can be added to benchmark the original image collection pipeline as a comparison.  
+After running the script, if no error messages, then congratulations! It works. you can also use ``python verify_image_observation.py --cuda --render`` to visualize the image observations.
+Besides, removing ``--cuda`` flag enables benchmarking the original image collection pipeline as a comparison.
+And ``--camera`` argument is for choosing sensors from [``rgb``, ``depth``, ``semantic``, ``main`` (default)].
+
diff --git a/metadrive/component/sensors/base_camera.py b/metadrive/component/sensors/base_camera.py
@@ -1,4 +1,5 @@
 import numpy as np
+import cv2
 from metadrive.component.sensors import BaseSensor
 from metadrive.utils.cuda import check_cudart_err
 
@@ -41,7 +42,7 @@ def __init__(self, engine, setup_pbr=False, need_cuda=False):
             # Too large height or width will cause corruption in Mac.
             self.logger.warning(
                 "You may using too large buffer! The height is {}, and width is {}. "
-                "It may lower the sample efficiency! Considering reduce buffer size or using cuda image by"
+                "It may lower the sample efficiency! Consider reducing buffer size or use cuda image by"
                 " set [image_on_cuda=True].".format(height, width)
             )
         self.cuda_graphics_resource = None
@@ -88,16 +89,18 @@ def get_image(self, base_object):
         """
         Borrow the camera to get observations
         """
+        self.sync_light(base_object)
         self.origin.reparentTo(base_object.origin)
-        ret = super(BaseCamera, self).get_image()
+        img = self.get_rgb_array_cpu()
         self.track(self.attached_object)
-        return ret
+        return img
 
     def save_image(self, base_object, name="debug.png"):
         img = self.get_image(base_object)
-        img.write(name)
+        cv2.imwrite(name, img)
 
     def perceive(self, base_object, clip=True) -> np.ndarray:
+        self.sync_light(base_object)
         self.track(base_object)
         if self.enable_cuda:
             assert self.cuda_rendered_result is not None
@@ -134,6 +137,10 @@ def track(self, base_object):
             self.attached_object = base_object
             self.origin.reparentTo(base_object.origin)
 
+    def sync_light(self, obj):
+        if self.engine.world_light is not None:
+            self.engine.world_light.set_pos(obj.position)
+
     def __del__(self):
         if self.enable_cuda:
             self.unregister()

diff --git a/metadrive/component/sensors/depth_camera.py b/metadrive/component/sensors/depth_camera.py
@@ -1,5 +1,6 @@
 import cv2
-from panda3d.core import Shader, RenderState, ShaderAttrib, GeoMipTerrain, PNMImage, Texture
+from panda3d.core import Shader, RenderState, ShaderAttrib, GeoMipTerrain, PNMImage, Texture, LightAttrib, \
+    TextureAttrib, ColorAttrib
 
 from metadrive.component.sensors.base_camera import BaseCamera
 from metadrive.constants import CamMask
@@ -43,7 +44,12 @@ def __init__(self, width, height, engine, *, cuda=False):
             vert_path = AssetLoader.file_path("shaders", "depth_cam.vert.glsl")
             frag_path = AssetLoader.file_path("shaders", "depth_cam.frag.glsl")
         custom_shader = Shader.load(Shader.SL_GLSL, vertex=vert_path, fragment=frag_path)
-        cam.node().setInitialState(RenderState.make(ShaderAttrib.make(custom_shader, 1)))
+        cam.node().setInitialState(
+            RenderState.make(
+                LightAttrib.makeAllOff(), TextureAttrib.makeOff(), ColorAttrib.makeOff(),
+                ShaderAttrib.make(custom_shader, 1)
+            )
+        )
 
         if self.VIEW_GROUND:
             ground = PNMImage(513, 513, 4)
@@ -70,13 +76,3 @@ def track(self, base_object):
             # self.GROUND_MODEL.setP(-base_object.origin.getR())
             # self.GROUND_MODEL.setR(-base_object.origin.getR())
         return super(DepthCamera, self).track(base_object)
-
-    def get_image(self, base_object):
-        self.origin.reparentTo(base_object.origin)
-        img = super(DepthCamera, self).get_rgb_array_cpu()
-        self.track(self.attached_object)
-        return img
-
-    def save_image(self, base_object, name="debug.png"):
-        img = self.get_image(base_object)
-        cv2.imwrite(name, img)
diff --git a/metadrive/component/sensors/semantic_camera.py b/metadrive/component/sensors/semantic_camera.py
@@ -73,13 +73,3 @@ def track(self, base_object):
             # self.GROUND_MODEL.setP(-base_object.origin.getR())
             # self.GROUND_MODEL.setR(-base_object.origin.getR())
         return super(SemanticCamera, self).track(base_object)
-
-    def get_image(self, base_object):
-        self.origin.reparentTo(base_object.origin)
-        img = super(SemanticCamera, self).get_rgb_array_cpu()
-        self.track(self.attached_object)
-        return img
-
-    def save_image(self, base_object, name="debug.png"):
-        img = self.get_image(base_object)
-        cv2.imwrite(name, img)
diff --git a/metadrive/engine/core/image_buffer.py b/metadrive/engine/core/image_buffer.py
@@ -99,21 +99,6 @@ def __init__(
 
         self.logger.debug("Load Image Buffer: {}".format(self.__class__.__name__))
 
-    def get_image(self):
-        """
-        Bugs here! when use offscreen mode, thus the front cam obs is not from front cam now
-        """
-        img = PNMImage()
-        self.buffer.getDisplayRegions()[1].getScreenshot(img)
-        return img
-
-    def save_image(self, name="debug.png"):
-        """
-        for debug use
-        """
-        img = self.get_image()
-        img.write(name)
-
     def get_rgb_array_cpu(self):
         origin_img = self.buffer.getDisplayRegion(1).getScreenshot()
         img = np.frombuffer(origin_img.getRamImage().getData(), dtype=np.uint8)

diff --git a/metadrive/engine/core/light.py b/metadrive/engine/core/light.py
@@ -49,13 +49,12 @@ def __init__(self, config):
         self._node_path_list.append(self.ambient_np)
 
     def step(self, pos):
+        raise DeprecationWarning("Use light.step_pos instead")
+
+    def set_pos(self, pos):
         # if not self.global_light:
         self.direction_np.setPos(pos[0] - 100, pos[1] + 100, 120)
         self.direction_np.lookAt(pos[0], pos[1], 0)
 
     def reset(self, random_seed=None, *args, **kwargs):
-        if self.direction_np is not None and not self.global_light:
-            self.direction_np.setHpr(-90, -120, 0)
-            self.direction_np.setY(20)
-            self.direction_np.setX(0)
-            self.direction_np.setZ(50)
+        raise DeprecationWarning("Use light.step_pos instead")
diff --git a/metadrive/engine/core/main_camera.py b/metadrive/engine/core/main_camera.py
@@ -227,7 +227,7 @@ def _chase_task(self, vehicle, task):
             )
 
         if self.world_light is not None:
-            self.world_light.step(current_pos)
+            self.world_light.set_pos(current_pos)
 
         # Don't use reparentTo()
         # pos = vehicle.convert_to_world_coordinates([0.8, 0.], vehicle.position)
@@ -345,6 +345,8 @@ def _top_down_task(self, task):
             self.camera_x += 1.0
 
         self.camera.setPos(self.camera_x, self.camera_y, self.top_down_camera_height)
+        if self.world_light is not None:
+            self.world_light.set_pos([self.camera_x, self.camera_y])
         if self.engine.global_config["show_coordinates"]:
             self.engine.set_coordinates_indicator_pos([self.camera_x, self.camera_y])
         self.camera.lookAt(self.camera_x, self.camera_y, 0)

diff --git a/metadrive/examples/verify_image_on_cuda.py → ...rive/examples/verify_image_observation.py b/metadrive/examples/verify_image_on_cuda.py → ...rive/examples/verify_image_observation.py
@@ -4,12 +4,27 @@
 import cv2
 import torch
 from torch.utils.dlpack import from_dlpack
-
+from metadrive.component.sensors.depth_camera import DepthCamera
+from metadrive.component.sensors.semantic_camera import SemanticCamera
+from metadrive.component.sensors.rgb_camera import RGBCamera
 from metadrive import MetaDriveEnv
 from metadrive.policy.idm_policy import IDMPolicy
 
 
-def _test_rgb_camera_as_obs(render=False, image_on_cuda=True, debug=False):
+def _test_rgb_camera_as_obs(render=False, image_on_cuda=True, debug=False, camera="main"):
+    res = (800, 600)
+    mapping = {
+        "depth": {
+            "depth_camera": (DepthCamera, *res)
+        },
+        "rgb": {
+            "rgb_camera": (RGBCamera, *res)
+        },
+        "semantic": {
+            "semantic_camera": (SemanticCamera, *res)
+        }
+    }
+
     env = MetaDriveEnv(
         dict(
             num_scenarios=1,
@@ -19,7 +34,8 @@ def _test_rgb_camera_as_obs(render=False, image_on_cuda=True, debug=False):
             image_observation=True,
             image_on_cuda=True if image_on_cuda else False,
             use_render=False,
-            vehicle_config=dict(image_source="main_camera"),
+            vehicle_config=dict(image_source="{}_camera".format(camera)),
+            sensors=mapping[camera] if camera != "main" else {},
             show_interface=False,
             show_logo=False,
             show_fps=False,
@@ -65,8 +81,9 @@ def _test_rgb_camera_as_obs(render=False, image_on_cuda=True, debug=False):
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("--render", action="store_true")
-    parser.add_argument("--native", action="store_true")
+    parser.add_argument("--cuda", action="store_true")
     parser.add_argument("--debug", action="store_true")
+    parser.add_argument("--camera", default="main", choices=["main", "rgb", "depth", "semantic"])
     args = parser.parse_args()
-    _test_rgb_camera_as_obs(args.render, image_on_cuda=not args.native, debug=args.debug)
+    _test_rgb_camera_as_obs(args.render, image_on_cuda=args.cuda, debug=args.debug, camera=args.camera)
     print("Test Successful !! The FPS should go beyond 400 FPS if you are running on GPUs better than RTX 3060.")
diff --git a/metadrive/tests/test_installation.py b/metadrive/tests/test_installation.py
@@ -19,7 +19,7 @@ def capture_headless_image(cuda, image_source="main_camera"):
             traffic_density=0.1,
             image_observation=True,
             sensors={
-                "rgb_camera": (RGBCamera, 84, 84),
+                "rgb_camera": (RGBCamera, 512, 512),
                 "depth_camera": (DepthCamera, 512, 512)
             },
             interface_panel=[],

diff --git a/metadrive/tests/vis_functionality/vis_rgb_cam.py b/metadrive/tests/vis_functionality/vis_rgb_cam.py
@@ -12,11 +12,11 @@
             # "debug_panda3d": True,
             "vehicle_config": dict(image_source="rgb_camera"),
             "sensors": {
-                "rgb_camera": (RGBCamera, 84, 84)
+                "rgb_camera": (RGBCamera, 521, 512)
             },
             "interface_panel": ["dashboard", "rgb_camera"],
             "manual_control": False,
-            "use_render": True,
+            "use_render": False,
             "image_observation": True,  # it is a switch telling metadrive to use rgb as observation
             "rgb_clip": True,  # clip rgb to range(0,1) instead of (0, 255)
             # "pstats": True,