Add new section to doc about image observation (#46)

* renderin config

* depth cam

* depth image bug?

* add example

* add script

* fix bug

* update doc

* fix typo

documentation/source/config_system.rst

@@ -51,7 +51,7 @@ We describe all optional map config as follows:
         - :code:`config`: XXX
         - :code:`lane_width` (float = 3.5): the width of each lane. This will be overwritten if :code:`random_lane_width = True`.
         - :code:`lane_num` (int = 3): number of lanes in each road. This will be overwritten if :code:`random_lane_num = True`.
-        - :code:`exit_length` (float = 50): TODO(LQY)
+        - :code:`exit_length` (float = 50): more than one exit whose length is *exit_length* are contained in some blocks like roundabout
 
 
 We also provide a shortcut to specify the map:

documentation/source/install.rst

@@ -20,7 +20,6 @@ We recommend to use the command following to install::
 
 To check whether MetaDrive is successfully installed, please run::
 
-    # Go to a folder where no sub-folder calls metadrive
     python -m metadrive.examples.profile_metadrive
 
 
@@ -44,9 +43,10 @@ If the above observation is not enough for your RL algorithms and you wish to us
 
 If your machine already has a screen, please try the following script to verify whether the Panda3D window can successfully pop up.
 
-    # Go to a folder where no sub-folder calls metadrive
     python -m metadrive.examples.drive_in_single_agent_env
 
+.. note:: Please do not run the above command in the folder that has a sub-folder called :code:`./metadrive`.
+
 If the screen successfully shows up, then you can move on to :ref:`use_native_rendering` and skip this section.
 
 

documentation/source/observation.rst

@@ -86,6 +86,25 @@ Use First-view Images in Training
 
 MetaDrive supports visuomotor tasks by turning on the rendering during the training.
 
-.. note:: This part is working in progress.
+.. image:: figs/rgb_obs.png
+   :width: 600
+   :align: center
+
+.. image:: figs/depth_obs.png
+   :width: 600
+   :align: center
+
+Special config needs to activate camera observation.
+
+1. In env config **offline_render** needs to be **True** to tell MetaDrive retrieving images from camera
+2. In vehicle_config (under env config), set **image_source** to **rgb_camera** or **depth_camera** to get sensory data
+3. The image size will be determined by the camera parameters. For example, **rgb_camera=(200, 88)** means that the image is in 200 x 88
+
+There is a demo script using camera output via::
+
+    python -m metadrive.examples.drive_in_single_agent_env --observation rgb_camera
+
+
+
 
 

metadrive/component/vehicle/base_vehicle.py

@@ -558,7 +558,7 @@ def _add_wheel(self, pos: Vec3, radius: float, front: bool, left):
         if self.render:
             model = 'right_tire_front.gltf' if front else 'right_tire_back.gltf'
             model_path = AssetLoader.file_path("models", self.path[0], model)
-            wheel_model = self.loader.loadModel(AssetLoader.file_path(model_path))
+            wheel_model = self.loader.loadModel(model_path)
             wheel_model.reparentTo(wheel_np)
             wheel_model.set_scale(1 if left else -1)
         wheel = self.system.create_wheel()

metadrive/component/vehicle_module/depth_camera.py

@@ -28,7 +28,7 @@ def __init__(self):
         cam.lookAt(0, 2.4, 1.3)
         lens.setFov(60)
         lens.setAspectRatio(2.0)
-        if get_engine().mode == RENDER_MODE_NONE or not AssetLoader.initialized():
+        if get_engine().mode == RENDER_MODE_NONE or not AssetLoader.initialized() or type(self)._singleton.init_num > 0:
             return
         # add shader for it
         if get_global_config()["headless_machine_render"]:
@@ -47,18 +47,13 @@ def __init__(self):
 
         if self.VIEW_GROUND:
             self.GROUND = GeoMipTerrain("mySimpleTerrain")
-
             self.GROUND.setHeightfield(AssetLoader.file_path("textures", "height_map.png"))
             # terrain.setBruteforce(True)
             # # Since the terrain is a texture, shader will not calculate the depth information, we add a moving terrain
             # # model to enable the depth information of terrain
             self.GROUND_MODEL = self.GROUND.getRoot()
             self.GROUND_MODEL.setPos(-128, 0, self.GROUND_HEIGHT)
+            self.GROUND_MODEL.reparentTo(type(self)._singleton.origin)
             self.GROUND_MODEL.hide(CamMask.AllOn)
             self.GROUND_MODEL.show(CamMask.DepthCam)
             self.GROUND.generate()
-
-    def set_virtual_ground(self, chassis_np: Vec3):
-        self.GROUND_MODEL.setPos(-128, 0, self.GROUND)
-        self.GROUND_MODEL.setP(-chassis_np.getP())
-        self.GROUND_MODEL.setH(chassis_np.getH())

metadrive/engine/core/onscreen_message.py

@@ -66,12 +66,12 @@ def render(self, data: Optional[Union[dict, str]] = None):
             if v[0] == self.frame:
                 # It was updated this frame (key equals value):
                 # isNew = " is"
-                isNew = "="
+                isNew = ":"
             else:
                 # This data is not for the current
                 # frame (key roughly equals value):
                 # isNew = "was"
-                isNew = "~"
+                isNew = ":"
             value = v[1]
             if type(value) == float:
                 value = "% 10.4f" % (value, )

metadrive/engine/core/sky_box.py

@@ -20,7 +20,7 @@ def __init__(self, pure_background: bool = False):
             return
         skybox = self.loader.loadModel(AssetLoader.file_path("models", "skybox.bam"))
 
-        skybox.hide(CamMask.MiniMap | CamMask.RgbCam | CamMask.Shadow | CamMask.ScreenshotCam)
+        skybox.hide(CamMask.MiniMap | CamMask.Shadow | CamMask.ScreenshotCam)
         skybox.set_scale(20000)
 
         skybox_texture = self.loader.loadTexture(AssetLoader.file_path("textures", "skybox.jpg"))

metadrive/examples/drive_in_single_agent_env.py

@@ -8,22 +8,27 @@
 import random
 
 from metadrive import MetaDriveEnv
+import argparse
 
 if __name__ == "__main__":
-    env = MetaDriveEnv(
-        dict(
-            # controller="joystick",
-            use_render=True,
-            manual_control=True,
-            traffic_density=0.1,
-            environment_num=100,
-            random_agent_model=True,
-            random_lane_width=True,
-            random_lane_num=True,
-            map=7,  # seven block
-            start_seed=random.randint(0, 1000)
-        )
+    config = dict(
+        # controller="joystick",
+        use_render=True,
+        manual_control=True,
+        traffic_density=0.1,
+        environment_num=100,
+        random_agent_model=True,
+        random_lane_width=True,
+        random_lane_num=True,
+        map=7,  # seven block
+        start_seed=random.randint(0, 1000)
     )
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--observation", type=str, default="lidar", choices=["lidar", "rgb_camera"])
+    args = parser.parse_args()
+    if args.observation == "rgb_camera":
+        config.update(dict(offscreen_render=True))
+    env = MetaDriveEnv(config)
     env.reset()
     for i in range(1, 1000000000):
         o, r, d, info = env.step([0, 0])

metadrive/tests/vis_env/vis_metadrive_env.py

@@ -6,14 +6,15 @@
     env = MetaDriveEnv(
         {
             "environment_num": 100,
-            "traffic_density": .0,
+            "traffic_density": .1,
             "traffic_mode": "trigger",
             "start_seed": 22,
             # "_disable_detector_mask":True,
             # "debug_physics_world": True,
             "global_light": True,
             # "debug_static_world":True,
             "cull_scene": False,
+            "offscreen_render": True,
             # "controller": "joystick",
             "manual_control": True,
             "use_render": True,
@@ -30,13 +31,14 @@
             # },
             "pstats": True,
             # "discrete_action": True,
-            "map": "SSS",
+            "map": "SSSSSS",
             "random_traffic": False,
             "random_lane_width": True,
             "random_agent_model": True,
             "driving_reward": 1.0,
             "vehicle_config": {
                 "enable_reverse": True,
+                "image_source": "depth_camera",
                 # "random_color": True
                 # "show_lidar": True,
                 # "spawn_lane_index":("1r1_0_", "1r1_1_", 0),
@@ -57,6 +59,10 @@
 
     for s in range(1, 100000):
         o, r, d, info = env.step(env.action_space.sample())
+        if s == 50:
+            env.vehicle.image_sensors["rgb_camera"].save_image(env.vehicle, name="rgb.png")
+            env.vehicle.image_sensors["depth_camera"].save_image(env.vehicle, name="depth.png")
+        print(o)
         # info["fuel"] = env.vehicle.energy_consumption
         # env.render(
         #     text={

metadrive/tests/vis_functionality/vis_depth_cam_ground.py

@@ -4,7 +4,7 @@
 if __name__ == "__main__":
 
     def get_image(env):
-        env.vehicle.image_sensors[env.vehicle.config["image_source"]].save_image()
+        env.vehicle.image_sensors[env.vehicle.config["image_source"]].save_image(env.vehicle, "debug.jpg")
         env.engine.screenshot()
 
     env = MetaDriveEnv(
@@ -36,7 +36,7 @@ def get_image(env):
             # for i in range(ImageObservation.STACK_SIZE):
             #     ObservationType.show_gray_scale_array(o["image"][:, :, i])
             env.render()
-        if d:
-            print("Reset")
-            env.reset()
+        # if d:
+        #     print("Reset")
+        #     env.reset()
     env.close()

metadrive/tests/vis_functionality/vis_depth_cam_no_ground.py

@@ -22,7 +22,7 @@
         }
     )
     env.reset()
-    env.engine.accept("m", env.vehicle.image_sensors["depth_camera"].save_image, extraArgs=[env.vehicle])
+    env.engine.accept("m", env.vehicle.image_sensors["depth_camera"].save_image, extraArgs=[env.vehicle, "debug.jpg"])
 
     for i in range(1, 100000):
         o, r, d, info = env.step([0, 1])