From b68e4cd52223db359001b70159d819f960bfe7f5 Mon Sep 17 00:00:00 2001 From: Leonard Bruns Date: Thu, 23 Nov 2023 11:06:51 +0100 Subject: [PATCH 01/25] Start nuScenes example --- examples/python/nuscenes/README.md | 25 ++++++++++++++ examples/python/nuscenes/main.py | 52 ++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 examples/python/nuscenes/README.md create mode 100644 examples/python/nuscenes/main.py diff --git a/examples/python/nuscenes/README.md b/examples/python/nuscenes/README.md new file mode 100644 index 000000000000..6fbc701b0a55 --- /dev/null +++ b/examples/python/nuscenes/README.md @@ -0,0 +1,25 @@ +--- +title: nuScenes +python: https://github.com/rerun-io/rerun/blob/latest/examples/python/nuscenes/main.py +tags: [lidar, 3D, 2D, object-detection, pinhole-camera] +description: "Visualize the nuScenes dataset, which contains lidar data and color images, and labeled bounding boxes." +thumbnail: https://static.rerun.io/arkit_scenes/fb9ec9e8d965369d39d51b17fc7fc5bae6be10cc/480w.png +thumbnail_dimensions: [480, 243] +demo: true +--- + + + + + + + ARKit Scenes screenshot + + +This example visualizes the [nuScenes dataset](https://www.nuscenes.org/) using Rerun. The dataset +contains lidar data, color images, and labeled bounding boxes. + +```bash +pip install -r examples/python/nuscenes/requirements.txt +python examples/python/nuscenes/main.py +``` diff --git a/examples/python/nuscenes/main.py b/examples/python/nuscenes/main.py new file mode 100644 index 000000000000..e0edc23cf576 --- /dev/null +++ b/examples/python/nuscenes/main.py @@ -0,0 +1,52 @@ +import argparse +import pathlib + +import rerun as rr + + +def download_minisplit(root_dir: pathlib.Path) -> None: + """ + Download nuScenes minisplit. + + Adopted from https://colab.research.google.com/github/nutonomy/nuscenes-devkit/blob/master/python-sdk/tutorials/nuscenes_tutorial.ipynb + """ + # TODO(leo) implement this + pass + + +def ensure_scene_available(root_dir: pathlib.Path, scene_id: str) -> None: + """ + Ensure that the specified scene is available. + + Downloads minisplit into root_dir if scene_id is part of it and root_dir is empty. + + Raises ValueError if scene is not available and cannot be downloaded. + """ + MINISPLIT_IDS = [] + + +def log_nuscenes() -> None: + pass + + +def main() -> None: + parser = argparse.ArgumentParser(description="Visualizes the nuScenes dataset using the Rerun SDK.") + parser.add_argument( + "--root_dir", + type=pathlib.Path, + default="dataset", + help="Root directory of nuScenes dataset", + ) + parser.add_argument("--scene_id", type=str, help="Scene id to visualize") + rr.script_add_args(parser) + args = parser.parse_args() + + rr.script_setup(args, "rerun_example_nuscenes") + recording_path = ensure_scene_available(args.root_dir, args.scene_id) + log_nuscenes(recording_path, args.include_highres) + + rr.script_teardown(args) + + +if __name__ == "__main__": + main() From 45fce68a0b841dca838b1fd0958d128fef22eb22 Mon Sep 17 00:00:00 2001 From: Leonard Bruns Date: Thu, 23 Nov 2023 15:57:11 +0100 Subject: [PATCH 02/25] Add logging of radar and lidar point clouds --- examples/python/nuscenes/main.py | 73 ++++++++++++++++++++--- examples/python/nuscenes/requirements.txt | 2 + 2 files changed, 67 insertions(+), 8 deletions(-) create mode 100644 examples/python/nuscenes/requirements.txt diff --git a/examples/python/nuscenes/main.py b/examples/python/nuscenes/main.py index e0edc23cf576..8bb537f78471 100644 --- a/examples/python/nuscenes/main.py +++ b/examples/python/nuscenes/main.py @@ -1,6 +1,8 @@ import argparse import pathlib +from nuscenes import nuscenes + import rerun as rr @@ -14,19 +16,70 @@ def download_minisplit(root_dir: pathlib.Path) -> None: pass -def ensure_scene_available(root_dir: pathlib.Path, scene_id: str) -> None: +def ensure_scene_available(root_dir: pathlib.Path, dataset_version: str, scene_name: str) -> None: """ Ensure that the specified scene is available. - Downloads minisplit into root_dir if scene_id is part of it and root_dir is empty. + Downloads minisplit into root_dir if scene_name is part of it and root_dir is empty. Raises ValueError if scene is not available and cannot be downloaded. """ - MINISPLIT_IDS = [] + nusc = nuscenes.NuScenes(version=dataset_version, dataroot=root_dir, verbose=True) + # TODO handle this + # try: + # except: + # if dataset_version == "v1.0-mini": + # # TODO handle download case + # nusc = nuscenes.NuScenes(version=dataset_version, dataroot=root_dir, verbose=True) + scene_names = [s["name"] for s in nusc.scene] + if scene_name not in scene_names: + raise ValueError(f"{scene_name=} not found in dataset") -def log_nuscenes() -> None: - pass + +def log_nuscenes(root_dir: pathlib.Path, dataset_version: str, scene_name: str) -> None: + nusc = nuscenes.NuScenes(version=dataset_version, dataroot=root_dir, verbose=True) + + scene = next(s for s in nusc.scene if s["name"] == scene_name) + + # TODO log sensor configuration + + # breakpoint() + current_sample = nusc.get('sample', scene["first_sample_token"]) + start_timestamp = current_sample["timestamp"] + while True: + # log data + for data_name, data_token in current_sample["data"].items(): + while True: + meta_data = nusc.get("sample_data", data_token) + rr.set_time_seconds("timestamp", (meta_data["timestamp"] - start_timestamp) * 1e-6) + data_file_path = root_dir / meta_data["filename"] + + if meta_data["sensor_modality"] == "lidar": + # log lidar points + print(meta_data["ego_pose_token"]) + pointcloud = nuscenes.LidarPointCloud.from_file(str(data_file_path)) + points = pointcloud.points[:3].T # shape after transposing: (num_points, 3) + rr.log(f"ego_vehicle/{data_name}", rr.Points3D(points)) + elif meta_data["sensor_modality"] == "radar": + pointcloud = nuscenes.RadarPointCloud.from_file(str(data_file_path)) + points = pointcloud.points[:3].T # shape after transposing: (num_points, 3) + rr.log(f"ego_vehicle/{data_name}", rr.Points3D(points)) + elif meta_data["sensor_modality"] == "camera": + # TODO log images + pass + + data_token = meta_data["next"] + if data_token == "" or nusc.get("sample_data", data_token)["is_key_frame"]: + break + + # TODO optional log annotations + + + if current_sample["next"] == "": + break + + current_sample = nusc.get("sample", current_sample["next"]) def main() -> None: @@ -37,13 +90,17 @@ def main() -> None: default="dataset", help="Root directory of nuScenes dataset", ) - parser.add_argument("--scene_id", type=str, help="Scene id to visualize") + parser.add_argument( + "--scene_name", type=str, default="scene-0061", help="Scene name to visualize (typically of form 'scene-xxxx')" + ) + parser.add_argument("--dataset_version", type=str, default="v1.0-mini", help="Scene id to visualize") rr.script_add_args(parser) args = parser.parse_args() rr.script_setup(args, "rerun_example_nuscenes") - recording_path = ensure_scene_available(args.root_dir, args.scene_id) - log_nuscenes(recording_path, args.include_highres) + + ensure_scene_available(args.root_dir, args.dataset_version, args.scene_name) + log_nuscenes(args.root_dir, args.dataset_version, args.scene_name) rr.script_teardown(args) diff --git a/examples/python/nuscenes/requirements.txt b/examples/python/nuscenes/requirements.txt new file mode 100644 index 000000000000..ae8aa662c8e5 --- /dev/null +++ b/examples/python/nuscenes/requirements.txt @@ -0,0 +1,2 @@ +nuscenes-devkit +rerun-sdk From b2e7ebb966a17398f61c5b2227d54e47ff880249 Mon Sep 17 00:00:00 2001 From: Leonard Bruns Date: Thu, 23 Nov 2023 16:38:39 +0100 Subject: [PATCH 03/25] Log ego pose of vehicle --- examples/python/nuscenes/main.py | 24 ++++++++++++++++++----- examples/python/nuscenes/requirements.txt | 1 + 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/examples/python/nuscenes/main.py b/examples/python/nuscenes/main.py index 8bb537f78471..f3d3ad8fe967 100644 --- a/examples/python/nuscenes/main.py +++ b/examples/python/nuscenes/main.py @@ -1,5 +1,6 @@ import argparse import pathlib +import numpy as np from nuscenes import nuscenes @@ -44,8 +45,9 @@ def log_nuscenes(root_dir: pathlib.Path, dataset_version: str, scene_name: str) # TODO log sensor configuration - # breakpoint() - current_sample = nusc.get('sample', scene["first_sample_token"]) + rr.log("world", rr.ViewCoordinates.RIGHT_HAND_Z_UP, timeless=True) + + current_sample = nusc.get("sample", scene["first_sample_token"]) start_timestamp = current_sample["timestamp"] while True: # log data @@ -53,6 +55,19 @@ def log_nuscenes(root_dir: pathlib.Path, dataset_version: str, scene_name: str) while True: meta_data = nusc.get("sample_data", data_token) rr.set_time_seconds("timestamp", (meta_data["timestamp"] - start_timestamp) * 1e-6) + + ego_pose = nusc.get("ego_pose", meta_data["ego_pose_token"]) + + rotation_xyzw = np.roll(ego_pose["rotation"], shift=-1) + rr.log( + "world/ego_vehicle", + rr.Transform3D( + translation=ego_pose["translation"], + rotation=rr.Quaternion(xyzw=rotation_xyzw), + from_parent=False, + ), + ) + data_file_path = root_dir / meta_data["filename"] if meta_data["sensor_modality"] == "lidar": @@ -60,11 +75,11 @@ def log_nuscenes(root_dir: pathlib.Path, dataset_version: str, scene_name: str) print(meta_data["ego_pose_token"]) pointcloud = nuscenes.LidarPointCloud.from_file(str(data_file_path)) points = pointcloud.points[:3].T # shape after transposing: (num_points, 3) - rr.log(f"ego_vehicle/{data_name}", rr.Points3D(points)) + rr.log(f"world/ego_vehicle/{data_name}", rr.Points3D(points)) elif meta_data["sensor_modality"] == "radar": pointcloud = nuscenes.RadarPointCloud.from_file(str(data_file_path)) points = pointcloud.points[:3].T # shape after transposing: (num_points, 3) - rr.log(f"ego_vehicle/{data_name}", rr.Points3D(points)) + rr.log(f"world/ego_vehicle/{data_name}", rr.Points3D(points)) elif meta_data["sensor_modality"] == "camera": # TODO log images pass @@ -75,7 +90,6 @@ def log_nuscenes(root_dir: pathlib.Path, dataset_version: str, scene_name: str) # TODO optional log annotations - if current_sample["next"] == "": break diff --git a/examples/python/nuscenes/requirements.txt b/examples/python/nuscenes/requirements.txt index ae8aa662c8e5..724e8a33081b 100644 --- a/examples/python/nuscenes/requirements.txt +++ b/examples/python/nuscenes/requirements.txt @@ -1,2 +1,3 @@ +numpy nuscenes-devkit rerun-sdk From fd0b89a0033f2dce0cd599ed9cb83a84f5ca85d8 Mon Sep 17 00:00:00 2001 From: Leonard Bruns Date: Thu, 23 Nov 2023 17:06:46 +0100 Subject: [PATCH 04/25] Log sensor configuration --- examples/python/nuscenes/main.py | 47 ++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/examples/python/nuscenes/main.py b/examples/python/nuscenes/main.py index f3d3ad8fe967..24c59830766b 100644 --- a/examples/python/nuscenes/main.py +++ b/examples/python/nuscenes/main.py @@ -43,7 +43,8 @@ def log_nuscenes(root_dir: pathlib.Path, dataset_version: str, scene_name: str) scene = next(s for s in nusc.scene if s["name"] == scene_name) - # TODO log sensor configuration + # each sensor only has to be logged once, maintain set of already logged sensors + logged_sensor_tokens = set() rr.log("world", rr.ViewCoordinates.RIGHT_HAND_Z_UP, timeless=True) @@ -51,22 +52,27 @@ def log_nuscenes(root_dir: pathlib.Path, dataset_version: str, scene_name: str) start_timestamp = current_sample["timestamp"] while True: # log data - for data_name, data_token in current_sample["data"].items(): + for sensor_name, data_token in current_sample["data"].items(): while True: meta_data = nusc.get("sample_data", data_token) - rr.set_time_seconds("timestamp", (meta_data["timestamp"] - start_timestamp) * 1e-6) + sensor_token = meta_data["calibrated_sensor_token"] + if sensor_token not in logged_sensor_tokens: + calibrated_sensor = nusc.get("calibrated_sensor", sensor_token) + rotation_xyzw = np.roll(calibrated_sensor["rotation"], shift=-1) + rr.log( + f"world/ego_vehicle/{sensor_name}", + rr.Transform3D( + translation=calibrated_sensor["translation"], + rotation=rr.Quaternion(xyzw=rotation_xyzw), + from_parent=False, + ), + timeless=True, + ) + logged_sensor_tokens.add(sensor_token) + breakpoint() - ego_pose = nusc.get("ego_pose", meta_data["ego_pose_token"]) + rr.set_time_seconds("timestamp", (meta_data["timestamp"] - start_timestamp) * 1e-6) - rotation_xyzw = np.roll(ego_pose["rotation"], shift=-1) - rr.log( - "world/ego_vehicle", - rr.Transform3D( - translation=ego_pose["translation"], - rotation=rr.Quaternion(xyzw=rotation_xyzw), - from_parent=False, - ), - ) data_file_path = root_dir / meta_data["filename"] @@ -75,11 +81,22 @@ def log_nuscenes(root_dir: pathlib.Path, dataset_version: str, scene_name: str) print(meta_data["ego_pose_token"]) pointcloud = nuscenes.LidarPointCloud.from_file(str(data_file_path)) points = pointcloud.points[:3].T # shape after transposing: (num_points, 3) - rr.log(f"world/ego_vehicle/{data_name}", rr.Points3D(points)) + rr.log(f"world/ego_vehicle/{sensor_name}", rr.Points3D(points)) + + ego_pose = nusc.get("ego_pose", meta_data["ego_pose_token"]) + rotation_xyzw = np.roll(ego_pose["rotation"], shift=-1) + rr.log( + "world/ego_vehicle", + rr.Transform3D( + translation=ego_pose["translation"], + rotation=rr.Quaternion(xyzw=rotation_xyzw), + from_parent=False, + ), + ) elif meta_data["sensor_modality"] == "radar": pointcloud = nuscenes.RadarPointCloud.from_file(str(data_file_path)) points = pointcloud.points[:3].T # shape after transposing: (num_points, 3) - rr.log(f"world/ego_vehicle/{data_name}", rr.Points3D(points)) + rr.log(f"world/ego_vehicle/{sensor_name}", rr.Points3D(points)) elif meta_data["sensor_modality"] == "camera": # TODO log images pass From 944b416c9c1e143de3ee0993e28008e5e5a798ec Mon Sep 17 00:00:00 2001 From: Leonard Bruns Date: Thu, 23 Nov 2023 17:41:23 +0100 Subject: [PATCH 05/25] Log images --- examples/python/nuscenes/main.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/examples/python/nuscenes/main.py b/examples/python/nuscenes/main.py index 24c59830766b..a6913291e77f 100644 --- a/examples/python/nuscenes/main.py +++ b/examples/python/nuscenes/main.py @@ -69,11 +69,19 @@ def log_nuscenes(root_dir: pathlib.Path, dataset_version: str, scene_name: str) timeless=True, ) logged_sensor_tokens.add(sensor_token) - breakpoint() + if len(calibrated_sensor["camera_intrinsic"]) != 0: + rr.log( + f"world/ego_vehicle/{sensor_name}", + rr.Pinhole( + image_from_camera=calibrated_sensor["camera_intrinsic"], + width=meta_data["width"], + height=meta_data["height"], + ), + timeless=True, + ) rr.set_time_seconds("timestamp", (meta_data["timestamp"] - start_timestamp) * 1e-6) - data_file_path = root_dir / meta_data["filename"] if meta_data["sensor_modality"] == "lidar": @@ -98,8 +106,7 @@ def log_nuscenes(root_dir: pathlib.Path, dataset_version: str, scene_name: str) points = pointcloud.points[:3].T # shape after transposing: (num_points, 3) rr.log(f"world/ego_vehicle/{sensor_name}", rr.Points3D(points)) elif meta_data["sensor_modality"] == "camera": - # TODO log images - pass + rr.log(f"world/ego_vehicle/{sensor_name}", rr.ImageEncoded(path=data_file_path)) data_token = meta_data["next"] if data_token == "" or nusc.get("sample_data", data_token)["is_key_frame"]: From 34891f34a89ba2812e7c500e52d3f54f44bce5e0 Mon Sep 17 00:00:00 2001 From: Leonard Bruns Date: Thu, 23 Nov 2023 18:14:07 +0100 Subject: [PATCH 06/25] Clean up code --- examples/python/nuscenes/main.py | 153 ++++++++++++++++++------------- 1 file changed, 87 insertions(+), 66 deletions(-) diff --git a/examples/python/nuscenes/main.py b/examples/python/nuscenes/main.py index a6913291e77f..04da8d145e1c 100644 --- a/examples/python/nuscenes/main.py +++ b/examples/python/nuscenes/main.py @@ -1,8 +1,10 @@ import argparse import pathlib -import numpy as np +from typing import Any, Union +import numpy as np from nuscenes import nuscenes +import numbers import rerun as rr @@ -44,82 +46,98 @@ def log_nuscenes(root_dir: pathlib.Path, dataset_version: str, scene_name: str) scene = next(s for s in nusc.scene if s["name"] == scene_name) # each sensor only has to be logged once, maintain set of already logged sensors - logged_sensor_tokens = set() + logged_sensor_tokens: set[str] = set() rr.log("world", rr.ViewCoordinates.RIGHT_HAND_Z_UP, timeless=True) current_sample = nusc.get("sample", scene["first_sample_token"]) start_timestamp = current_sample["timestamp"] while True: - # log data - for sensor_name, data_token in current_sample["data"].items(): - while True: - meta_data = nusc.get("sample_data", data_token) - sensor_token = meta_data["calibrated_sensor_token"] - if sensor_token not in logged_sensor_tokens: - calibrated_sensor = nusc.get("calibrated_sensor", sensor_token) - rotation_xyzw = np.roll(calibrated_sensor["rotation"], shift=-1) - rr.log( - f"world/ego_vehicle/{sensor_name}", - rr.Transform3D( - translation=calibrated_sensor["translation"], - rotation=rr.Quaternion(xyzw=rotation_xyzw), - from_parent=False, - ), - timeless=True, - ) - logged_sensor_tokens.add(sensor_token) - if len(calibrated_sensor["camera_intrinsic"]) != 0: - rr.log( - f"world/ego_vehicle/{sensor_name}", - rr.Pinhole( - image_from_camera=calibrated_sensor["camera_intrinsic"], - width=meta_data["width"], - height=meta_data["height"], - ), - timeless=True, - ) - - rr.set_time_seconds("timestamp", (meta_data["timestamp"] - start_timestamp) * 1e-6) - - data_file_path = root_dir / meta_data["filename"] - - if meta_data["sensor_modality"] == "lidar": - # log lidar points - print(meta_data["ego_pose_token"]) - pointcloud = nuscenes.LidarPointCloud.from_file(str(data_file_path)) - points = pointcloud.points[:3].T # shape after transposing: (num_points, 3) - rr.log(f"world/ego_vehicle/{sensor_name}", rr.Points3D(points)) - - ego_pose = nusc.get("ego_pose", meta_data["ego_pose_token"]) - rotation_xyzw = np.roll(ego_pose["rotation"], shift=-1) - rr.log( - "world/ego_vehicle", - rr.Transform3D( - translation=ego_pose["translation"], - rotation=rr.Quaternion(xyzw=rotation_xyzw), - from_parent=False, - ), - ) - elif meta_data["sensor_modality"] == "radar": - pointcloud = nuscenes.RadarPointCloud.from_file(str(data_file_path)) - points = pointcloud.points[:3].T # shape after transposing: (num_points, 3) - rr.log(f"world/ego_vehicle/{sensor_name}", rr.Points3D(points)) - elif meta_data["sensor_modality"] == "camera": - rr.log(f"world/ego_vehicle/{sensor_name}", rr.ImageEncoded(path=data_file_path)) - - data_token = meta_data["next"] - if data_token == "" or nusc.get("sample_data", data_token)["is_key_frame"]: - break - - # TODO optional log annotations + log_nuscenes_sample(current_sample, nusc, logged_sensor_tokens, start_timestamp) if current_sample["next"] == "": break - current_sample = nusc.get("sample", current_sample["next"]) +def log_nuscenes_sample( + sample: dict[str, Any], nusc: nuscenes.NuScenes, logged_sensor_tokens: set[str], start_timestamp: numbers.Number +) -> None: + # each sample is a keyframe with annotations + + # log data + for sensor_name, sample_data_token in sample["data"].items(): + # TODO optional log annotations + while True: + sample_data = nusc.get("sample_data", sample_data_token) + log_nuscenes_sample_data(sample_data, nusc, logged_sensor_tokens, start_timestamp) + + sample_data_token = sample_data["next"] + if sample_data_token == "" or nusc.get("sample_data", sample_data_token)["is_key_frame"]: + break + + +def log_nuscenes_sample_data( + sample_data: dict[str, Any], + nusc: nuscenes.NuScenes, + logged_sensor_tokens: set[str], + start_timestamp: numbers.Number, +): + sensor_name = sample_data["channel"] + calibrated_sensor_token = sample_data["calibrated_sensor_token"] + if calibrated_sensor_token not in logged_sensor_tokens: + calibrated_sensor = nusc.get("calibrated_sensor", calibrated_sensor_token) + rotation_xyzw = np.roll(calibrated_sensor["rotation"], shift=-1) + rr.log( + f"world/ego_vehicle/{sensor_name}", + rr.Transform3D( + translation=calibrated_sensor["translation"], + rotation=rr.Quaternion(xyzw=rotation_xyzw), + from_parent=False, + ), + timeless=True, + ) + logged_sensor_tokens.add(calibrated_sensor_token) + if len(calibrated_sensor["camera_intrinsic"]) != 0: + rr.log( + f"world/ego_vehicle/{sensor_name}", + rr.Pinhole( + image_from_camera=calibrated_sensor["camera_intrinsic"], + width=sample_data["width"], + height=sample_data["height"], + ), + timeless=True, + ) + + rr.set_time_seconds("timestamp", (sample_data["timestamp"] - start_timestamp) * 1e-6) + + data_file_path = nusc.dataroot / sample_data["filename"] + + if sample_data["sensor_modality"] == "lidar": + # TODO optional color for lidar points (depth-like color map) + # log lidar points + pointcloud = nuscenes.LidarPointCloud.from_file(str(data_file_path)) + points = pointcloud.points[:3].T # shape after transposing: (num_points, 3) + rr.log(f"world/ego_vehicle/{sensor_name}", rr.Points3D(points)) + + ego_pose = nusc.get("ego_pose", sample_data["ego_pose_token"]) + rotation_xyzw = np.roll(ego_pose["rotation"], shift=-1) + rr.log( + "world/ego_vehicle", + rr.Transform3D( + translation=ego_pose["translation"], + rotation=rr.Quaternion(xyzw=rotation_xyzw), + from_parent=False, + ), + ) + elif sample_data["sensor_modality"] == "radar": + pointcloud = nuscenes.RadarPointCloud.from_file(str(data_file_path)) + points = pointcloud.points[:3].T # shape after transposing: (num_points, 3) + rr.log(f"world/ego_vehicle/{sensor_name}", rr.Points3D(points)) + elif sample_data["sensor_modality"] == "camera": + rr.log(f"world/ego_vehicle/{sensor_name}", rr.ImageEncoded(path=data_file_path)) + + def main() -> None: parser = argparse.ArgumentParser(description="Visualizes the nuScenes dataset using the Rerun SDK.") parser.add_argument( @@ -129,7 +147,10 @@ def main() -> None: help="Root directory of nuScenes dataset", ) parser.add_argument( - "--scene_name", type=str, default="scene-0061", help="Scene name to visualize (typically of form 'scene-xxxx')" + "--scene_name", + type=str, + default="scene-0061", + help="Scene name to visualize (typically of form 'scene-xxxx')", ) parser.add_argument("--dataset_version", type=str, default="v1.0-mini", help="Scene id to visualize") rr.script_add_args(parser) From 41dfdabb4c093405852d32c983b7c7b6c3c04d43 Mon Sep 17 00:00:00 2001 From: Leonard Bruns Date: Thu, 23 Nov 2023 18:48:11 +0100 Subject: [PATCH 07/25] Add jet colormap --- examples/python/nuscenes/main.py | 12 +++++++++--- examples/python/nuscenes/requirements.txt | 1 + 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/examples/python/nuscenes/main.py b/examples/python/nuscenes/main.py index 04da8d145e1c..ff3360981fb2 100644 --- a/examples/python/nuscenes/main.py +++ b/examples/python/nuscenes/main.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 import argparse import pathlib from typing import Any, Union @@ -5,9 +6,14 @@ import numpy as np from nuscenes import nuscenes import numbers +import matplotlib import rerun as rr +cmap = matplotlib.colormaps["turbo_r"] +norm = matplotlib.colors.Normalize( + vmin=3.0, vmax=50.0, +) def download_minisplit(root_dir: pathlib.Path) -> None: """ @@ -64,8 +70,6 @@ def log_nuscenes_sample( sample: dict[str, Any], nusc: nuscenes.NuScenes, logged_sensor_tokens: set[str], start_timestamp: numbers.Number ) -> None: # each sample is a keyframe with annotations - - # log data for sensor_name, sample_data_token in sample["data"].items(): # TODO optional log annotations while True: @@ -118,7 +122,9 @@ def log_nuscenes_sample_data( # log lidar points pointcloud = nuscenes.LidarPointCloud.from_file(str(data_file_path)) points = pointcloud.points[:3].T # shape after transposing: (num_points, 3) - rr.log(f"world/ego_vehicle/{sensor_name}", rr.Points3D(points)) + point_distances = np.linalg.norm(points, axis=1) + point_colors = cmap(norm(point_distances)) + rr.log(f"world/ego_vehicle/{sensor_name}", rr.Points3D(points, colors=point_colors)) ego_pose = nusc.get("ego_pose", sample_data["ego_pose_token"]) rotation_xyzw = np.roll(ego_pose["rotation"], shift=-1) diff --git a/examples/python/nuscenes/requirements.txt b/examples/python/nuscenes/requirements.txt index 724e8a33081b..9ddcccf2ce52 100644 --- a/examples/python/nuscenes/requirements.txt +++ b/examples/python/nuscenes/requirements.txt @@ -1,3 +1,4 @@ +matplotlib numpy nuscenes-devkit rerun-sdk From b53bb2282ef5fba8bd523f3a8b0e8e0af9d22b4e Mon Sep 17 00:00:00 2001 From: Leonard Bruns Date: Thu, 23 Nov 2023 18:50:58 +0100 Subject: [PATCH 08/25] Remove old TODO --- examples/python/nuscenes/main.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/python/nuscenes/main.py b/examples/python/nuscenes/main.py index ff3360981fb2..d83ffcc82390 100644 --- a/examples/python/nuscenes/main.py +++ b/examples/python/nuscenes/main.py @@ -118,8 +118,6 @@ def log_nuscenes_sample_data( data_file_path = nusc.dataroot / sample_data["filename"] if sample_data["sensor_modality"] == "lidar": - # TODO optional color for lidar points (depth-like color map) - # log lidar points pointcloud = nuscenes.LidarPointCloud.from_file(str(data_file_path)) points = pointcloud.points[:3].T # shape after transposing: (num_points, 3) point_distances = np.linalg.norm(points, axis=1) From ac13954fda770f8b5784a42c18ed3f15a1269f0b Mon Sep 17 00:00:00 2001 From: Leonard Bruns Date: Thu, 30 Nov 2023 10:33:31 +0100 Subject: [PATCH 09/25] Implement ensure_scene_available --- examples/python/nuscenes/main.py | 34 ++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) mode change 100644 => 100755 examples/python/nuscenes/main.py diff --git a/examples/python/nuscenes/main.py b/examples/python/nuscenes/main.py old mode 100644 new mode 100755 index d83ffcc82390..a10c205eddeb --- a/examples/python/nuscenes/main.py +++ b/examples/python/nuscenes/main.py @@ -12,15 +12,18 @@ cmap = matplotlib.colormaps["turbo_r"] norm = matplotlib.colors.Normalize( - vmin=3.0, vmax=50.0, + vmin=3.0, + vmax=50.0, ) + def download_minisplit(root_dir: pathlib.Path) -> None: """ Download nuScenes minisplit. Adopted from https://colab.research.google.com/github/nutonomy/nuscenes-devkit/blob/master/python-sdk/tutorials/nuscenes_tutorial.ipynb """ + print("Downloading nuScenes minisplit...") # TODO(leo) implement this pass @@ -33,13 +36,28 @@ def ensure_scene_available(root_dir: pathlib.Path, dataset_version: str, scene_n Raises ValueError if scene is not available and cannot be downloaded. """ - nusc = nuscenes.NuScenes(version=dataset_version, dataroot=root_dir, verbose=True) - # TODO handle this - # try: - # except: - # if dataset_version == "v1.0-mini": - # # TODO handle download case - # nusc = nuscenes.NuScenes(version=dataset_version, dataroot=root_dir, verbose=True) + try: + nusc = nuscenes.NuScenes(version=dataset_version, dataroot=root_dir, verbose=True) + except AssertionError as e: + MINISPLIT_SCENES = [ + "scene-0061", + "scene-0103", + "scene-0553", + "scene-0655", + "scene-0757", + "scene-0796", + "scene-0916", + "scene-1077", + "scene-1094", + "scene-1100", + ] + + if dataset_version == "v1.0-mini" and scene_name in MINISPLIT_SCENES: + download_minisplit(root_dir) + nusc = nuscenes.NuScenes(version=dataset_version, dataroot=root_dir, verbose=True) + else: + print(f"Could not find dataset at {root_dir} and could not automatically download specified scene.") + exit() scene_names = [s["name"] for s in nusc.scene] if scene_name not in scene_names: From 7d798c6baa5cfaf7345e0625275d61160fa56442 Mon Sep 17 00:00:00 2001 From: Leonard Bruns Date: Thu, 30 Nov 2023 11:43:18 +0100 Subject: [PATCH 10/25] Add automatic downloading --- examples/python/nuscenes/main.py | 47 ++++++++++---------------------- 1 file changed, 14 insertions(+), 33 deletions(-) diff --git a/examples/python/nuscenes/main.py b/examples/python/nuscenes/main.py index a10c205eddeb..4fc8291ed268 100755 --- a/examples/python/nuscenes/main.py +++ b/examples/python/nuscenes/main.py @@ -1,14 +1,16 @@ #!/usr/bin/env python3 +from __future__ import annotations + import argparse +import numbers import pathlib -from typing import Any, Union +from typing import Any -import numpy as np -from nuscenes import nuscenes -import numbers import matplotlib - +import numpy as np import rerun as rr +from download_dataset import MINISPLIT_SCENES, download_minisplit +from nuscenes import nuscenes cmap = matplotlib.colormaps["turbo_r"] norm = matplotlib.colors.Normalize( @@ -17,17 +19,6 @@ ) -def download_minisplit(root_dir: pathlib.Path) -> None: - """ - Download nuScenes minisplit. - - Adopted from https://colab.research.google.com/github/nutonomy/nuscenes-devkit/blob/master/python-sdk/tutorials/nuscenes_tutorial.ipynb - """ - print("Downloading nuScenes minisplit...") - # TODO(leo) implement this - pass - - def ensure_scene_available(root_dir: pathlib.Path, dataset_version: str, scene_name: str) -> None: """ Ensure that the specified scene is available. @@ -38,20 +29,7 @@ def ensure_scene_available(root_dir: pathlib.Path, dataset_version: str, scene_n """ try: nusc = nuscenes.NuScenes(version=dataset_version, dataroot=root_dir, verbose=True) - except AssertionError as e: - MINISPLIT_SCENES = [ - "scene-0061", - "scene-0103", - "scene-0553", - "scene-0655", - "scene-0757", - "scene-0796", - "scene-0916", - "scene-1077", - "scene-1094", - "scene-1100", - ] - + except AssertionError: # dataset initialization failed if dataset_version == "v1.0-mini" and scene_name in MINISPLIT_SCENES: download_minisplit(root_dir) nusc = nuscenes.NuScenes(version=dataset_version, dataroot=root_dir, verbose=True) @@ -85,7 +63,10 @@ def log_nuscenes(root_dir: pathlib.Path, dataset_version: str, scene_name: str) def log_nuscenes_sample( - sample: dict[str, Any], nusc: nuscenes.NuScenes, logged_sensor_tokens: set[str], start_timestamp: numbers.Number + sample: dict[str, Any], + nusc: nuscenes.NuScenes, + logged_sensor_tokens: set[str], + start_timestamp: numbers.Number, ) -> None: # each sample is a keyframe with annotations for sensor_name, sample_data_token in sample["data"].items(): @@ -178,9 +159,9 @@ def main() -> None: rr.script_add_args(parser) args = parser.parse_args() - rr.script_setup(args, "rerun_example_nuscenes") - ensure_scene_available(args.root_dir, args.dataset_version, args.scene_name) + + rr.script_setup(args, "rerun_example_nuscenes") log_nuscenes(args.root_dir, args.dataset_version, args.scene_name) rr.script_teardown(args) From b26c64471e1d8e368e14d6aed855b3b19ae42072 Mon Sep 17 00:00:00 2001 From: Leonard Bruns Date: Thu, 30 Nov 2023 12:06:16 +0100 Subject: [PATCH 11/25] Add minimal nuscenes lidar example --- .../python/minimal_lidar/download_dataset.py | 65 +++++++++++++ examples/python/minimal_lidar/main.py | 95 +++++++++++++++++++ .../python/minimal_lidar/requirements.txt | 4 + examples/python/nuscenes/main.py | 2 +- examples/python/requirements.txt | 2 + 5 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 examples/python/minimal_lidar/download_dataset.py create mode 100755 examples/python/minimal_lidar/main.py create mode 100644 examples/python/minimal_lidar/requirements.txt diff --git a/examples/python/minimal_lidar/download_dataset.py b/examples/python/minimal_lidar/download_dataset.py new file mode 100644 index 000000000000..fbb384643ffe --- /dev/null +++ b/examples/python/minimal_lidar/download_dataset.py @@ -0,0 +1,65 @@ +"""Module to download nuScenes minisplit.""" +from __future__ import annotations + +import os +import pathlib +import tarfile + +import requests +import tqdm + +MINISPLIT_SCENES = [ + "scene-0061", + "scene-0103", + "scene-0553", + "scene-0655", + "scene-0757", + "scene-0796", + "scene-0916", + "scene-1077", + "scene-1094", + "scene-1100", +] +MINISPLIT_URL = "https://www.nuscenes.org/data/v1.0-mini.tgz" + + +def download_file(url: str, dst_file_path: pathlib.Path) -> None: + """Download file from url to dst_fpath.""" + dst_file_path.parent.mkdir(parents=True, exist_ok=True) + print(f"Downloading {url} to {dst_file_path}") + response = requests.get(url, stream=True) + with tqdm.tqdm.wrapattr( + open(dst_file_path, "wb"), + "write", + miniters=1, + total=int(response.headers.get("content-length", 0)), + desc=f"Downloading {dst_file_path.name}", + ) as f: + for chunk in response.iter_content(chunk_size=4096): + f.write(chunk) + + +def untar_file(tar_file_path: pathlib.Path, dst_path: pathlib.Path, keep_tar: bool = True) -> bool: + """Untar tar file at tar_file_path to dst.""" + print(f"Untar file {tar_file_path}") + try: + with tarfile.open(tar_file_path, "r") as tf: + tf.extractall(dst_path) + except Exception as error: + print(f"Error unzipping {tar_file_path}, error: {error}") + return False + if not keep_tar: + os.remove(tar_file_path) + return True + + +def download_minisplit(root_dir: pathlib.Path) -> None: + """ + Download nuScenes minisplit. + + Adopted from https://colab.research.google.com/github/nutonomy/nuscenes-devkit/blob/master/python-sdk/tutorials/nuscenes_tutorial.ipynb + """ + zip_file_path = pathlib.Path("./v1.0-mini.tgz") + if not zip_file_path.is_file(): + download_file(MINISPLIT_URL, zip_file_path) + untar_file(zip_file_path, root_dir, keep_tar=True) diff --git a/examples/python/minimal_lidar/main.py b/examples/python/minimal_lidar/main.py new file mode 100755 index 000000000000..bc5fb39e7297 --- /dev/null +++ b/examples/python/minimal_lidar/main.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import pathlib + +import matplotlib +import numpy as np +import rerun as rr +from download_dataset import MINISPLIT_SCENES, download_minisplit +from nuscenes import nuscenes + +cmap = matplotlib.colormaps["turbo_r"] +norm = matplotlib.colors.Normalize( + vmin=3.0, + vmax=75.0, +) + + +def ensure_scene_available(root_dir: pathlib.Path, dataset_version: str, scene_name: str) -> None: + """ + Ensure that the specified scene is available. + + Downloads minisplit into root_dir if scene_name is part of it and root_dir is empty. + + Raises ValueError if scene is not available and cannot be downloaded. + """ + try: + nusc = nuscenes.NuScenes(version=dataset_version, dataroot=root_dir, verbose=True) + except AssertionError: # dataset initialization failed + if dataset_version == "v1.0-mini" and scene_name in MINISPLIT_SCENES: + download_minisplit(root_dir) + nusc = nuscenes.NuScenes(version=dataset_version, dataroot=root_dir, verbose=True) + else: + print(f"Could not find dataset at {root_dir} and could not automatically download specified scene.") + exit() + + scene_names = [s["name"] for s in nusc.scene] + if scene_name not in scene_names: + raise ValueError(f"{scene_name=} not found in dataset") + + +def log_nuscenes_lidar(root_dir: pathlib.Path, dataset_version: str, scene_name: str) -> None: + nusc = nuscenes.NuScenes(version=dataset_version, dataroot=root_dir, verbose=True) + + scene = next(s for s in nusc.scene if s["name"] == scene_name) + + rr.log("world", rr.ViewCoordinates.RIGHT_HAND_Z_UP, timeless=True) + + first_sample = nusc.get("sample", scene["first_sample_token"]) + start_timestamp = first_sample["timestamp"] # used to start from 0 + current_lidar_token = first_sample["data"]["LIDAR_TOP"] + while current_lidar_token != "": + sample_data = nusc.get("sample_data", current_lidar_token) + + data_file_path = nusc.dataroot / sample_data["filename"] + pointcloud = nuscenes.LidarPointCloud.from_file(str(data_file_path)) + points = pointcloud.points[:3].T # shape after transposing: (num_points, 3) + point_distances = np.linalg.norm(points, axis=1) + point_colors = cmap(norm(point_distances)) + + rr.set_time_seconds("timestamp", (sample_data["timestamp"] - start_timestamp) * 1e-6) + rr.log("world/lidar", rr.Points3D(points, colors=point_colors)) + + current_lidar_token = sample_data["next"] + + +def main() -> None: + parser = argparse.ArgumentParser(description="Visualizes lidar scans using the Rerun SDK.") + parser.add_argument( + "--root_dir", + type=pathlib.Path, + default="dataset", + help="Root directory of nuScenes dataset", + ) + parser.add_argument( + "--scene_name", + type=str, + default="scene-0061", + help="Scene name to visualize (typically of form 'scene-xxxx')", + ) + parser.add_argument("--dataset_version", type=str, default="v1.0-mini", help="Scene id to visualize") + rr.script_add_args(parser) + args = parser.parse_args() + + ensure_scene_available(args.root_dir, args.dataset_version, args.scene_name) + + rr.script_setup(args, "rerun_example_minimal_lidar") + log_nuscenes_lidar(args.root_dir, args.dataset_version, args.scene_name) + + rr.script_teardown(args) + + +if __name__ == "__main__": + main() diff --git a/examples/python/minimal_lidar/requirements.txt b/examples/python/minimal_lidar/requirements.txt new file mode 100644 index 000000000000..9ddcccf2ce52 --- /dev/null +++ b/examples/python/minimal_lidar/requirements.txt @@ -0,0 +1,4 @@ +matplotlib +numpy +nuscenes-devkit +rerun-sdk diff --git a/examples/python/nuscenes/main.py b/examples/python/nuscenes/main.py index 4fc8291ed268..13af7afa5e3d 100755 --- a/examples/python/nuscenes/main.py +++ b/examples/python/nuscenes/main.py @@ -15,7 +15,7 @@ cmap = matplotlib.colormaps["turbo_r"] norm = matplotlib.colors.Normalize( vmin=3.0, - vmax=50.0, + vmax=75.0, ) diff --git a/examples/python/requirements.txt b/examples/python/requirements.txt index 8db00a0eb0ea..48c3a6818de0 100644 --- a/examples/python/requirements.txt +++ b/examples/python/requirements.txt @@ -12,9 +12,11 @@ -r live_camera_edge_detection/requirements.txt -r live_depth_sensor/requirements.txt -r minimal/requirements.txt +-r minimal_lidar/requirements.txt -r minimal_options/requirements.txt -r multiprocessing/requirements.txt -r multithreading/requirements.txt +-r nuscenes/requirements.txt -r notebook/requirements.txt -r nv12/requirements.txt -r objectron/requirements.txt From 44a2212d24626b07a80fe91353debe83cb48b2ac Mon Sep 17 00:00:00 2001 From: Leonard Bruns Date: Thu, 30 Nov 2023 14:50:36 +0100 Subject: [PATCH 12/25] Simplify nuscenes logging and add annotation logging --- examples/python/nuscenes/download_dataset.py | 65 +++++++ examples/python/nuscenes/main.py | 191 ++++++++++++------- 2 files changed, 184 insertions(+), 72 deletions(-) create mode 100644 examples/python/nuscenes/download_dataset.py diff --git a/examples/python/nuscenes/download_dataset.py b/examples/python/nuscenes/download_dataset.py new file mode 100644 index 000000000000..fbb384643ffe --- /dev/null +++ b/examples/python/nuscenes/download_dataset.py @@ -0,0 +1,65 @@ +"""Module to download nuScenes minisplit.""" +from __future__ import annotations + +import os +import pathlib +import tarfile + +import requests +import tqdm + +MINISPLIT_SCENES = [ + "scene-0061", + "scene-0103", + "scene-0553", + "scene-0655", + "scene-0757", + "scene-0796", + "scene-0916", + "scene-1077", + "scene-1094", + "scene-1100", +] +MINISPLIT_URL = "https://www.nuscenes.org/data/v1.0-mini.tgz" + + +def download_file(url: str, dst_file_path: pathlib.Path) -> None: + """Download file from url to dst_fpath.""" + dst_file_path.parent.mkdir(parents=True, exist_ok=True) + print(f"Downloading {url} to {dst_file_path}") + response = requests.get(url, stream=True) + with tqdm.tqdm.wrapattr( + open(dst_file_path, "wb"), + "write", + miniters=1, + total=int(response.headers.get("content-length", 0)), + desc=f"Downloading {dst_file_path.name}", + ) as f: + for chunk in response.iter_content(chunk_size=4096): + f.write(chunk) + + +def untar_file(tar_file_path: pathlib.Path, dst_path: pathlib.Path, keep_tar: bool = True) -> bool: + """Untar tar file at tar_file_path to dst.""" + print(f"Untar file {tar_file_path}") + try: + with tarfile.open(tar_file_path, "r") as tf: + tf.extractall(dst_path) + except Exception as error: + print(f"Error unzipping {tar_file_path}, error: {error}") + return False + if not keep_tar: + os.remove(tar_file_path) + return True + + +def download_minisplit(root_dir: pathlib.Path) -> None: + """ + Download nuScenes minisplit. + + Adopted from https://colab.research.google.com/github/nutonomy/nuscenes-devkit/blob/master/python-sdk/tutorials/nuscenes_tutorial.ipynb + """ + zip_file_path = pathlib.Path("./v1.0-mini.tgz") + if not zip_file_path.is_file(): + download_file(MINISPLIT_URL, zip_file_path) + untar_file(zip_file_path, root_dir, keep_tar=True) diff --git a/examples/python/nuscenes/main.py b/examples/python/nuscenes/main.py index 13af7afa5e3d..e02ebaad24cc 100755 --- a/examples/python/nuscenes/main.py +++ b/examples/python/nuscenes/main.py @@ -52,93 +52,140 @@ def log_nuscenes(root_dir: pathlib.Path, dataset_version: str, scene_name: str) rr.log("world", rr.ViewCoordinates.RIGHT_HAND_Z_UP, timeless=True) - current_sample = nusc.get("sample", scene["first_sample_token"]) - start_timestamp = current_sample["timestamp"] - while True: - log_nuscenes_sample(current_sample, nusc, logged_sensor_tokens, start_timestamp) - - if current_sample["next"] == "": - break - current_sample = nusc.get("sample", current_sample["next"]) - - -def log_nuscenes_sample( - sample: dict[str, Any], - nusc: nuscenes.NuScenes, - logged_sensor_tokens: set[str], - start_timestamp: numbers.Number, -) -> None: - # each sample is a keyframe with annotations - for sensor_name, sample_data_token in sample["data"].items(): - # TODO optional log annotations - while True: - sample_data = nusc.get("sample_data", sample_data_token) - log_nuscenes_sample_data(sample_data, nusc, logged_sensor_tokens, start_timestamp) - - sample_data_token = sample_data["next"] - if sample_data_token == "" or nusc.get("sample_data", sample_data_token)["is_key_frame"]: - break - - -def log_nuscenes_sample_data( - sample_data: dict[str, Any], - nusc: nuscenes.NuScenes, - logged_sensor_tokens: set[str], - start_timestamp: numbers.Number, -): - sensor_name = sample_data["channel"] - calibrated_sensor_token = sample_data["calibrated_sensor_token"] - if calibrated_sensor_token not in logged_sensor_tokens: - calibrated_sensor = nusc.get("calibrated_sensor", calibrated_sensor_token) - rotation_xyzw = np.roll(calibrated_sensor["rotation"], shift=-1) + first_sample_token = scene["first_sample_token"] + first_sample = nusc.get("sample", scene["first_sample_token"]) + start_timestamp = first_sample["timestamp"] + + first_lidar_token = "" + first_radar_tokens = [] + first_camera_tokens = [] + for sample_data_token in first_sample["data"].values(): + sample_data = nusc.get("sample_data", sample_data_token) + log_sensor_calibration(sample_data, nusc) + + if sample_data["sensor_modality"] == "lidar": + first_lidar_token = sample_data_token + elif sample_data["sensor_modality"] == "radar": + first_radar_tokens.append(sample_data_token) + elif sample_data["sensor_modality"] == "camera": + first_camera_tokens.append(sample_data_token) + + log_lidar_and_ego_pose(first_lidar_token, start_timestamp, nusc) + log_cameras(first_camera_tokens, start_timestamp, nusc) + log_radars(first_radar_tokens, start_timestamp, nusc) + log_annotations(first_sample_token, start_timestamp, nusc) + + +def log_lidar_and_ego_pose(first_lidar_token: str, start_timestamp: numbers.Number, nusc: nuscenes.NuScenes): + current_lidar_token = first_lidar_token + + while current_lidar_token != "": + sample_data = nusc.get("sample_data", current_lidar_token) + sensor_name = sample_data["channel"] + rr.set_time_seconds("timestamp", (sample_data["timestamp"] - start_timestamp) * 1e-6) + + ego_pose = nusc.get("ego_pose", sample_data["ego_pose_token"]) + rotation_xyzw = np.roll(ego_pose["rotation"], shift=-1) rr.log( - f"world/ego_vehicle/{sensor_name}", + "world/ego_vehicle", rr.Transform3D( - translation=calibrated_sensor["translation"], + translation=ego_pose["translation"], rotation=rr.Quaternion(xyzw=rotation_xyzw), from_parent=False, ), - timeless=True, ) - logged_sensor_tokens.add(calibrated_sensor_token) - if len(calibrated_sensor["camera_intrinsic"]) != 0: - rr.log( - f"world/ego_vehicle/{sensor_name}", - rr.Pinhole( - image_from_camera=calibrated_sensor["camera_intrinsic"], - width=sample_data["width"], - height=sample_data["height"], - ), - timeless=True, - ) - - rr.set_time_seconds("timestamp", (sample_data["timestamp"] - start_timestamp) * 1e-6) - - data_file_path = nusc.dataroot / sample_data["filename"] - - if sample_data["sensor_modality"] == "lidar": + current_lidar_token = sample_data["next"] + + data_file_path = nusc.dataroot / sample_data["filename"] pointcloud = nuscenes.LidarPointCloud.from_file(str(data_file_path)) points = pointcloud.points[:3].T # shape after transposing: (num_points, 3) point_distances = np.linalg.norm(points, axis=1) point_colors = cmap(norm(point_distances)) rr.log(f"world/ego_vehicle/{sensor_name}", rr.Points3D(points, colors=point_colors)) - ego_pose = nusc.get("ego_pose", sample_data["ego_pose_token"]) - rotation_xyzw = np.roll(ego_pose["rotation"], shift=-1) + +def log_cameras(first_camera_tokens: list[str], start_timestamp: numbers.Number, nusc: nuscenes.NuScenes): + for first_camera_token in first_camera_tokens: + current_camera_token = first_camera_token + while current_camera_token != "": + sample_data = nusc.get("sample_data", current_camera_token) + sensor_name = sample_data["channel"] + rr.set_time_seconds("timestamp", (sample_data["timestamp"] - start_timestamp) * 1e-6) + data_file_path = nusc.dataroot / sample_data["filename"] + rr.log(f"world/ego_vehicle/{sensor_name}", rr.ImageEncoded(path=data_file_path)) + current_camera_token = sample_data["next"] + + +def log_radars(first_radar_tokens: list[str], start_timestamp: numbers.Number, nusc: nuscenes.NuScenes): + for first_radar_token in first_radar_tokens: + current_camera_token = first_radar_token + while current_camera_token != "": + sample_data = nusc.get("sample_data", current_camera_token) + sensor_name = sample_data["channel"] + rr.set_time_seconds("timestamp", (sample_data["timestamp"] - start_timestamp) * 1e-6) + data_file_path = nusc.dataroot / sample_data["filename"] + pointcloud = nuscenes.RadarPointCloud.from_file(str(data_file_path)) + points = pointcloud.points[:3].T # shape after transposing: (num_points, 3) + point_distances = np.linalg.norm(points, axis=1) + point_colors = cmap(norm(point_distances)) + rr.log(f"world/ego_vehicle/{sensor_name}", rr.Points3D(points, colors=point_colors)) + current_camera_token = sample_data["next"] + + +def log_annotations(first_sample_token: str, start_timestamp: numbers.Number, nusc: nuscenes.NuScenes): + label2id = {} + current_sample_token = first_sample_token + while current_sample_token != "": + sample = nusc.get("sample", current_sample_token) + rr.set_time_seconds("timestamp", (sample["timestamp"] - start_timestamp) * 1e-6) + ann_tokens = sample["anns"] + sizes = [] + centers = [] + rotations = [] + class_ids = [] + for ann_token in ann_tokens: + ann = nusc.get("sample_annotation", ann_token) + + rotation_xyzw = np.roll(ann["rotation"], shift=-1) + width, length, height = ann["size"] + sizes.append((length, width, height)) # x, y, z sizes + centers.append(ann["translation"]) + rotations.append(rr.Quaternion(xyzw=rotation_xyzw)) + if ann["category_name"] not in label2id: + label2id[ann["category_name"]] = len(label2id) + class_ids.append(label2id[ann["category_name"]]) + rr.log( - "world/ego_vehicle", - rr.Transform3D( - translation=ego_pose["translation"], - rotation=rr.Quaternion(xyzw=rotation_xyzw), - from_parent=False, + "world/anns", + rr.Boxes3D(sizes=sizes, centers=centers, rotations=rotations, class_ids=class_ids) + ) + current_sample_token = sample["next"] + + +def log_sensor_calibration(sample_data: dict[str, Any], nusc: nuscenes.NuScenes) -> None: + sensor_name = sample_data["channel"] + calibrated_sensor_token = sample_data["calibrated_sensor_token"] + calibrated_sensor = nusc.get("calibrated_sensor", calibrated_sensor_token) + rotation_xyzw = np.roll(calibrated_sensor["rotation"], shift=-1) + rr.log( + f"world/ego_vehicle/{sensor_name}", + rr.Transform3D( + translation=calibrated_sensor["translation"], + rotation=rr.Quaternion(xyzw=rotation_xyzw), + from_parent=False, + ), + timeless=True, + ) + if len(calibrated_sensor["camera_intrinsic"]) != 0: + rr.log( + f"world/ego_vehicle/{sensor_name}", + rr.Pinhole( + image_from_camera=calibrated_sensor["camera_intrinsic"], + width=sample_data["width"], + height=sample_data["height"], ), + timeless=True, ) - elif sample_data["sensor_modality"] == "radar": - pointcloud = nuscenes.RadarPointCloud.from_file(str(data_file_path)) - points = pointcloud.points[:3].T # shape after transposing: (num_points, 3) - rr.log(f"world/ego_vehicle/{sensor_name}", rr.Points3D(points)) - elif sample_data["sensor_modality"] == "camera": - rr.log(f"world/ego_vehicle/{sensor_name}", rr.ImageEncoded(path=data_file_path)) def main() -> None: From c7f705c23d9c0c5c363ab534a9fe4bad5267813d Mon Sep 17 00:00:00 2001 From: Leonard Bruns Date: Thu, 30 Nov 2023 15:10:18 +0100 Subject: [PATCH 13/25] Update minimal_lidar example README --- examples/python/minimal_lidar/README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 examples/python/minimal_lidar/README.md diff --git a/examples/python/minimal_lidar/README.md b/examples/python/minimal_lidar/README.md new file mode 100644 index 000000000000..283188c4b97c --- /dev/null +++ b/examples/python/minimal_lidar/README.md @@ -0,0 +1,23 @@ +--- +title: Lidar +python: https://github.com/rerun-io/rerun/blob/latest/examples/python/minimal_lidar/main.py +tags: [lidar, 3D] +description: "Visualize the lidar data from the nuScenes dataset." +thumbnail: https://static.rerun.io/minimal_lidar/bcea9337044919c1524429bd26bc51a3c4db8ccb/480w.png +thumbnail_dimensions: [480, 286] +--- + + + + + + + + + +This example visualizes only the lidar data from the [nuScenes dataset](https://www.nuscenes.org/) using Rerun. For a moe extensive example including other sensors and annotations check out the [nuScenes example](https://www.rerun.io/docs/examples/real-data/nuscenes). + +```bash +pip install -r examples/python/minimal_lidar/requirements.txt +python examples/python/minimal_lidar/main.py +``` From c48650da059503828e8ff194c144852a80d89106 Mon Sep 17 00:00:00 2001 From: Leonard Bruns Date: Thu, 30 Nov 2023 15:18:02 +0100 Subject: [PATCH 14/25] Update nuScenes example README --- examples/python/nuscenes/README.md | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/examples/python/nuscenes/README.md b/examples/python/nuscenes/README.md index 6fbc701b0a55..91c113204d2f 100644 --- a/examples/python/nuscenes/README.md +++ b/examples/python/nuscenes/README.md @@ -2,22 +2,21 @@ title: nuScenes python: https://github.com/rerun-io/rerun/blob/latest/examples/python/nuscenes/main.py tags: [lidar, 3D, 2D, object-detection, pinhole-camera] -description: "Visualize the nuScenes dataset, which contains lidar data and color images, and labeled bounding boxes." -thumbnail: https://static.rerun.io/arkit_scenes/fb9ec9e8d965369d39d51b17fc7fc5bae6be10cc/480w.png -thumbnail_dimensions: [480, 243] -demo: true +description: "Visualize the nuScenes dataset including lidar, radar, images, and bounding boxes." +thumbnail: https://static.rerun.io/nuscenes/64a50a9d67cbb69ae872551989ee807b195f6b5d/480w.png +thumbnail_dimensions: [480, 282] --- - - - - - ARKit Scenes screenshot + + + + + This example visualizes the [nuScenes dataset](https://www.nuscenes.org/) using Rerun. The dataset -contains lidar data, color images, and labeled bounding boxes. +contains lidar data, radar data, color images, and labeled bounding boxes. ```bash pip install -r examples/python/nuscenes/requirements.txt From f6575b7d313de179d19ab7ddb7d5da782a3305ce Mon Sep 17 00:00:00 2001 From: Leonard Bruns Date: Thu, 30 Nov 2023 15:22:46 +0100 Subject: [PATCH 15/25] Add lidar and nuScenes example to manifest.yml --- examples/manifest.yml | 6 ++++++ .../python/{minimal_lidar => lidar}/README.md | 18 +++++++++--------- examples/python/lidar/dataset | 1 + .../download_dataset.py | 0 .../python/{minimal_lidar => lidar}/main.py | 2 +- .../{minimal_lidar => lidar}/requirements.txt | 0 examples/python/requirements.txt | 4 ++-- 7 files changed, 19 insertions(+), 12 deletions(-) rename examples/python/{minimal_lidar => lidar}/README.md (54%) create mode 120000 examples/python/lidar/dataset rename examples/python/{minimal_lidar => lidar}/download_dataset.py (100%) rename examples/python/{minimal_lidar => lidar}/main.py (98%) rename examples/python/{minimal_lidar => lidar}/requirements.txt (100%) diff --git a/examples/manifest.yml b/examples/manifest.yml index fd5d7c5b371e..87f1fe7d4086 100644 --- a/examples/manifest.yml +++ b/examples/manifest.yml @@ -72,12 +72,18 @@ root: - name: human-pose-tracking python: python/human_pose_tracking + - name: lidar + python: python/lidar + - name: live-camera-edge-detection python: python/live_camera_edge_detection - name: live-depth-sensor python: python/live_depth_sensor + - name: nuscenes + python: python/nuscenes + - name: objectron python: python/objectron rust: rust/objectron diff --git a/examples/python/minimal_lidar/README.md b/examples/python/lidar/README.md similarity index 54% rename from examples/python/minimal_lidar/README.md rename to examples/python/lidar/README.md index 283188c4b97c..af0447c40c74 100644 --- a/examples/python/minimal_lidar/README.md +++ b/examples/python/lidar/README.md @@ -1,23 +1,23 @@ --- title: Lidar -python: https://github.com/rerun-io/rerun/blob/latest/examples/python/minimal_lidar/main.py +python: https://github.com/rerun-io/rerun/blob/latest/examples/python/lidar/main.py tags: [lidar, 3D] description: "Visualize the lidar data from the nuScenes dataset." -thumbnail: https://static.rerun.io/minimal_lidar/bcea9337044919c1524429bd26bc51a3c4db8ccb/480w.png +thumbnail: https://static.rerun.io/lidar/bcea9337044919c1524429bd26bc51a3c4db8ccb/480w.png thumbnail_dimensions: [480, 286] --- - - - - - + + + + + This example visualizes only the lidar data from the [nuScenes dataset](https://www.nuscenes.org/) using Rerun. For a moe extensive example including other sensors and annotations check out the [nuScenes example](https://www.rerun.io/docs/examples/real-data/nuscenes). ```bash -pip install -r examples/python/minimal_lidar/requirements.txt -python examples/python/minimal_lidar/main.py +pip install -r examples/python/lidar/requirements.txt +python examples/python/lidar/main.py ``` diff --git a/examples/python/lidar/dataset b/examples/python/lidar/dataset new file mode 120000 index 000000000000..407167dd09c5 --- /dev/null +++ b/examples/python/lidar/dataset @@ -0,0 +1 @@ +../nuscenes/dataset \ No newline at end of file diff --git a/examples/python/minimal_lidar/download_dataset.py b/examples/python/lidar/download_dataset.py similarity index 100% rename from examples/python/minimal_lidar/download_dataset.py rename to examples/python/lidar/download_dataset.py diff --git a/examples/python/minimal_lidar/main.py b/examples/python/lidar/main.py similarity index 98% rename from examples/python/minimal_lidar/main.py rename to examples/python/lidar/main.py index bc5fb39e7297..331de02907a6 100755 --- a/examples/python/minimal_lidar/main.py +++ b/examples/python/lidar/main.py @@ -85,7 +85,7 @@ def main() -> None: ensure_scene_available(args.root_dir, args.dataset_version, args.scene_name) - rr.script_setup(args, "rerun_example_minimal_lidar") + rr.script_setup(args, "rerun_example_lidar") log_nuscenes_lidar(args.root_dir, args.dataset_version, args.scene_name) rr.script_teardown(args) diff --git a/examples/python/minimal_lidar/requirements.txt b/examples/python/lidar/requirements.txt similarity index 100% rename from examples/python/minimal_lidar/requirements.txt rename to examples/python/lidar/requirements.txt diff --git a/examples/python/requirements.txt b/examples/python/requirements.txt index 48c3a6818de0..927b7c617abb 100644 --- a/examples/python/requirements.txt +++ b/examples/python/requirements.txt @@ -9,15 +9,15 @@ -r dna/requirements.txt -r face_tracking/requirements.txt -r human_pose_tracking/requirements.txt +-r lidar/requirements.txt -r live_camera_edge_detection/requirements.txt -r live_depth_sensor/requirements.txt -r minimal/requirements.txt --r minimal_lidar/requirements.txt -r minimal_options/requirements.txt -r multiprocessing/requirements.txt -r multithreading/requirements.txt --r nuscenes/requirements.txt -r notebook/requirements.txt +-r nuscenes/requirements.txt -r nv12/requirements.txt -r objectron/requirements.txt -r open_photogrammetry_format/requirements.txt From 8b6a78a93e69250b95170c140c7d9cde2648833d Mon Sep 17 00:00:00 2001 From: Leonard Bruns Date: Thu, 30 Nov 2023 15:34:00 +0100 Subject: [PATCH 16/25] Add nuScenes to dictionary --- docs/cspell.json | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/cspell.json b/docs/cspell.json index 3d8494ea388e..9bc9e15152ce 100644 --- a/docs/cspell.json +++ b/docs/cspell.json @@ -210,6 +210,7 @@ "Nikhila", "nohash", "noqa", + "nuScenes", "numpy", "nyud", "obbs", From 67eab1de77303b655d7cb069f39fd7fde6459ef5 Mon Sep 17 00:00:00 2001 From: Leonard Bruns Date: Thu, 30 Nov 2023 15:40:06 +0100 Subject: [PATCH 17/25] Fix links --- examples/python/lidar/README.md | 4 ++-- examples/python/nuscenes/README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/python/lidar/README.md b/examples/python/lidar/README.md index af0447c40c74..5a168643c7ba 100644 --- a/examples/python/lidar/README.md +++ b/examples/python/lidar/README.md @@ -1,6 +1,6 @@ --- title: Lidar -python: https://github.com/rerun-io/rerun/blob/latest/examples/python/lidar/main.py +python: https://github.com/rerun-io/rerun/blob/latest/examples/python/lidar/main.py?speculative-link tags: [lidar, 3D] description: "Visualize the lidar data from the nuScenes dataset." thumbnail: https://static.rerun.io/lidar/bcea9337044919c1524429bd26bc51a3c4db8ccb/480w.png @@ -15,7 +15,7 @@ thumbnail_dimensions: [480, 286] -This example visualizes only the lidar data from the [nuScenes dataset](https://www.nuscenes.org/) using Rerun. For a moe extensive example including other sensors and annotations check out the [nuScenes example](https://www.rerun.io/docs/examples/real-data/nuscenes). +This example visualizes only the lidar data from the [nuScenes dataset](https://www.nuscenes.org/) using Rerun. For a moe extensive example including other sensors and annotations check out the [nuScenes example](examples/real-data/nuscenes). ```bash pip install -r examples/python/lidar/requirements.txt diff --git a/examples/python/nuscenes/README.md b/examples/python/nuscenes/README.md index 91c113204d2f..ad1a7b6d4e5c 100644 --- a/examples/python/nuscenes/README.md +++ b/examples/python/nuscenes/README.md @@ -1,6 +1,6 @@ --- title: nuScenes -python: https://github.com/rerun-io/rerun/blob/latest/examples/python/nuscenes/main.py +python: https://github.com/rerun-io/rerun/blob/latest/examples/python/nuscenes/main.py?speculative-link tags: [lidar, 3D, 2D, object-detection, pinhole-camera] description: "Visualize the nuScenes dataset including lidar, radar, images, and bounding boxes." thumbnail: https://static.rerun.io/nuscenes/64a50a9d67cbb69ae872551989ee807b195f6b5d/480w.png From dbc3ccf205792ac89652a70ea6733b239ccf8358 Mon Sep 17 00:00:00 2001 From: Leonard Bruns Date: Thu, 30 Nov 2023 15:41:12 +0100 Subject: [PATCH 18/25] Fix links again --- examples/python/lidar/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/python/lidar/README.md b/examples/python/lidar/README.md index 5a168643c7ba..6888ca8bb3d2 100644 --- a/examples/python/lidar/README.md +++ b/examples/python/lidar/README.md @@ -15,7 +15,7 @@ thumbnail_dimensions: [480, 286] -This example visualizes only the lidar data from the [nuScenes dataset](https://www.nuscenes.org/) using Rerun. For a moe extensive example including other sensors and annotations check out the [nuScenes example](examples/real-data/nuscenes). +This example visualizes only the lidar data from the [nuScenes dataset](https://www.nuscenes.org/) using Rerun. For a moe extensive example including other sensors and annotations check out the [nuScenes example](nuscenes). ```bash pip install -r examples/python/lidar/requirements.txt From e901a0df3da632a7dd1a834bd92ba6ca559c8da7 Mon Sep 17 00:00:00 2001 From: Leonard Bruns Date: Thu, 30 Nov 2023 15:45:16 +0100 Subject: [PATCH 19/25] Use absolute link for nuScenes example link --- examples/python/lidar/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/python/lidar/README.md b/examples/python/lidar/README.md index 6888ca8bb3d2..35c363c7988e 100644 --- a/examples/python/lidar/README.md +++ b/examples/python/lidar/README.md @@ -15,7 +15,7 @@ thumbnail_dimensions: [480, 286] -This example visualizes only the lidar data from the [nuScenes dataset](https://www.nuscenes.org/) using Rerun. For a moe extensive example including other sensors and annotations check out the [nuScenes example](nuscenes). +This example visualizes only the lidar data from the [nuScenes dataset](https://www.nuscenes.org/) using Rerun. For a moe extensive example including other sensors and annotations check out the [nuScenes example](https://www.rerun.io/examples/real-data/nuscenes?speculative-link). ```bash pip install -r examples/python/lidar/requirements.txt From c8aa8b3d8883e24c40531dbec2c4a783ec941d4b Mon Sep 17 00:00:00 2001 From: Leonard Bruns Date: Thu, 30 Nov 2023 15:55:35 +0100 Subject: [PATCH 20/25] Improve docs --- examples/python/nuscenes/main.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/examples/python/nuscenes/main.py b/examples/python/nuscenes/main.py index e02ebaad24cc..04cfcfac7ca7 100755 --- a/examples/python/nuscenes/main.py +++ b/examples/python/nuscenes/main.py @@ -43,13 +43,11 @@ def ensure_scene_available(root_dir: pathlib.Path, dataset_version: str, scene_n def log_nuscenes(root_dir: pathlib.Path, dataset_version: str, scene_name: str) -> None: + """Log nuScenes scene.""" nusc = nuscenes.NuScenes(version=dataset_version, dataroot=root_dir, verbose=True) scene = next(s for s in nusc.scene if s["name"] == scene_name) - # each sensor only has to be logged once, maintain set of already logged sensors - logged_sensor_tokens: set[str] = set() - rr.log("world", rr.ViewCoordinates.RIGHT_HAND_Z_UP, timeless=True) first_sample_token = scene["first_sample_token"] @@ -76,7 +74,8 @@ def log_nuscenes(root_dir: pathlib.Path, dataset_version: str, scene_name: str) log_annotations(first_sample_token, start_timestamp, nusc) -def log_lidar_and_ego_pose(first_lidar_token: str, start_timestamp: numbers.Number, nusc: nuscenes.NuScenes): +def log_lidar_and_ego_pose(first_lidar_token: str, start_timestamp: numbers.Number, nusc: nuscenes.NuScenes) -> None: + """Log lidar data and vehicle pose.""" current_lidar_token = first_lidar_token while current_lidar_token != "": @@ -104,7 +103,8 @@ def log_lidar_and_ego_pose(first_lidar_token: str, start_timestamp: numbers.Numb rr.log(f"world/ego_vehicle/{sensor_name}", rr.Points3D(points, colors=point_colors)) -def log_cameras(first_camera_tokens: list[str], start_timestamp: numbers.Number, nusc: nuscenes.NuScenes): +def log_cameras(first_camera_tokens: list[str], start_timestamp: numbers.Number, nusc: nuscenes.NuScenes) -> None: + """Log camera data.""" for first_camera_token in first_camera_tokens: current_camera_token = first_camera_token while current_camera_token != "": @@ -116,7 +116,8 @@ def log_cameras(first_camera_tokens: list[str], start_timestamp: numbers.Number, current_camera_token = sample_data["next"] -def log_radars(first_radar_tokens: list[str], start_timestamp: numbers.Number, nusc: nuscenes.NuScenes): +def log_radars(first_radar_tokens: list[str], start_timestamp: numbers.Number, nusc: nuscenes.NuScenes) -> None: + """Log radar data.""" for first_radar_token in first_radar_tokens: current_camera_token = first_radar_token while current_camera_token != "": @@ -132,8 +133,9 @@ def log_radars(first_radar_tokens: list[str], start_timestamp: numbers.Number, n current_camera_token = sample_data["next"] -def log_annotations(first_sample_token: str, start_timestamp: numbers.Number, nusc: nuscenes.NuScenes): - label2id = {} +def log_annotations(first_sample_token: str, start_timestamp: numbers.Number, nusc: nuscenes.NuScenes) -> None: + """Log 3D bounding boxes.""" + label2id: dict[str, int] = {} current_sample_token = first_sample_token while current_sample_token != "": sample = nusc.get("sample", current_sample_token) @@ -163,6 +165,7 @@ def log_annotations(first_sample_token: str, start_timestamp: numbers.Number, nu def log_sensor_calibration(sample_data: dict[str, Any], nusc: nuscenes.NuScenes) -> None: + """Log sensor calibration (pinhole camera, sensor poses, etc.).""" sensor_name = sample_data["channel"] calibrated_sensor_token = sample_data["calibrated_sensor_token"] calibrated_sensor = nusc.get("calibrated_sensor", calibrated_sensor_token) From d7ecf0f5d03e2a9c701e7289682ead4e38fc5a52 Mon Sep 17 00:00:00 2001 From: Leonard Bruns Date: Thu, 30 Nov 2023 16:06:48 +0100 Subject: [PATCH 21/25] Fix formatting --- examples/python/nuscenes/main.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/examples/python/nuscenes/main.py b/examples/python/nuscenes/main.py index 04cfcfac7ca7..8201c8682896 100755 --- a/examples/python/nuscenes/main.py +++ b/examples/python/nuscenes/main.py @@ -157,10 +157,7 @@ def log_annotations(first_sample_token: str, start_timestamp: numbers.Number, nu label2id[ann["category_name"]] = len(label2id) class_ids.append(label2id[ann["category_name"]]) - rr.log( - "world/anns", - rr.Boxes3D(sizes=sizes, centers=centers, rotations=rotations, class_ids=class_ids) - ) + rr.log("world/anns", rr.Boxes3D(sizes=sizes, centers=centers, rotations=rotations, class_ids=class_ids)) current_sample_token = sample["next"] From 79994838ed0fe03c81eb2b4e19314475de6357ca Mon Sep 17 00:00:00 2001 From: Leonard Bruns Date: Thu, 30 Nov 2023 17:19:07 +0100 Subject: [PATCH 22/25] Address review comments --- examples/python/lidar/main.py | 6 ++++-- examples/python/nuscenes/main.py | 35 +++++++++++++++++--------------- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/examples/python/lidar/main.py b/examples/python/lidar/main.py index 331de02907a6..84eabe8bf87b 100755 --- a/examples/python/lidar/main.py +++ b/examples/python/lidar/main.py @@ -10,6 +10,8 @@ from download_dataset import MINISPLIT_SCENES, download_minisplit from nuscenes import nuscenes +# currently need to calculate the color manually +# see https://github.com/rerun-io/rerun/issues/4409 cmap = matplotlib.colormaps["turbo_r"] norm = matplotlib.colors.Normalize( vmin=3.0, @@ -48,7 +50,6 @@ def log_nuscenes_lidar(root_dir: pathlib.Path, dataset_version: str, scene_name: rr.log("world", rr.ViewCoordinates.RIGHT_HAND_Z_UP, timeless=True) first_sample = nusc.get("sample", scene["first_sample_token"]) - start_timestamp = first_sample["timestamp"] # used to start from 0 current_lidar_token = first_sample["data"]["LIDAR_TOP"] while current_lidar_token != "": sample_data = nusc.get("sample_data", current_lidar_token) @@ -59,7 +60,8 @@ def log_nuscenes_lidar(root_dir: pathlib.Path, dataset_version: str, scene_name: point_distances = np.linalg.norm(points, axis=1) point_colors = cmap(norm(point_distances)) - rr.set_time_seconds("timestamp", (sample_data["timestamp"] - start_timestamp) * 1e-6) + # timestamps are in microseconds + rr.set_time_seconds("timestamp", sample_data["timestamp"] * 1e-6) rr.log("world/lidar", rr.Points3D(points, colors=point_colors)) current_lidar_token = sample_data["next"] diff --git a/examples/python/nuscenes/main.py b/examples/python/nuscenes/main.py index 8201c8682896..5c6d0b262e58 100755 --- a/examples/python/nuscenes/main.py +++ b/examples/python/nuscenes/main.py @@ -12,6 +12,8 @@ from download_dataset import MINISPLIT_SCENES, download_minisplit from nuscenes import nuscenes +# currently need to calculate the color manually +# see https://github.com/rerun-io/rerun/issues/4409 cmap = matplotlib.colormaps["turbo_r"] norm = matplotlib.colors.Normalize( vmin=3.0, @@ -52,7 +54,6 @@ def log_nuscenes(root_dir: pathlib.Path, dataset_version: str, scene_name: str) first_sample_token = scene["first_sample_token"] first_sample = nusc.get("sample", scene["first_sample_token"]) - start_timestamp = first_sample["timestamp"] first_lidar_token = "" first_radar_tokens = [] @@ -68,23 +69,25 @@ def log_nuscenes(root_dir: pathlib.Path, dataset_version: str, scene_name: str) elif sample_data["sensor_modality"] == "camera": first_camera_tokens.append(sample_data_token) - log_lidar_and_ego_pose(first_lidar_token, start_timestamp, nusc) - log_cameras(first_camera_tokens, start_timestamp, nusc) - log_radars(first_radar_tokens, start_timestamp, nusc) - log_annotations(first_sample_token, start_timestamp, nusc) + log_lidar_and_ego_pose(first_lidar_token, nusc) + log_cameras(first_camera_tokens, nusc) + log_radars(first_radar_tokens, nusc) + log_annotations(first_sample_token, nusc) -def log_lidar_and_ego_pose(first_lidar_token: str, start_timestamp: numbers.Number, nusc: nuscenes.NuScenes) -> None: +def log_lidar_and_ego_pose(first_lidar_token: str, nusc: nuscenes.NuScenes) -> None: """Log lidar data and vehicle pose.""" current_lidar_token = first_lidar_token while current_lidar_token != "": sample_data = nusc.get("sample_data", current_lidar_token) sensor_name = sample_data["channel"] - rr.set_time_seconds("timestamp", (sample_data["timestamp"] - start_timestamp) * 1e-6) + + # timestamps are in microseconds + rr.set_time_seconds("timestamp", sample_data["timestamp"] * 1e-6) ego_pose = nusc.get("ego_pose", sample_data["ego_pose_token"]) - rotation_xyzw = np.roll(ego_pose["rotation"], shift=-1) + rotation_xyzw = np.roll(ego_pose["rotation"], shift=-1) # go from wxyz to xyzw rr.log( "world/ego_vehicle", rr.Transform3D( @@ -103,27 +106,27 @@ def log_lidar_and_ego_pose(first_lidar_token: str, start_timestamp: numbers.Numb rr.log(f"world/ego_vehicle/{sensor_name}", rr.Points3D(points, colors=point_colors)) -def log_cameras(first_camera_tokens: list[str], start_timestamp: numbers.Number, nusc: nuscenes.NuScenes) -> None: +def log_cameras(first_camera_tokens: list[str], nusc: nuscenes.NuScenes) -> None: """Log camera data.""" for first_camera_token in first_camera_tokens: current_camera_token = first_camera_token while current_camera_token != "": sample_data = nusc.get("sample_data", current_camera_token) sensor_name = sample_data["channel"] - rr.set_time_seconds("timestamp", (sample_data["timestamp"] - start_timestamp) * 1e-6) + rr.set_time_seconds("timestamp", sample_data["timestamp"] * 1e-6) data_file_path = nusc.dataroot / sample_data["filename"] rr.log(f"world/ego_vehicle/{sensor_name}", rr.ImageEncoded(path=data_file_path)) current_camera_token = sample_data["next"] -def log_radars(first_radar_tokens: list[str], start_timestamp: numbers.Number, nusc: nuscenes.NuScenes) -> None: +def log_radars(first_radar_tokens: list[str], nusc: nuscenes.NuScenes) -> None: """Log radar data.""" for first_radar_token in first_radar_tokens: current_camera_token = first_radar_token while current_camera_token != "": sample_data = nusc.get("sample_data", current_camera_token) sensor_name = sample_data["channel"] - rr.set_time_seconds("timestamp", (sample_data["timestamp"] - start_timestamp) * 1e-6) + rr.set_time_seconds("timestamp", sample_data["timestamp"] * 1e-6) data_file_path = nusc.dataroot / sample_data["filename"] pointcloud = nuscenes.RadarPointCloud.from_file(str(data_file_path)) points = pointcloud.points[:3].T # shape after transposing: (num_points, 3) @@ -133,13 +136,13 @@ def log_radars(first_radar_tokens: list[str], start_timestamp: numbers.Number, n current_camera_token = sample_data["next"] -def log_annotations(first_sample_token: str, start_timestamp: numbers.Number, nusc: nuscenes.NuScenes) -> None: +def log_annotations(first_sample_token: str, nusc: nuscenes.NuScenes) -> None: """Log 3D bounding boxes.""" label2id: dict[str, int] = {} current_sample_token = first_sample_token while current_sample_token != "": sample = nusc.get("sample", current_sample_token) - rr.set_time_seconds("timestamp", (sample["timestamp"] - start_timestamp) * 1e-6) + rr.set_time_seconds("timestamp", sample["timestamp"] * 1e-6) ann_tokens = sample["anns"] sizes = [] centers = [] @@ -148,7 +151,7 @@ def log_annotations(first_sample_token: str, start_timestamp: numbers.Number, nu for ann_token in ann_tokens: ann = nusc.get("sample_annotation", ann_token) - rotation_xyzw = np.roll(ann["rotation"], shift=-1) + rotation_xyzw = np.roll(ann["rotation"], shift=-1) # go from wxyz to xyzw width, length, height = ann["size"] sizes.append((length, width, height)) # x, y, z sizes centers.append(ann["translation"]) @@ -166,7 +169,7 @@ def log_sensor_calibration(sample_data: dict[str, Any], nusc: nuscenes.NuScenes) sensor_name = sample_data["channel"] calibrated_sensor_token = sample_data["calibrated_sensor_token"] calibrated_sensor = nusc.get("calibrated_sensor", calibrated_sensor_token) - rotation_xyzw = np.roll(calibrated_sensor["rotation"], shift=-1) + rotation_xyzw = np.roll(calibrated_sensor["rotation"], shift=-1) # go from wxyz to xyzw rr.log( f"world/ego_vehicle/{sensor_name}", rr.Transform3D( From 50fba83ce8235e0458e8317e7df923d0d59203ef Mon Sep 17 00:00:00 2001 From: Leonard Bruns Date: Thu, 30 Nov 2023 17:27:22 +0100 Subject: [PATCH 23/25] Remove unused numbers module --- examples/python/nuscenes/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/python/nuscenes/main.py b/examples/python/nuscenes/main.py index 5c6d0b262e58..b09047406b98 100755 --- a/examples/python/nuscenes/main.py +++ b/examples/python/nuscenes/main.py @@ -2,7 +2,6 @@ from __future__ import annotations import argparse -import numbers import pathlib from typing import Any From 6918f9655500440af83e2933f0bc03363f61ba94 Mon Sep 17 00:00:00 2001 From: Leonard Bruns Date: Thu, 7 Dec 2023 10:29:24 +0100 Subject: [PATCH 24/25] Add commented out annotation context logging with link to issue; add .gitignore for dataset dir --- examples/python/lidar/.gitignore | 1 + examples/python/nuscenes/.gitignore | 1 + examples/python/nuscenes/main.py | 4 ++++ 3 files changed, 6 insertions(+) create mode 100644 examples/python/lidar/.gitignore create mode 100644 examples/python/nuscenes/.gitignore diff --git a/examples/python/lidar/.gitignore b/examples/python/lidar/.gitignore new file mode 100644 index 000000000000..d1ac3a94954e --- /dev/null +++ b/examples/python/lidar/.gitignore @@ -0,0 +1 @@ +dataset/** diff --git a/examples/python/nuscenes/.gitignore b/examples/python/nuscenes/.gitignore new file mode 100644 index 000000000000..d1ac3a94954e --- /dev/null +++ b/examples/python/nuscenes/.gitignore @@ -0,0 +1 @@ +dataset/** diff --git a/examples/python/nuscenes/main.py b/examples/python/nuscenes/main.py index b09047406b98..e3cab7b493b3 100755 --- a/examples/python/nuscenes/main.py +++ b/examples/python/nuscenes/main.py @@ -162,6 +162,10 @@ def log_annotations(first_sample_token: str, nusc: nuscenes.NuScenes) -> None: rr.log("world/anns", rr.Boxes3D(sizes=sizes, centers=centers, rotations=rotations, class_ids=class_ids)) current_sample_token = sample["next"] + # skipping for now since labels take too much space in 3D view (see https://github.com/rerun-io/rerun/issues/4451) + # annotation_context = [(i, label) for label, i in label2id.items()] + # rr.log("world/anns", rr.AnnotationContext(annotation_context), timeless=True) + def log_sensor_calibration(sample_data: dict[str, Any], nusc: nuscenes.NuScenes) -> None: """Log sensor calibration (pinhole camera, sensor poses, etc.).""" From 72f9caff3681fe2cde30f7c0f2ec193ded29831b Mon Sep 17 00:00:00 2001 From: Leonard Bruns Date: Thu, 7 Dec 2023 14:20:44 +0100 Subject: [PATCH 25/25] Download to example dir by default --- examples/python/lidar/main.py | 7 ++++++- examples/python/nuscenes/main.py | 8 ++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/examples/python/lidar/main.py b/examples/python/lidar/main.py index 84eabe8bf87b..17751652f700 100755 --- a/examples/python/lidar/main.py +++ b/examples/python/lidar/main.py @@ -2,7 +2,9 @@ from __future__ import annotations import argparse +import os import pathlib +from typing import Final import matplotlib import numpy as np @@ -10,6 +12,9 @@ from download_dataset import MINISPLIT_SCENES, download_minisplit from nuscenes import nuscenes +EXAMPLE_DIR: Final = pathlib.Path(os.path.dirname(__file__)) +DATASET_DIR: Final = EXAMPLE_DIR / "dataset" + # currently need to calculate the color manually # see https://github.com/rerun-io/rerun/issues/4409 cmap = matplotlib.colormaps["turbo_r"] @@ -72,7 +77,7 @@ def main() -> None: parser.add_argument( "--root_dir", type=pathlib.Path, - default="dataset", + default=DATASET_DIR, help="Root directory of nuScenes dataset", ) parser.add_argument( diff --git a/examples/python/nuscenes/main.py b/examples/python/nuscenes/main.py index e3cab7b493b3..d8e489358979 100755 --- a/examples/python/nuscenes/main.py +++ b/examples/python/nuscenes/main.py @@ -2,8 +2,9 @@ from __future__ import annotations import argparse +import os import pathlib -from typing import Any +from typing import Any, Final import matplotlib import numpy as np @@ -11,6 +12,9 @@ from download_dataset import MINISPLIT_SCENES, download_minisplit from nuscenes import nuscenes +EXAMPLE_DIR: Final = pathlib.Path(os.path.dirname(__file__)) +DATASET_DIR: Final = EXAMPLE_DIR / "dataset" + # currently need to calculate the color manually # see https://github.com/rerun-io/rerun/issues/4409 cmap = matplotlib.colormaps["turbo_r"] @@ -199,7 +203,7 @@ def main() -> None: parser.add_argument( "--root_dir", type=pathlib.Path, - default="dataset", + default=DATASET_DIR, help="Root directory of nuScenes dataset", ) parser.add_argument(