diff --git a/examples/manifest.yml b/examples/manifest.yml index 8d933f972de6..b12802eefabe 100644 --- a/examples/manifest.yml +++ b/examples/manifest.yml @@ -67,6 +67,9 @@ root: - name: live-camera-edge-detection python: python/live_camera_edge_detection + - name: live-depth-sensor + python: python/live_depth_sensor + - name: rgbd python: python/rgbd diff --git a/examples/python/live_camera_edge_detection/main.py b/examples/python/live_camera_edge_detection/main.py index c6bef71af359..11ca63e361d7 100755 --- a/examples/python/live_camera_edge_detection/main.py +++ b/examples/python/live_camera_edge_detection/main.py @@ -74,8 +74,6 @@ def main() -> None: rr.script_setup(args, "live_camera_edge_detection") - print(args.connect) - if not args.connect: print( """ diff --git a/examples/python/live_depth_sensor/README.md b/examples/python/live_depth_sensor/README.md new file mode 100644 index 000000000000..581efda9c7b8 --- /dev/null +++ b/examples/python/live_depth_sensor/README.md @@ -0,0 +1,29 @@ +--- +title: Live Depth Sensor +python: https://github.com/rerun-io/rerun/blob/latest/examples/python/live_depth_sensor/main.py +tags: [2D, 3D, live, depth, realsense] +thumbnail: https://static.rerun.io/8b7fe937b90b05972e01b0e79b4b87dde4a47914_live_depth_sensor_480w.png +--- + + + + + + + Live Depth Sensor example screenshot + + + +A minimal example of streaming frames live from an Intel RealSense depth sensor. + +NOTE: this example currently runs forever and will eventually exhaust your +system memory. It is advised you run an independent rerun viewer with a memory +limit: +``` +rerun --memory-limit 4GB +``` + +And then connect using: +``` +python examples/python/live_depth_sensor/main.py --connect +``` diff --git a/examples/python/live_depth_sensor/main.py b/examples/python/live_depth_sensor/main.py new file mode 100755 index 000000000000..3267f1e0fa61 --- /dev/null +++ b/examples/python/live_depth_sensor/main.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +""" +A minimal example of streaming frames live from an Intel RealSense depth sensor. + +NOTE: this example currently runs forever and will eventually exhaust your +system memory. It is advised you run an independent rerun viewer with a memory +limit: +``` +rerun --memory-limit 4GB +``` + +And then connect using: +``` +python examples/python/live_depth_sensor/main.py --connect +``` + +""" +from __future__ import annotations + +import argparse + +import numpy as np +import pyrealsense2 as rs +import rerun as rr # pip install rerun-sdk + + +def run_realsense(num_frames: int | None) -> None: + # Visualize the data as RDF + rr.log_view_coordinates("realsense", xyz="RDF", timeless=True) + + # Open the pipe + pipe = rs.pipeline() + profile = pipe.start() + + # We don't log the depth exstrinsics. We treat the "realsense" space as being at + # the origin of the depth sensor so that "realsense/depth" = Identity + + # Get and log depth intrinsics + depth_profile = profile.get_stream(rs.stream.depth) + depth_intr = depth_profile.as_video_stream_profile().get_intrinsics() + + rr.log_pinhole( + "realsense/depth/image", + child_from_parent=np.array( + ( + (depth_intr.fx, 0, depth_intr.ppx), + (0, depth_intr.fy, depth_intr.ppy), + (0, 0, 1), + ), + ), + width=depth_intr.width, + height=depth_intr.height, + timeless=True, + ) + + # Get and log color extrinsics + rgb_profile = profile.get_stream(rs.stream.color) + + rgb_from_depth = depth_profile.get_extrinsics_to(rgb_profile) + rr.log_transform3d( + "realsense/rgb", + transform=rr.TranslationAndMat3( + translation=rgb_from_depth.translation, matrix=np.reshape(rgb_from_depth.rotation, (3, 3)) + ), + from_parent=True, + timeless=True, + ) + + # Get and log color intrinsics + rgb_intr = rgb_profile.as_video_stream_profile().get_intrinsics() + + rr.log_pinhole( + "realsense/rgb/image", + child_from_parent=np.array( + ( + (rgb_intr.fx, 0, rgb_intr.ppx), + (0, rgb_intr.fy, rgb_intr.ppy), + (0, 0, 1), + ), + ), + width=rgb_intr.width, + height=rgb_intr.height, + timeless=True, + ) + + # Read frames in a loop + frame_nr = 0 + try: + while True: + if num_frames and frame_nr >= num_frames: + break + + rr.set_time_sequence("frame_nr", frame_nr) + frame_nr += 1 + + frames = pipe.wait_for_frames() + for f in frames: + # Log the depth frame + depth_frame = frames.get_depth_frame() + depth_units = depth_frame.get_units() + depth_image = np.asanyarray(depth_frame.get_data()) + rr.log_depth_image("realsense/depth/image", depth_image, meter=1.0 / depth_units) + + # Log the color frame + color_frame = frames.get_color_frame() + color_image = np.asanyarray(color_frame.get_data()) + rr.log_image("realsense/rgb/image", color_image) + finally: + pipe.stop() + + +def main() -> None: + parser = argparse.ArgumentParser(description="Streams frames from a connected realsense depth sensor.") + parser.add_argument("--num-frames", type=int, default=None, help="The number of frames to log") + + rr.script_add_args(parser) + args = parser.parse_args() + + rr.script_setup(args, "live_depth_sensor") + + if not args.connect: + print( + """ +################################################################################ +NOTE: this example currently runs forever and will eventually exhaust your +system memory. It is advised you run an independent rerun viewer with a memory +limit: +``` +rerun --memory-limit 4GB +``` + +And then connect using: +``` +python examples/python/live_depth_sensor/main.py --connect +``` +################################################################################ + """ + ) + + run_realsense(args.num_frames) + + rr.script_teardown(args) + + +if __name__ == "__main__": + main() diff --git a/examples/python/live_depth_sensor/requirements.txt b/examples/python/live_depth_sensor/requirements.txt new file mode 100644 index 000000000000..6c49749fd9f8 --- /dev/null +++ b/examples/python/live_depth_sensor/requirements.txt @@ -0,0 +1,3 @@ +numpy +pyrealsense2 +rerun-sdk diff --git a/examples/python/requirements.txt b/examples/python/requirements.txt index 4859da7e8a65..c55241dc4dbb 100644 --- a/examples/python/requirements.txt +++ b/examples/python/requirements.txt @@ -10,6 +10,7 @@ -r face_tracking/requirements.txt -r human_pose_tracking/requirements.txt -r live_camera_edge_detection/requirements.txt +-r live_depth_sensor/requirements.txt -r minimal/requirements.txt -r minimal_options/requirements.txt -r multiprocessing/requirements.txt @@ -24,4 +25,4 @@ -r signed_distance_fields/requirements.txt -r structure_from_motion/requirements.txt -r template/requirements.txt --r text_logging/requirements.txt \ No newline at end of file +-r text_logging/requirements.txt diff --git a/scripts/run_all.py b/scripts/run_all.py index 983d58999c8e..fad09785be7f 100755 --- a/scripts/run_all.py +++ b/scripts/run_all.py @@ -93,10 +93,12 @@ def collect_examples(fast: bool) -> list[str]: "examples/python/text_logging", ] else: - slow_list = ["examples/python/ros_node/main.py"] + # ros requires complex system dependencies to be installed + # depth_sensor requires a specific piece of hardware to be attached + skip_list = ["examples/python/ros_node/main.py", "examples/python/live_depth_sensor/main.py"] return [ - os.path.dirname(main_path) for main_path in glob("examples/python/**/main.py") if main_path not in slow_list + os.path.dirname(main_path) for main_path in glob("examples/python/**/main.py") if main_path not in skip_list ]