Minimal example of running an intel realsense depth sensor live

examples/manifest.yml

@@ -67,6 +67,9 @@ root:
         - name: live-camera-edge-detection
           python: python/live_camera_edge_detection
 
+        - name: live-depth-sensor
+          python: python/live_depth_sensor
+
         - name: rgbd
           python: python/rgbd
 

examples/python/live_depth_sensor/README.md

@@ -0,0 +1,29 @@
+---
+title: Live Depth Sensor
+python: https://github.com/rerun-io/rerun/blob/latest/examples/python/live_depth_sensor/main.py
+tags: [2D, 3D, live, depth, realsense]
+thumbnail: https://static.rerun.io/8b7fe937b90b05972e01b0e79b4b87dde4a47914_live_depth_sensor_480w.png
+---
+
+<picture>
+  <source media="(max-width: 480px)" srcset="https://static.rerun.io/8b7fe937b90b05972e01b0e79b4b87dde4a47914_live_depth_sensor_480w.png">
+  <source media="(max-width: 768px)" srcset="https://static.rerun.io/47892d2f54f3e4a529ad3d89aef1077f0ee00851_live_depth_sensor_768w.png">
+  <source media="(max-width: 1024px)" srcset="https://static.rerun.io/8d80e7bc742fd81540b108f41213b9908af40ce5_live_depth_sensor_1024w.png">
+  <source media="(max-width: 1200px)" srcset="https://static.rerun.io/dd76ee07bd01527b6b7dcc26794851ce1e4f782e_live_depth_sensor_1200w.png">
+  <img src="https://static.rerun.io/d3c0392bebe2003d24110a779d6f6748167772d8_live_depth_sensor_full.png" alt="Live Depth Sensor example screenshot">
+</picture>
+
+
+A minimal example of streaming frames live from an intel realsense depth sensor.
+
+NOTE: this example currently runs forever and will eventually exhaust your
+system memory. It is advised you run an independent rerun viewer with a memory
+limit:
+```
+rerun --memory-limit 4GB
+```
+
+And then connect using:
+```
+python examples/python/live_depth_sensor/main.py --connect
+```

examples/python/live_depth_sensor/main.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+"""
+A minimal example of streaming frames live from an intel realsense depth sensor.
+
+NOTE: this example currently runs forever and will eventually exhaust your
+system memory. It is advised you run an independent rerun viewer with a memory
+limit:
+```
+rerun --memory-limit 4GB
+```
+
+And then connect using:
+```
+python examples/python/live_depth_sensor/main.py --connect
+```
+
+"""
+from __future__ import annotations
+
+import argparse
+
+import numpy as np
+import pyrealsense2 as rs
+import rerun as rr  # pip install rerun-sdk
+
+
+def run_realsense(num_frames: int | None) -> None:
+    # Visualize the data as RDF
+    rr.log_view_coordinates("realsense", xyz="RDF", timeless=True)
+
+    # Open the pipe
+    pipe = rs.pipeline()
+    profile = pipe.start()
+
+    # Get and log depth exstrinsics
+    depth_profile = profile.get_stream(rs.stream.depth)
+    depth_intr = depth_profile.as_video_stream_profile().get_intrinsics()
+
+    rr.log_pinhole(
+        "realsense/camera/depth",
+        child_from_parent=np.array(
+            (
+                (depth_intr.fx, 0, depth_intr.ppx),
+                (0, depth_intr.fy, depth_intr.ppy),
+                (0, 0, 1),
+            ),
+        ),
+        width=depth_intr.width,
+        height=depth_intr.height,
+        timeless=True,
+    )
+
+    # Get and log color extrinsics
+    rgb_profile = profile.get_stream(rs.stream.color)
+
+    depth_extr = depth_profile.get_extrinsics_to(rgb_profile)
+    rr.log_transform3d(
+        "realsense/camera/ext",
+        transform=rr.TranslationAndMat3(
+            translation=depth_extr.translation, matrix=np.reshape(depth_extr.rotation, (3, 3))
+        ),
+        from_parent=True,
+        timeless=True,
+    )
+
+    # Get and log color intrinsics
+    rgb_intr = rgb_profile.as_video_stream_profile().get_intrinsics()
+
+    rr.log_pinhole(
+        "realsense/camera/ext/rgb",
+        child_from_parent=np.array(
+            (
+                (rgb_intr.fx, 0, rgb_intr.ppx),
+                (0, rgb_intr.fy, rgb_intr.ppy),
+                (0, 0, 1),
+            ),
+        ),
+        width=rgb_intr.width,
+        height=rgb_intr.height,
+        timeless=True,
+    )
+
+    # Read frames in a loop
+    frame_nr = 0
+    try:
+        while True:
+            if num_frames and frame_nr >= num_frames:
+                break
+
+            rr.set_time_sequence("frame_nr", frame_nr)
+            frame_nr += 1
+
+            frames = pipe.wait_for_frames()
+            for f in frames:
+                # Log the depth frame
+                depth_frame = frames.get_depth_frame()
+                depth_units = depth_frame.get_units()
+                depth_image = np.asanyarray(depth_frame.get_data())
+                rr.log_depth_image("realsense/camera/depth", depth_image, meter=1.0 / depth_units)
+
+                # Log the color frame
+                color_frame = frames.get_color_frame()
+                color_image = np.asanyarray(color_frame.get_data())
+                rr.log_image("realsense/camera/ext/rgb", color_image)
+    finally:
+        pipe.stop()
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Streams frames from a connected realsense depth sensor.")
+    parser.add_argument("--num-frames", type=int, default=None, help="The number of frames to log")
+
+    rr.script_add_args(parser)
+    args = parser.parse_args()
+
+    rr.script_setup(args, "live_depth_sensor")
+
+    print(args.connect)
+
+    if not args.connect:
+        print(
+            """
+################################################################################
+NOTE: this example currently runs forever and will eventually exhaust your
+system memory. It is advised you run an independent rerun viewer with a memory
+limit:
+```
+rerun --memory-limit 4GB
+```
+
+And then connect using:
+```
+python examples/python/live_depth_sensor/main.py --connect
+```
+################################################################################
+        """
+        )
+
+    run_realsense(args.num_frames)
+
+    rr.script_teardown(args)
+
+
+if __name__ == "__main__":
+    main()

examples/python/live_depth_sensor/requirements.txt

@@ -0,0 +1,3 @@
+numpy
+pyrealsense2
+rerun-sdk

examples/python/requirements.txt

@@ -10,6 +10,7 @@
 -r face_tracking/requirements.txt
 -r human_pose_tracking/requirements.txt
 -r live_camera_edge_detection/requirements.txt
+-r live_depth_sensor/requirements.txt
 -r minimal/requirements.txt
 -r minimal_options/requirements.txt
 -r multiprocessing/requirements.txt
@@ -24,4 +25,4 @@
 -r signed_distance_fields/requirements.txt
 -r structure_from_motion/requirements.txt
 -r template/requirements.txt
--r text_logging/requirements.txt
+-r text_logging/requirements.txt