Minimal example of running an intel realsense depth sensor live (#2541)

### What

Applies the stored intrinsics and extrinsics and logs the depth and rgb
frames.

To run:
```
python examples/python/live_depth_sensor/main.py
```


![image](https://github.com/rerun-io/rerun/assets/3312232/8513d16d-2970-4f2e-afbd-f28df346b0e3)

Resolves:
 - #2379
 
### Checklist
* [ ] I have read and agree to [Contributor
Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and
the [Code of
Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
* [ ] I've included a screenshot or gif (if applicable)

<!-- This line will get updated when the PR build summary job finishes.
-->
PR Build Summary: https://build.rerun.io/pr/2541

<!-- pr-link-docs:start -->
Docs preview: https://rerun.io/preview/afb479d/docs
Examples preview: https://rerun.io/preview/afb479d/examples
<!-- pr-link-docs:end -->

examples/manifest.yml

@@ -67,6 +67,9 @@ root:
         - name: live-camera-edge-detection
           python: python/live_camera_edge_detection
 
+        - name: live-depth-sensor
+          python: python/live_depth_sensor
+
         - name: rgbd
           python: python/rgbd
 

examples/python/live_camera_edge_detection/main.py

@@ -74,8 +74,6 @@ def main() -> None:
 
     rr.script_setup(args, "live_camera_edge_detection")
 
-    print(args.connect)
-
     if not args.connect:
         print(
             """

examples/python/live_depth_sensor/README.md

@@ -0,0 +1,29 @@
+---
+title: Live Depth Sensor
+python: https://github.com/rerun-io/rerun/blob/latest/examples/python/live_depth_sensor/main.py
+tags: [2D, 3D, live, depth, realsense]
+thumbnail: https://static.rerun.io/8b7fe937b90b05972e01b0e79b4b87dde4a47914_live_depth_sensor_480w.png
+---
+
+<picture>
+  <source media="(max-width: 480px)" srcset="https://static.rerun.io/8b7fe937b90b05972e01b0e79b4b87dde4a47914_live_depth_sensor_480w.png">
+  <source media="(max-width: 768px)" srcset="https://static.rerun.io/47892d2f54f3e4a529ad3d89aef1077f0ee00851_live_depth_sensor_768w.png">
+  <source media="(max-width: 1024px)" srcset="https://static.rerun.io/8d80e7bc742fd81540b108f41213b9908af40ce5_live_depth_sensor_1024w.png">
+  <source media="(max-width: 1200px)" srcset="https://static.rerun.io/dd76ee07bd01527b6b7dcc26794851ce1e4f782e_live_depth_sensor_1200w.png">
+  <img src="https://static.rerun.io/d3c0392bebe2003d24110a779d6f6748167772d8_live_depth_sensor_full.png" alt="Live Depth Sensor example screenshot">
+</picture>
+
+
+A minimal example of streaming frames live from an Intel RealSense depth sensor.
+
+NOTE: this example currently runs forever and will eventually exhaust your
+system memory. It is advised you run an independent rerun viewer with a memory
+limit:
+```
+rerun --memory-limit 4GB
+```
+
+And then connect using:
+```
+python examples/python/live_depth_sensor/main.py --connect
+```

examples/python/live_depth_sensor/main.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+"""
+A minimal example of streaming frames live from an Intel RealSense depth sensor.
+
+NOTE: this example currently runs forever and will eventually exhaust your
+system memory. It is advised you run an independent rerun viewer with a memory
+limit:
+```
+rerun --memory-limit 4GB
+```
+
+And then connect using:
+```
+python examples/python/live_depth_sensor/main.py --connect
+```
+
+"""
+from __future__ import annotations
+
+import argparse
+
+import numpy as np
+import pyrealsense2 as rs
+import rerun as rr  # pip install rerun-sdk
+
+
+def run_realsense(num_frames: int | None) -> None:
+    # Visualize the data as RDF
+    rr.log_view_coordinates("realsense", xyz="RDF", timeless=True)
+
+    # Open the pipe
+    pipe = rs.pipeline()
+    profile = pipe.start()
+
+    # We don't log the depth exstrinsics. We treat the "realsense" space as being at
+    # the origin of the depth sensor so that "realsense/depth" = Identity
+
+    # Get and log depth intrinsics
+    depth_profile = profile.get_stream(rs.stream.depth)
+    depth_intr = depth_profile.as_video_stream_profile().get_intrinsics()
+
+    rr.log_pinhole(
+        "realsense/depth/image",
+        child_from_parent=np.array(
+            (
+                (depth_intr.fx, 0, depth_intr.ppx),
+                (0, depth_intr.fy, depth_intr.ppy),
+                (0, 0, 1),
+            ),
+        ),
+        width=depth_intr.width,
+        height=depth_intr.height,
+        timeless=True,
+    )
+
+    # Get and log color extrinsics
+    rgb_profile = profile.get_stream(rs.stream.color)
+
+    rgb_from_depth = depth_profile.get_extrinsics_to(rgb_profile)
+    rr.log_transform3d(
+        "realsense/rgb",
+        transform=rr.TranslationAndMat3(
+            translation=rgb_from_depth.translation, matrix=np.reshape(rgb_from_depth.rotation, (3, 3))
+        ),
+        from_parent=True,
+        timeless=True,
+    )
+
+    # Get and log color intrinsics
+    rgb_intr = rgb_profile.as_video_stream_profile().get_intrinsics()
+
+    rr.log_pinhole(
+        "realsense/rgb/image",
+        child_from_parent=np.array(
+            (
+                (rgb_intr.fx, 0, rgb_intr.ppx),
+                (0, rgb_intr.fy, rgb_intr.ppy),
+                (0, 0, 1),
+            ),
+        ),
+        width=rgb_intr.width,
+        height=rgb_intr.height,
+        timeless=True,
+    )
+
+    # Read frames in a loop
+    frame_nr = 0
+    try:
+        while True:
+            if num_frames and frame_nr >= num_frames:
+                break
+
+            rr.set_time_sequence("frame_nr", frame_nr)
+            frame_nr += 1
+
+            frames = pipe.wait_for_frames()
+            for f in frames:
+                # Log the depth frame
+                depth_frame = frames.get_depth_frame()
+                depth_units = depth_frame.get_units()
+                depth_image = np.asanyarray(depth_frame.get_data())
+                rr.log_depth_image("realsense/depth/image", depth_image, meter=1.0 / depth_units)
+
+                # Log the color frame
+                color_frame = frames.get_color_frame()
+                color_image = np.asanyarray(color_frame.get_data())
+                rr.log_image("realsense/rgb/image", color_image)
+    finally:
+        pipe.stop()
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Streams frames from a connected realsense depth sensor.")
+    parser.add_argument("--num-frames", type=int, default=None, help="The number of frames to log")
+
+    rr.script_add_args(parser)
+    args = parser.parse_args()
+
+    rr.script_setup(args, "live_depth_sensor")
+
+    if not args.connect:
+        print(
+            """
+################################################################################
+NOTE: this example currently runs forever and will eventually exhaust your
+system memory. It is advised you run an independent rerun viewer with a memory
+limit:
+```
+rerun --memory-limit 4GB
+```
+
+And then connect using:
+```
+python examples/python/live_depth_sensor/main.py --connect
+```
+################################################################################
+        """
+        )
+
+    run_realsense(args.num_frames)
+
+    rr.script_teardown(args)
+
+
+if __name__ == "__main__":
+    main()

examples/python/live_depth_sensor/requirements.txt

@@ -0,0 +1,3 @@
+numpy
+pyrealsense2
+rerun-sdk

examples/python/requirements.txt

@@ -10,6 +10,7 @@
 -r face_tracking/requirements.txt
 -r human_pose_tracking/requirements.txt
 -r live_camera_edge_detection/requirements.txt
+-r live_depth_sensor/requirements.txt
 -r minimal/requirements.txt
 -r minimal_options/requirements.txt
 -r multiprocessing/requirements.txt
@@ -24,4 +25,4 @@
 -r signed_distance_fields/requirements.txt
 -r structure_from_motion/requirements.txt
 -r template/requirements.txt
--r text_logging/requirements.txt
+-r text_logging/requirements.txt

scripts/run_all.py

@@ -93,10 +93,12 @@ def collect_examples(fast: bool) -> list[str]:
             "examples/python/text_logging",
         ]
     else:
-        slow_list = ["examples/python/ros_node/main.py"]
+        # ros requires complex system dependencies to be installed
+        # depth_sensor requires a specific piece of hardware to be attached
+        skip_list = ["examples/python/ros_node/main.py", "examples/python/live_depth_sensor/main.py"]
 
         return [
-            os.path.dirname(main_path) for main_path in glob("examples/python/**/main.py") if main_path not in slow_list
+            os.path.dirname(main_path) for main_path in glob("examples/python/**/main.py") if main_path not in skip_list
         ]