Codegen'd Rust/Arrow (de)ser 9: Rust backport! (#2571)

**Best reviewed on a commit-by-commit basis; in particular the `rerun codegen` commit is nothing but generated code.** You can now use archetypes with the existing SDK: ```rust use rerun::experimental::archetypes::Points2D; MsgSender::from_archetype( "2d_layering/points_between_top_and_middle", &Points2D::new( (0..256).map(|i| (32.0 + (i / 16) as f32 * 16.0, 64.0 + (i % 16) as f32 * 16.0)), ) .with_draw_order(1.51), )? .send(rec_stream)?; ``` Closes #2387 --- - #2484 - #2485 - #2487 - #2545 - #2546 - #2549 - #2554 - #2570 - #2571 --- * [x] I have read and agree to [Contributor Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md) * [x] I've included a screenshot or gif (if applicable) * [x] I have tested [demo.rerun.io](https://demo.rerun.io/pr/{{ pr.number }}) (if applicable) - [PR Build Summary](https://build.rerun.io/pr/{{ pr.number }}) - [Docs preview](https://rerun.io/preview/{{ "pr:%s"|format(pr.branch)|encode_uri_component }}/docs) - [Examples preview](https://rerun.io/preview/{{ "pr:%s"|format(pr.branch)|encode_uri_component }}/examples)
rerun-io · Jul 3, 2023 · 9f883fa · 9f883fa · github-actions · Jul 3, 2023
1 parent 60eb04c
commit 9f883fa
Show file tree

Hide file tree

Showing 22 changed files with 968 additions and 824 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/crates/re_sdk/Cargo.toml b/crates/re_sdk/Cargo.toml
@@ -38,6 +38,7 @@ re_log_types.workspace = true
 re_log.workspace = true
 re_memory.workspace = true
 re_sdk_comms = { workspace = true, features = ["client"] }
+re_types = { workspace = true, features = ["ecolor", "glam"] }
 
 ahash.workspace = true
 crossbeam.workspace = true

diff --git a/crates/re_sdk/src/lib.rs b/crates/re_sdk/src/lib.rs
@@ -94,6 +94,11 @@ pub mod coordinates {
     pub use re_components::coordinates::{Axis3, Handedness, Sign, SignedAxis3};
 }
 
+/// Experimental APIs, to try out upcoming Rerun features.
+pub mod experimental {
+    pub use re_types::{archetypes, components, datatypes, Archetype, Component, Datatype};
+}
+
 /// Re-exports of other crates.
 pub mod external {
     pub use re_log;

diff --git a/crates/re_sdk/src/msg_sender.rs b/crates/re_sdk/src/msg_sender.rs
@@ -1,4 +1,7 @@
-use re_log_types::{DataRow, DataTableError, InstanceKey, RowId, StoreId};
+use re_log_types::{
+    external::arrow2::datatypes::DataType, DataRow, DataTableError, InstanceKey, RowId, StoreId,
+};
+use re_types::Archetype;
 
 use crate::{
     log::DataCell,
@@ -102,6 +105,32 @@ impl MsgSender {
         }
     }
 
+    /// Starts a new `MsgSender` for the given entity path, and fill it with the contents of the
+    /// passed-in [`Archetype`].
+    ///
+    /// WARNING: This is an experimental feature!
+    pub fn from_archetype(
+        ent_path: impl Into<EntityPath>,
+        arch: &impl Archetype,
+    ) -> Result<Self, MsgSenderError> {
+        let serialized = arch.to_arrow();
+
+        let mut this = Self::new(ent_path);
+        for (field, array) in serialized {
+            // NOTE: Unreachable, a top-level Field will always be a component, and thus an
+            // extension.
+            let DataType::Extension(_, _, legacy_fqname) = field.data_type else { unreachable!() };
+            this = this.with_cell(DataCell::from_arrow(
+                // NOTE: Unwrapping is safe as we always include the legacy fqname into the Field's
+                // metadata while migrating towards HOPE.
+                legacy_fqname.as_deref().unwrap().into(),
+                array,
+            ))?;
+        }
+
+        Ok(this)
+    }
+
     /// Read the file at the given path and log it.
     ///
     /// Supported file extensions are:

diff --git a/crates/re_types/source_hash.txt b/crates/re_types/source_hash.txt
@@ -1,4 +1,4 @@
 # This is a sha256 hash for all direct and indirect dependencies of this crate's build script.
 # It can be safely removed at anytime to force the build script to run again.
 # Check out build.rs to see how it's computed.
-128cd542f3f9e6cf3b2a44aeb022cd5f3ad819b00ce5371eeb311300f3e9a7f1
+afc01539cb778ef699e8cef27436420fe8dc4ee078fdd56d874e2409b3749d88
diff --git a/crates/re_types/src/archetypes/fuzzy.rs b/crates/re_types/src/archetypes/fuzzy.rs
diff --git a/crates/re_types/src/archetypes/points2d.rs b/crates/re_types/src/archetypes/points2d.rs
@@ -111,7 +111,11 @@ impl crate::Archetype for Points2D {
                     let array =
                         <crate::components::Point2D>::try_to_arrow(self.points.iter(), None);
                     array.map(|array| {
-                        let datatype = array.data_type().clone();
+                        let datatype = ::arrow2::datatypes::DataType::Extension(
+                            "rerun.components.Point2D".into(),
+                            Box::new(array.data_type().clone()),
+                            Some("rerun.point2d".into()),
+                        );
                         (
                             ::arrow2::datatypes::Field::new("points", datatype, false),
                             array,
@@ -126,7 +130,11 @@ impl crate::Archetype for Points2D {
                     .map(|many| {
                         let array = <crate::components::Radius>::try_to_arrow(many.iter(), None);
                         array.map(|array| {
-                            let datatype = array.data_type().clone();
+                            let datatype = ::arrow2::datatypes::DataType::Extension(
+                                "rerun.components.Radius".into(),
+                                Box::new(array.data_type().clone()),
+                                Some("rerun.radius".into()),
+                            );
                             (
                                 ::arrow2::datatypes::Field::new("radii", datatype, false),
                                 array,
@@ -141,7 +149,11 @@ impl crate::Archetype for Points2D {
                     .map(|many| {
                         let array = <crate::components::Color>::try_to_arrow(many.iter(), None);
                         array.map(|array| {
-                            let datatype = array.data_type().clone();
+                            let datatype = ::arrow2::datatypes::DataType::Extension(
+                                "rerun.components.Color".into(),
+                                Box::new(array.data_type().clone()),
+                                Some("rerun.colorrgba".into()),
+                            );
                             (
                                 ::arrow2::datatypes::Field::new("colors", datatype, false),
                                 array,
@@ -156,7 +168,11 @@ impl crate::Archetype for Points2D {
                     .map(|many| {
                         let array = <crate::components::Label>::try_to_arrow(many.iter(), None);
                         array.map(|array| {
-                            let datatype = array.data_type().clone();
+                            let datatype = ::arrow2::datatypes::DataType::Extension(
+                                "rerun.components.Label".into(),
+                                Box::new(array.data_type().clone()),
+                                Some("rerun.label".into()),
+                            );
                             (
                                 ::arrow2::datatypes::Field::new("labels", datatype, false),
                                 array,
@@ -171,7 +187,11 @@ impl crate::Archetype for Points2D {
                     .map(|single| {
                         let array = <crate::components::DrawOrder>::try_to_arrow([single], None);
                         array.map(|array| {
-                            let datatype = array.data_type().clone();
+                            let datatype = ::arrow2::datatypes::DataType::Extension(
+                                "rerun.components.DrawOrder".into(),
+                                Box::new(array.data_type().clone()),
+                                Some("rerun.draw_order".into()),
+                            );
                             (
                                 ::arrow2::datatypes::Field::new("draw_order", datatype, false),
                                 array,
@@ -186,7 +206,11 @@ impl crate::Archetype for Points2D {
                     .map(|many| {
                         let array = <crate::components::ClassId>::try_to_arrow(many.iter(), None);
                         array.map(|array| {
-                            let datatype = array.data_type().clone();
+                            let datatype = ::arrow2::datatypes::DataType::Extension(
+                                "rerun.components.ClassId".into(),
+                                Box::new(array.data_type().clone()),
+                                Some("rerun.class_id".into()),
+                            );
                             (
                                 ::arrow2::datatypes::Field::new("class_ids", datatype, false),
                                 array,
@@ -202,7 +226,11 @@ impl crate::Archetype for Points2D {
                         let array =
                             <crate::components::KeypointId>::try_to_arrow(many.iter(), None);
                         array.map(|array| {
-                            let datatype = array.data_type().clone();
+                            let datatype = ::arrow2::datatypes::DataType::Extension(
+                                "rerun.components.KeypointId".into(),
+                                Box::new(array.data_type().clone()),
+                                Some("rerun.keypoint_id".into()),
+                            );
                             (
                                 ::arrow2::datatypes::Field::new("keypoint_ids", datatype, false),
                                 array,
@@ -218,7 +246,11 @@ impl crate::Archetype for Points2D {
                         let array =
                             <crate::components::InstanceKey>::try_to_arrow(many.iter(), None);
                         array.map(|array| {
-                            let datatype = array.data_type().clone();
+                            let datatype = ::arrow2::datatypes::DataType::Extension(
+                                "rerun.components.InstanceKey".into(),
+                                Box::new(array.data_type().clone()),
+                                Some("rerun.instance_key".into()),
+                            );
                             (
                                 ::arrow2::datatypes::Field::new("instance_keys", datatype, false),
                                 array,

diff --git a/crates/re_types/src/components/class_id.rs b/crates/re_types/src/components/class_id.rs
@@ -38,11 +38,7 @@ impl crate::Component for ClassId {
     #[inline]
     fn to_arrow_datatype() -> arrow2::datatypes::DataType {
         use ::arrow2::datatypes::*;
-        DataType::Extension(
-            "rerun.components.ClassId".to_owned(),
-            Box::new(DataType::UInt16),
-            None,
-        )
+        DataType::UInt16
     }
 
     #[allow(unused_imports, clippy::wildcard_imports)]
@@ -79,6 +75,8 @@ impl crate::Component for ClassId {
                         Box::new(DataType::UInt16),
                         None,
                     )
+                    .to_logical_type()
+                    .clone()
                 },
                 data0.into_iter().map(|v| v.unwrap_or_default()).collect(),
                 data0_bitmap,

diff --git a/crates/re_types/src/components/color.rs b/crates/re_types/src/components/color.rs
@@ -48,11 +48,7 @@ impl crate::Component for Color {
     #[inline]
     fn to_arrow_datatype() -> arrow2::datatypes::DataType {
         use ::arrow2::datatypes::*;
-        DataType::Extension(
-            "rerun.components.Color".to_owned(),
-            Box::new(DataType::UInt32),
-            None,
-        )
+        DataType::UInt32
     }
 
     #[allow(unused_imports, clippy::wildcard_imports)]
@@ -89,6 +85,8 @@ impl crate::Component for Color {
                         Box::new(DataType::UInt32),
                         None,
                     )
+                    .to_logical_type()
+                    .clone()
                 },
                 data0.into_iter().map(|v| v.unwrap_or_default()).collect(),
                 data0_bitmap,

diff --git a/crates/re_types/src/components/draw_order.rs b/crates/re_types/src/components/draw_order.rs
@@ -43,11 +43,7 @@ impl crate::Component for DrawOrder {
     #[inline]
     fn to_arrow_datatype() -> arrow2::datatypes::DataType {
         use ::arrow2::datatypes::*;
-        DataType::Extension(
-            "rerun.components.DrawOrder".to_owned(),
-            Box::new(DataType::Float32),
-            None,
-        )
+        DataType::Float32
     }
 
     #[allow(unused_imports, clippy::wildcard_imports)]
@@ -84,6 +80,8 @@ impl crate::Component for DrawOrder {
                         Box::new(DataType::Float32),
                         None,
                     )
+                    .to_logical_type()
+                    .clone()
                 },
                 data0.into_iter().map(|v| v.unwrap_or_default()).collect(),
                 data0_bitmap,
Benchmark suite	Current: `9f883fa`	Previous: `60eb04c`	Ratio
`mono_points_arrow_batched/encode_log_msg`	`519349` ns/iter (`± 1380`)	`410576` ns/iter (`± 1058`)	`1.26`
`batch_points_arrow/encode_log_msg`	`89811` ns/iter (`± 90`)	`49537` ns/iter (`± 114`)	`1.81`