Codegen'd Rust/Arrow (de)ser 1: upgrading to actual TokenStreams (#…

…2484)

**Best reviewed on a commit-by-commit basis; in particular the `rerun
codegen` commit is nothing but generated code.**

This PR upgrades the Rust code generator to make use of actual
`TokenStream`s (i.e. `syn` & `quote`) rather than manipulating raw
strings; because what's coming in the following PRs puts us way past the
complexity threshold for using simple strings.

Interestingly, while `syn` & `quote` were obviously designed with Rust
in mind, they aren't specific to Rust so much as they are made to work
with bracket-base languages.
I.e. the Rust code generator becomes a nice starting point for a C or
C++ one.

---

- #2484
- #2485 
- #2487 
- #2520 

---

<!-- This line will get updated when the PR build summary job finishes.
-->
PR Build Summary: https://build.rerun.io/pr/2484

<!-- pr-link-docs:start -->
Docs preview: https://rerun.io/preview/cf52fbc/docs
Examples preview: https://rerun.io/preview/cf52fbc/examples
<!-- pr-link-docs:end -->

crates/re_types/source_hash.txt

@@ -1,4 +1,4 @@
 # This is a sha256 hash for all direct and indirect dependencies of this crate's build script.
 # It can be safely removed at anytime to force the build script to run again.
 # Check out build.rs to see how it's computed.
-3fb506e31fddc92682449571abf2af20254349733bbe601289de768e4cfd317b
+ba3dbc22cd914eac35c130d4bac2754a325ca5cd6e5154cd1511963b2bdc0769

crates/re_types/src/archetypes/points2d.rs

@@ -1,63 +1,63 @@
 // NOTE: This file was autogenerated by re_types_builder; DO NOT EDIT.
 
-/// A 2D point cloud with positions and optional colors, radii, labels, etc.
+#[doc = "A 2D point cloud with positions and optional colors, radii, labels, etc."]
 #[derive(Debug, Clone, PartialEq)]
 pub struct Points2D {
-    /// All the actual 2D points that make up the point cloud.
+    #[doc = "All the actual 2D points that make up the point cloud."]
     pub points: Vec<crate::components::Point2D>,
 
-    /// Optional radii for the points, effectively turning them into circles.
+    #[doc = "Optional radii for the points, effectively turning them into circles."]
     pub radii: Option<Vec<crate::components::Radius>>,
 
-    /// Optional colors for the points.
+    #[doc = "Optional colors for the points."]
     pub colors: Option<Vec<crate::components::Color>>,
 
-    /// Optional text labels for the points.
+    #[doc = "Optional text labels for the points."]
     pub labels: Option<Vec<crate::components::Label>>,
 
-    /// An optional floating point value that specifies the 2D drawing order.
-    /// Objects with higher values are drawn on top of those with lower values.
-    ///
-    /// The default for 2D points is 30.0.
+    #[doc = "An optional floating point value that specifies the 2D drawing order."]
+    #[doc = "Objects with higher values are drawn on top of those with lower values."]
+    #[doc = ""]
+    #[doc = "The default for 2D points is 30.0."]
     pub draw_order: Option<crate::components::DrawOrder>,
 
-    /// Optional class Ids for the points.
-    ///
-    /// The class ID provides colors and labels if not specified explicitly.
+    #[doc = "Optional class Ids for the points."]
+    #[doc = ""]
+    #[doc = "The class ID provides colors and labels if not specified explicitly."]
     pub class_ids: Option<Vec<crate::components::ClassId>>,
 
-    /// Optional keypoint IDs for the points, identifying them within a class.
-    ///
-    /// If keypoint IDs are passed in but no class IDs were specified, the class ID will
-    /// default to 0.
-    /// This is useful to identify points within a single classification (which is identified
-    /// with `class_id`).
-    /// E.g. the classification might be 'Person' and the keypoints refer to joints on a
-    /// detected skeleton.
+    #[doc = "Optional keypoint IDs for the points, identifying them within a class."]
+    #[doc = ""]
+    #[doc = "If keypoint IDs are passed in but no class IDs were specified, the class ID will"]
+    #[doc = "default to 0."]
+    #[doc = "This is useful to identify points within a single classification (which is identified"]
+    #[doc = "with `class_id`)."]
+    #[doc = "E.g. the classification might be 'Person' and the keypoints refer to joints on a"]
+    #[doc = "detected skeleton."]
     pub keypoint_ids: Option<Vec<crate::components::KeypointId>>,
 
-    /// Unique identifiers for each individual point in the batch.
+    #[doc = "Unique identifiers for each individual point in the batch."]
     pub instance_keys: Option<Vec<crate::components::InstanceKey>>,
 }
 
 impl Points2D {
-    pub const REQUIRED_COMPONENTS: [crate::ComponentName; 1] =
+    pub const REQUIRED_COMPONENTS: [crate::ComponentName; 1usize] =
         [crate::ComponentName::Borrowed("rerun.components.Point2D")];
 
-    pub const RECOMMENDED_COMPONENTS: [crate::ComponentName; 2] = [
+    pub const RECOMMENDED_COMPONENTS: [crate::ComponentName; 2usize] = [
         crate::ComponentName::Borrowed("rerun.components.Radius"),
         crate::ComponentName::Borrowed("rerun.components.Color"),
     ];
 
-    pub const OPTIONAL_COMPONENTS: [crate::ComponentName; 5] = [
+    pub const OPTIONAL_COMPONENTS: [crate::ComponentName; 5usize] = [
         crate::ComponentName::Borrowed("rerun.components.Label"),
         crate::ComponentName::Borrowed("rerun.components.DrawOrder"),
         crate::ComponentName::Borrowed("rerun.components.ClassId"),
         crate::ComponentName::Borrowed("rerun.components.KeypointId"),
         crate::ComponentName::Borrowed("rerun.components.InstanceKey"),
     ];
 
-    pub const ALL_COMPONENTS: [crate::ComponentName; 8] = [
+    pub const ALL_COMPONENTS: [crate::ComponentName; 8usize] = [
         crate::ComponentName::Borrowed("rerun.components.Point2D"),
         crate::ComponentName::Borrowed("rerun.components.Radius"),
         crate::ComponentName::Borrowed("rerun.components.Color"),
@@ -86,10 +86,9 @@ impl crate::Archetype for Points2D {
         Self::OPTIONAL_COMPONENTS.to_vec()
     }
 
-    #[allow(clippy::unimplemented)]
+    #[allow(clippy::todo)]
     fn to_arrow_datatypes() -> Vec<arrow2::datatypes::DataType> {
-        // TODO(#2368): dump the arrow registry into the generated code
-        unimplemented!("query the registry for all fqnames"); // NOLINT
+        todo!("query the registry for all fqnames");
     }
 }
 

crates/re_types/src/components/class_id.rs

@@ -1,8 +1,8 @@
 // NOTE: This file was autogenerated by re_types_builder; DO NOT EDIT.
 
-/// A 16-bit ID representing a type of semantic class.
-///
-/// Used to look up a `crate::components::ClassDescription` within the `crate::components::AnnotationContext`.
+#[doc = "A 16-bit ID representing a type of semantic class."]
+#[doc = ""]
+#[doc = "Used to look up a `crate::components::ClassDescription` within the `crate::components::AnnotationContext`."]
 #[derive(Copy, Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub struct ClassId(pub u16);
 

crates/re_types/src/components/color.rs

@@ -1,8 +1,17 @@
 // NOTE: This file was autogenerated by re_types_builder; DO NOT EDIT.
 
-/// An RGBA color tuple with unmultiplied/separate alpha, in sRGB gamma space with linear alpha.
+#[doc = "An RGBA color tuple with unmultiplied/separate alpha, in sRGB gamma space with linear alpha."]
 #[derive(
-    Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, bytemuck::Pod, bytemuck::Zeroable,
+    Debug,
+    Default,
+    Clone,
+    Copy,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    bytemuck :: Pod,
+    bytemuck :: Zeroable,
 )]
 #[repr(transparent)]
 pub struct Color(pub u32);

crates/re_types/src/components/draw_order.rs

@@ -1,12 +1,12 @@
 // NOTE: This file was autogenerated by re_types_builder; DO NOT EDIT.
 
-/// Draw order used for the display order of 2D elements.
-///
-/// Higher values are drawn on top of lower values.
-/// An entity can have only a single draw order component.
-/// Within an entity draw order is governed by the order of the components.
-///
-/// Draw order for entities with the same draw order is generally undefined.
+#[doc = "Draw order used for the display order of 2D elements."]
+#[doc = ""]
+#[doc = "Higher values are drawn on top of lower values."]
+#[doc = "An entity can have only a single draw order component."]
+#[doc = "Within an entity draw order is governed by the order of the components."]
+#[doc = ""]
+#[doc = "Draw order for entities with the same draw order is generally undefined."]
 #[derive(Debug, Clone, Copy)]
 #[repr(transparent)]
 pub struct DrawOrder(pub f32);

crates/re_types/src/components/instance_key.rs

@@ -1,6 +1,6 @@
 // NOTE: This file was autogenerated by re_types_builder; DO NOT EDIT.
 
-/// A unique numeric identifier for each individual instance within a batch.
+#[doc = "A unique numeric identifier for each individual instance within a batch."]
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
 pub struct InstanceKey(pub u64);
 

crates/re_types/src/components/keypoint_id.rs

@@ -1,10 +1,10 @@
 // NOTE: This file was autogenerated by re_types_builder; DO NOT EDIT.
 
-/// A 16-bit ID representing a type of semantic keypoint within a class.
-///
-/// `KeypointId`s are only meaningful within the context of a `crate::components::ClassDescription`.
-///
-/// Used to look up an `crate::components::AnnotationInfo` for a Keypoint within the `crate::components::AnnotationContext`.
+#[doc = "A 16-bit ID representing a type of semantic keypoint within a class."]
+#[doc = ""]
+#[doc = "`KeypointId`s are only meaningful within the context of a `crate::components::ClassDescription`."]
+#[doc = ""]
+#[doc = "Used to look up an `crate::components::AnnotationInfo` for a Keypoint within the `crate::components::AnnotationContext`."]
 #[derive(Copy, Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub struct KeypointId(pub u16);
 

crates/re_types/src/components/label.rs

@@ -1,6 +1,6 @@
 // NOTE: This file was autogenerated by re_types_builder; DO NOT EDIT.
 
-/// A String label component.
+#[doc = "A String label component."]
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
 #[repr(transparent)]
 pub struct Label(pub String);

crates/re_types/src/components/point2d.rs

@@ -1,6 +1,6 @@
 // NOTE: This file was autogenerated by re_types_builder; DO NOT EDIT.
 
-/// A point in 2D space.
+#[doc = "A point in 2D space."]
 #[derive(Debug, Default, Clone, Copy, PartialEq, PartialOrd)]
 pub struct Point2D(pub crate::datatypes::Vec2D);
 
@@ -19,7 +19,7 @@ impl crate::Component for Point2D {
                 is_nullable: false,
                 metadata: [].into(),
             }),
-            2,
+            2usize,
         )
     }
 }

crates/re_types/src/components/radius.rs

@@ -1,6 +1,6 @@
 // NOTE: This file was autogenerated by re_types_builder; DO NOT EDIT.
 
-/// A Radius component.
+#[doc = "A Radius component."]
 #[derive(Debug, Clone, Copy, PartialEq, PartialOrd)]
 pub struct Radius(pub f32);
 

crates/re_types/src/datatypes/vec2d.rs

@@ -1,8 +1,8 @@
 // NOTE: This file was autogenerated by re_types_builder; DO NOT EDIT.
 
-/// A vector in 2D space.
+#[doc = "A vector in 2D space."]
 #[derive(Debug, Default, Clone, Copy, PartialEq, PartialOrd)]
-pub struct Vec2D(pub [f32; 2]);
+pub struct Vec2D(pub [f32; 2usize]);
 
 impl crate::Datatype for Vec2D {
     fn name() -> crate::DatatypeName {
@@ -19,7 +19,7 @@ impl crate::Datatype for Vec2D {
                 is_nullable: false,
                 metadata: [].into(),
             }),
-            2,
+            2usize,
         )
     }
 }

crates/re_types_builder/Cargo.toml

@@ -25,6 +25,10 @@ arrow2.workspace = true
 convert_case = "0.6"
 flatbuffers = "23.0"
 indent = "0.1"
+itertools.workspace = true
+proc-macro2 = { version = "1.0", default-features = false }
+quote = "1.0"
+syn = "2.0"
 unindent.workspace = true
 xshell = "0.2"
 

crates/re_types_builder/src/codegen/common.rs

@@ -2,7 +2,8 @@
 
 use crate::Docs;
 
-pub fn quote_doc_from_docs(docs: &Docs, tags: &[&str]) -> Vec<String> {
+/// Retrieves the global and tagged documentation from a [`Docs`] object.
+pub fn get_documentation(docs: &Docs, tags: &[&str]) -> Vec<String> {
     fn trim_mono_start_whitespace_if_needed(line: &str) -> &str {
         if line.chars().next().map_or(false, |c| c.is_whitespace()) {
             // NOTE: don't trim! only that very specific space should go away

crates/re_types_builder/src/codegen/mod.rs

@@ -3,7 +3,7 @@ pub trait CodeGenerator {
     /// Generates user-facing code from [`crate::Objects`].
     ///
     /// Returns the paths of all generated files.
-    fn quote(
+    fn generate(
         &mut self,
         objs: &crate::Objects,
         arrow_registry: &crate::ArrowRegistry,
@@ -18,7 +18,7 @@ pub const AUTOGEN_WARNING: &str =
 // ---
 
 mod common;
-use self::common::{quote_doc_from_docs, StringExt};
+use self::common::{get_documentation, StringExt};
 
 mod python;
 mod rust;

crates/re_types_builder/src/codegen/python.rs

@@ -28,7 +28,7 @@ impl PythonCodeGenerator {
 }
 
 impl CodeGenerator for PythonCodeGenerator {
-    fn quote(&mut self, objs: &Objects, arrow_registry: &ArrowRegistry) -> Vec<PathBuf> {
+    fn generate(&mut self, objs: &Objects, arrow_registry: &ArrowRegistry) -> Vec<PathBuf> {
         let mut filepaths = Vec::new();
 
         let datatypes_path = self.pkg_path.join("datatypes");
@@ -442,7 +442,7 @@ fn quote_module_prelude() -> String {
 }
 
 fn quote_doc_from_docs(docs: &Docs) -> String {
-    let lines = crate::codegen::quote_doc_from_docs(docs, &["py", "python"]);
+    let lines = crate::codegen::get_documentation(docs, &["py", "python"]);
 
     if lines.is_empty() {
         return String::new();