Codegen'd Rust/Arrow (de)ser 2: matching legacy definitions (#2485)

**Best reviewed on a commit-by-commit basis; in particular the `rerun
codegen` commit is nothing but generated code.**

This PR updates (or rather, downdates?) the IDL definitions of
`Points2D` and its components to make sure they match the legacy ones
(e.g. `Point2D` is back to being a `struct { x, y }` rather than a
`[f32; 2]`).

We'll revert that once we're done with the backporting/migration and get
rid of the legacy stuff.

UPDATE: This now also includes the `attr.rerun.legacy_fqname` attribute,
which does what you think it does.

---

- #2484
- #2485 
- #2487 
- #2520 

---

<!-- This line will get updated when the PR build summary job finishes.
-->
PR Build Summary: https://build.rerun.io/pr/2485

<!-- pr-link-docs:start -->
Docs preview: https://rerun.io/preview/41e265f/docs
Examples preview: https://rerun.io/preview/41e265f/examples
<!-- pr-link-docs:end -->

crates/re_types/definitions/rerun/attributes.fbs

@@ -17,3 +17,10 @@ attribute "attr.rerun.component_recommended";
 ///
 /// Only applies to the fields of an archetype.
 attribute "attr.rerun.component_optional";
+
+/// Applies to any component object in order to override its fully-qualified name.
+///
+/// Only used to match legacy fqnames while we're in the process of migrating.
+//
+// TODO(cmc): remove once legacy is gone.
+attribute "attr.rerun.legacy_fqname";

crates/re_types/definitions/rerun/components/class_id.fbs

@@ -15,6 +15,7 @@ struct ClassId (
   "attr.arrow.transparent",
   "attr.python.aliases": "int",
   "attr.python.array_aliases": "npt.NDArray[np.uint8], npt.NDArray[np.uint16], npt.NDArray[np.uint32], npt.NDArray[np.uint64]",
+  "attr.rerun.legacy_fqname": "rerun.class_id",
   "attr.rust.derive": "Copy, Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash",
   "attr.rust.tuple_struct",
   order: 100

crates/re_types/definitions/rerun/components/color.fbs

@@ -18,6 +18,7 @@ struct Color (
   "attr.arrow.transparent",
   "attr.python.aliases": "int, npt.NDArray[np.uint8], npt.NDArray[np.uint32], npt.NDArray[np.float32], npt.NDArray[np.float64]",
   "attr.python.array_aliases": "Sequence[int], npt.NDArray[np.uint8], npt.NDArray[np.uint32], npt.NDArray[np.float32], npt.NDArray[np.float64]",
+  "attr.rerun.legacy_fqname": "rerun.colorrgba",
   "attr.rust.derive": "Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, bytemuck::Pod, bytemuck::Zeroable",
   "attr.rust.repr": "transparent",
   "attr.rust.tuple_struct",

crates/re_types/definitions/rerun/components/draw_order.fbs

@@ -20,6 +20,7 @@ struct DrawOrder (
   "attr.arrow.transparent",
   "attr.python.aliases": "float",
   "attr.python.array_aliases": "npt.NDArray[np.float32]",
+  "attr.rerun.legacy_fqname": "rerun.draw_order",
   "attr.rust.derive": "Debug, Clone, Copy",
   "attr.rust.repr": "transparent",
   "attr.rust.tuple_struct",

crates/re_types/definitions/rerun/components/instance_key.fbs

@@ -14,6 +14,7 @@ struct InstanceKey (
   "attr.arrow.transparent",
   "attr.python.aliases": "int",
   "attr.python.array_aliases": "npt.NDArray[np.uint64]",
+  "attr.rerun.legacy_fqname": "rerun.instance_key",
   "attr.rust.tuple_struct",
   "attr.rust.derive": "Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord",
   order: 100

crates/re_types/definitions/rerun/components/keypoint_id.fbs

@@ -22,6 +22,7 @@ struct KeypointId (
   "attr.arrow.transparent",
   "attr.python.aliases": "float",
   "attr.python.array_aliases": "npt.NDArray[np.uint8], npt.NDArray[np.uint16], npt.NDArray[np.uint32]",
+  "attr.rerun.legacy_fqname": "rerun.keypoint_id",
   "attr.rust.derive": "Copy, Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash",
   "attr.rust.tuple_struct",
   order: 200

crates/re_types/definitions/rerun/components/label.fbs

@@ -13,6 +13,7 @@ namespace rerun.components;
 table Label (
   "attr.arrow.transparent",
   "attr.python.aliases": "str",
+  "attr.rerun.legacy_fqname": "rerun.label",
   "attr.rust.derive": "Debug, Clone, PartialEq, Eq, PartialOrd, Ord",
   "attr.rust.repr": "transparent",
   "attr.rust.tuple_struct",

crates/re_types/definitions/rerun/components/point2d.fbs

@@ -11,15 +11,31 @@ namespace rerun.components;
 
 /// A point in 2D space.
 struct Point2D (
-  "attr.arrow.transparent",
   "attr.python.aliases": "npt.NDArray[np.float32], Sequence[float], Tuple[float, float]",
   "attr.python.array_aliases": "npt.NDArray[np.float32], Sequence[float]",
-  "attr.rust.tuple_struct",
+  "attr.rerun.legacy_fqname": "rerun.point2d",
   "attr.rust.derive": "Debug, Default, Clone, Copy, PartialEq, PartialOrd",
   order: 100
 ) {
-  position: rerun.datatypes.Vec2D (
-    "attr.python.transparent",
-    order: 100
-  );
+  x: float (order: 100);
+  y: float (order: 200);
 }
+
+// ---
+
+// TODO(cmc): use the following definition instead once we've finalized the switch to HOPE.
+
+// /// A point in 2D space.
+// struct Point2D (
+//   "attr.arrow.transparent",
+//   "attr.python.aliases": "npt.NDArray[np.float32], Sequence[float], Tuple[float, float]",
+//   "attr.python.array_aliases": "npt.NDArray[np.float32], Sequence[float]",
+//   "attr.rust.tuple_struct",
+//   "attr.rust.derive": "Debug, Default, Clone, Copy, PartialEq, PartialOrd",
+//   order: 100
+// ) {
+//   position: rerun.datatypes.Vec2D (
+//     "attr.python.transparent",
+//     order: 100
+//   );
+// }

crates/re_types/definitions/rerun/components/radius.fbs

@@ -14,6 +14,7 @@ struct Radius (
   "attr.arrow.transparent",
   "attr.python.aliases": "float",
   "attr.python.array_aliases": "npt.NDArray[np.float32]",
+  "attr.rerun.legacy_fqname": "rerun.radius",
   "attr.rust.tuple_struct",
   "attr.rust.derive": "Debug, Clone, Copy, PartialEq, PartialOrd",
   order: 100

crates/re_types/source_hash.txt

@@ -1,4 +1,4 @@
 # This is a sha256 hash for all direct and indirect dependencies of this crate's build script.
 # It can be safely removed at anytime to force the build script to run again.
 # Check out build.rs to see how it's computed.
-ba3dbc22cd914eac35c130d4bac2754a325ca5cd6e5154cd1511963b2bdc0769
+c19f042ca7d4df439ce98030fdd82f4c8be1fec2f1046a525c48f4a91d5dc0e6

crates/re_types/src/components/class_id.rs

@@ -8,7 +8,7 @@ pub struct ClassId(pub u16);
 
 impl crate::Component for ClassId {
     fn name() -> crate::ComponentName {
-        crate::ComponentName::Borrowed("rerun.components.ClassId")
+        crate::ComponentName::Borrowed("rerun.class_id")
     }
 
     #[allow(clippy::wildcard_imports)]

crates/re_types/src/components/color.rs

@@ -18,7 +18,7 @@ pub struct Color(pub u32);
 
 impl crate::Component for Color {
     fn name() -> crate::ComponentName {
-        crate::ComponentName::Borrowed("rerun.components.Color")
+        crate::ComponentName::Borrowed("rerun.colorrgba")
     }
 
     #[allow(clippy::wildcard_imports)]

crates/re_types/src/components/draw_order.rs

@@ -13,7 +13,7 @@ pub struct DrawOrder(pub f32);
 
 impl crate::Component for DrawOrder {
     fn name() -> crate::ComponentName {
-        crate::ComponentName::Borrowed("rerun.components.DrawOrder")
+        crate::ComponentName::Borrowed("rerun.draw_order")
     }
 
     #[allow(clippy::wildcard_imports)]

crates/re_types/src/components/instance_key.rs

@@ -6,7 +6,7 @@ pub struct InstanceKey(pub u64);
 
 impl crate::Component for InstanceKey {
     fn name() -> crate::ComponentName {
-        crate::ComponentName::Borrowed("rerun.components.InstanceKey")
+        crate::ComponentName::Borrowed("rerun.instance_key")
     }
 
     #[allow(clippy::wildcard_imports)]

crates/re_types/src/components/keypoint_id.rs

@@ -10,7 +10,7 @@ pub struct KeypointId(pub u16);
 
 impl crate::Component for KeypointId {
     fn name() -> crate::ComponentName {
-        crate::ComponentName::Borrowed("rerun.components.KeypointId")
+        crate::ComponentName::Borrowed("rerun.keypoint_id")
     }
 
     #[allow(clippy::wildcard_imports)]

crates/re_types/src/components/label.rs

@@ -7,7 +7,7 @@ pub struct Label(pub String);
 
 impl crate::Component for Label {
     fn name() -> crate::ComponentName {
-        crate::ComponentName::Borrowed("rerun.components.Label")
+        crate::ComponentName::Borrowed("rerun.label")
     }
 
     #[allow(clippy::wildcard_imports)]

crates/re_types/src/components/point2d.rs

@@ -2,24 +2,36 @@
 
 #[doc = "A point in 2D space."]
 #[derive(Debug, Default, Clone, Copy, PartialEq, PartialOrd)]
-pub struct Point2D(pub crate::datatypes::Vec2D);
+pub struct Point2D {
+    pub x: f32,
+    pub y: f32,
+}
 
 impl crate::Component for Point2D {
     fn name() -> crate::ComponentName {
-        crate::ComponentName::Borrowed("rerun.components.Point2D")
+        crate::ComponentName::Borrowed("rerun.point2d")
     }
 
     #[allow(clippy::wildcard_imports)]
     fn to_arrow_datatype() -> arrow2::datatypes::DataType {
         use ::arrow2::datatypes::*;
-        DataType::FixedSizeList(
-            Box::new(Field {
-                name: "item".to_owned(),
-                data_type: DataType::Float32,
-                is_nullable: false,
-                metadata: [].into(),
-            }),
-            2usize,
+        DataType::Extension(
+            "rerun.components.Point2D".to_owned(),
+            Box::new(DataType::Struct(vec![
+                Field {
+                    name: "x".to_owned(),
+                    data_type: DataType::Float32,
+                    is_nullable: true,
+                    metadata: [].into(),
+                },
+                Field {
+                    name: "y".to_owned(),
+                    data_type: DataType::Float32,
+                    is_nullable: true,
+                    metadata: [].into(),
+                },
+            ])),
+            None,
         )
     }
 }

crates/re_types/src/components/point2d_ext.rs

@@ -6,29 +6,21 @@ impl Point2D {
 
     #[inline]
     pub const fn new(x: f32, y: f32) -> Self {
-        Self(crate::datatypes::Vec2D::new(x, y))
-    }
-}
-
-impl std::ops::Deref for Point2D {
-    type Target = crate::datatypes::Vec2D;
-
-    fn deref(&self) -> &Self::Target {
-        &self.0
+        Self { x, y }
     }
 }
 
 impl From<(f32, f32)> for Point2D {
     #[inline]
-    fn from(xy: (f32, f32)) -> Self {
-        Self(xy.into())
+    fn from((x, y): (f32, f32)) -> Self {
+        Self { x, y }
     }
 }
 
 impl From<[f32; 2]> for Point2D {
     #[inline]
-    fn from(p: [f32; 2]) -> Self {
-        Self(p.into())
+    fn from([x, y]: [f32; 2]) -> Self {
+        Self { x, y }
     }
 }
 
@@ -44,14 +36,74 @@ impl From<glam::Vec2> for Point2D {
 impl From<Point2D> for glam::Vec2 {
     #[inline]
     fn from(pt: Point2D) -> Self {
-        Self::new(pt.x(), pt.y())
+        Self::new(pt.x, pt.y)
     }
 }
 
 #[cfg(feature = "glam")]
 impl From<Point2D> for glam::Vec3 {
     #[inline]
     fn from(pt: Point2D) -> Self {
-        Self::new(pt.x(), pt.y(), 0.0)
+        Self::new(pt.x, pt.y, 0.0)
     }
 }
+
+// ---
+
+// TODO(cmc): use the following definition instead once we've finalized the switch to HOPE.
+
+// impl Point2D {
+//     pub const ZERO: Self = Self::new(0.0, 0.0);
+//     pub const ONE: Self = Self::new(1.0, 1.0);
+
+//     #[inline]
+//     pub const fn new(x: f32, y: f32) -> Self {
+//         Self(crate::datatypes::Vec2D::new(x, y))
+//     }
+// }
+
+// impl std::ops::Deref for Point2D {
+//     type Target = crate::datatypes::Vec2D;
+
+//     fn deref(&self) -> &Self::Target {
+//         &self.0
+//     }
+// }
+
+// impl From<(f32, f32)> for Point2D {
+//     #[inline]
+//     fn from(xy: (f32, f32)) -> Self {
+//         Self(xy.into())
+//     }
+// }
+
+// impl From<[f32; 2]> for Point2D {
+//     #[inline]
+//     fn from(p: [f32; 2]) -> Self {
+//         Self(p.into())
+//     }
+// }
+
+// #[cfg(feature = "glam")]
+// impl From<glam::Vec2> for Point2D {
+//     #[inline]
+//     fn from(pt: glam::Vec2) -> Self {
+//         Self::new(pt.x, pt.y)
+//     }
+// }
+
+// #[cfg(feature = "glam")]
+// impl From<Point2D> for glam::Vec2 {
+//     #[inline]
+//     fn from(pt: Point2D) -> Self {
+//         Self::new(pt.x(), pt.y())
+//     }
+// }
+
+// #[cfg(feature = "glam")]
+// impl From<Point2D> for glam::Vec3 {
+//     #[inline]
+//     fn from(pt: Point2D) -> Self {
+//         Self::new(pt.x(), pt.y(), 0.0)
+//     }
+// }

crates/re_types/src/components/radius.rs

@@ -6,7 +6,7 @@ pub struct Radius(pub f32);
 
 impl crate::Component for Radius {
     fn name() -> crate::ComponentName {
-        crate::ComponentName::Borrowed("rerun.components.Radius")
+        crate::ComponentName::Borrowed("rerun.radius")
     }
 
     #[allow(clippy::wildcard_imports)]

crates/re_types_builder/src/codegen/python.rs

@@ -11,6 +11,7 @@ use crate::{
     codegen::{StringExt as _, AUTOGEN_WARNING},
     ArrowRegistry, CodeGenerator, Docs, ElementType, Object, ObjectField, ObjectKind, Objects,
     Type, ATTR_PYTHON_ALIASES, ATTR_PYTHON_ARRAY_ALIASES, ATTR_PYTHON_TRANSPARENT,
+    ATTR_RERUN_LEGACY_FQNAME,
 };
 
 // ---
@@ -760,6 +761,10 @@ fn quote_arrow_support_from_obj(arrow_registry: &ArrowRegistry, obj: &Object) ->
                 ])
                 .to_case(Case::Snake);
 
+            let legacy_fqname = obj
+                .try_get_attr::<String>(ATTR_RERUN_LEGACY_FQNAME)
+                .unwrap_or_else(|| fqname.clone());
+
             unindent::unindent(&format!(
                 r#"
 
@@ -770,7 +775,7 @@ fn quote_arrow_support_from_obj(arrow_registry: &ArrowRegistry, obj: &Object) ->
                 class {arrow}(pa.ExtensionType): # type: ignore[misc]
                     def __init__(self: type[pa.ExtensionType]) -> None:
                         pa.ExtensionType.__init__(
-                            self, {datatype}, "{fqname}"
+                            self, {datatype}, "{legacy_fqname}"
                         )
 
                     def __arrow_ext_serialize__(self: type[pa.ExtensionType]) -> bytes:
@@ -787,7 +792,8 @@ fn quote_arrow_support_from_obj(arrow_registry: &ArrowRegistry, obj: &Object) ->
                     def __arrow_ext_class__(self: type[pa.ExtensionType]) -> type[pa.ExtensionArray]:
                         return {many}
 
-                pa.register_extension_type({arrow}())
+                # TODO(cmc): bring back registration to pyarrow once legacy types are gone
+                # pa.register_extension_type({arrow}())
 
                 class {many}(pa.ExtensionArray, {many}Ext):  # type: ignore[misc]
                     @staticmethod