Codegen/IDL 3: introduce re_types

Cargo.toml

@@ -56,6 +56,7 @@ re_tensor_ops = { path = "crates/re_tensor_ops", version = "0.7.0-alpha.0", defa
 re_time_panel = { path = "crates/re_time_panel", version = "=0.7.0-alpha.0", default-features = false }
 re_tracing = { path = "crates/re_tracing", version = "0.7.0-alpha.0", default-features = false }
 re_tuid = { path = "crates/re_tuid", version = "0.7.0-alpha.0", default-features = false }
+re_types = { path = "crates/re_types", version = "=0.7.0-alpha.0", default-features = false }
 re_types_builder = { path = "crates/re_types_builder", version = "=0.7.0-alpha.0", default-features = false }
 re_ui = { path = "crates/re_ui", version = "0.7.0-alpha.0", default-features = false }
 re_viewer = { path = "crates/re_viewer", version = "0.7.0-alpha.0", default-features = false }

crates/re_types/Cargo.toml

@@ -0,0 +1,61 @@
+[package]
+name = "re_types"
+authors.workspace = true
+description = "The built-in Rerun data types, component types, and archetypes."
+edition.workspace = true
+homepage.workspace = true
+include.workspace = true
+license.workspace = true
+publish = true
+readme = "README.md"
+repository.workspace = true
+rust-version.workspace = true
+version.workspace = true
+
+
+[package.metadata.docs.rs]
+all-features = true
+
+
+[features]
+default = []
+
+## Enable color conversions.
+ecolor = ["dep:ecolor"]
+
+## Add support for some math operations using [`glam`](https://crates.io/crates/glam/).
+glam = ["dep:glam", "dep:macaw"]
+
+
+[dependencies]
+
+# External
+arrow2 = { workspace = true, features = [
+  "io_ipc",
+  "io_print",
+  "compute_concatenate",
+] }
+bytemuck = { version = "1.11", features = ["derive", "extern_crate_alloc"] }
+document-features = "0.2"
+
+# External (optional)
+ecolor = { workspace = true, optional = true }
+glam = { workspace = true, optional = true }
+macaw = { workspace = true, optional = true }
+
+
+[dev-dependencies]
+
+# External
+glam.workspace = true
+itertools.workspace = true
+
+
+[build-dependencies]
+
+# Rerun
+re_build_tools.workspace = true
+re_types_builder.workspace = true
+
+# External
+xshell = "0.2"

crates/re_types/README.md

@@ -0,0 +1,12 @@
+# re_types
+
+Part of the [`rerun`](https://github.com/rerun-io/rerun) family of crates.
+
+[![Latest version](https://img.shields.io/crates/v/re_types.svg)](https://crates.io/crates/re_types)
+[![Documentation](https://docs.rs/re_types/badge.svg)](https://docs.rs/re_types)
+![MIT](https://img.shields.io/badge/license-MIT-blue.svg)
+![Apache](https://img.shields.io/badge/license-Apache-blue.svg)
+
+The standard Rerun data types, component types, and archetypes.
+
+This crate includes both the language-agnostic definitions (flatbuffers IDL) as well as the generated code.

crates/re_types/build.rs

@@ -0,0 +1,110 @@
+//! Generates Rust & Python code from flatbuffers definitions.
+
+use xshell::{cmd, Shell};
+
+use re_build_tools::{
+    compute_crate_hash, compute_dir_hash, compute_strings_hash, is_tracked_env_var_set, iter_dir,
+    read_versioning_hash, rerun_if_changed, rerun_if_changed_or_doesnt_exist,
+    write_versioning_hash,
+};
+
+// NOTE: Don't need to add extra context to xshell invocations, it does so on its own.
+
+// ---
+
+const SOURCE_HASH_PATH: &str = "./source_hash.txt";
+const DEFINITIONS_DIR_PATH: &str = "./definitions";
+const RUST_OUTPUT_DIR_PATH: &str = ".";
+const PYTHON_OUTPUT_DIR_PATH: &str = "../../rerun_py/rerun_sdk/rerun2";
+
+fn main() {
+    if std::env::var("CI").is_ok() {
+        // Don't run on CI!
+        //
+        // The code we're generating here is actual source code that gets committed into the
+        // repository.
+        return;
+    }
+
+    if !is_tracked_env_var_set("IS_IN_RERUN_WORKSPACE") {
+        // Only run if we are in the rerun workspace, not on users machines.
+        return;
+    }
+    if is_tracked_env_var_set("RERUN_IS_PUBLISHING") {
+        // We don't need to rebuild - we should have done so beforehand!
+        // See `RELEASES.md`
+        return;
+    }
+
+    rerun_if_changed_or_doesnt_exist(SOURCE_HASH_PATH);
+    for path in iter_dir(DEFINITIONS_DIR_PATH, Some(&[".fbs"])) {
+        rerun_if_changed(&path);
+    }
+
+    // NOTE: We need to hash both the flatbuffers definitions as well as the source code of the
+    // code generator itself!
+    let cur_hash = read_versioning_hash(SOURCE_HASH_PATH);
+    let re_types_builder_hash = compute_crate_hash("re_types_builder");
+    let definitions_hash = compute_dir_hash(DEFINITIONS_DIR_PATH, Some(&[".fbs"]));
+    let new_hash = compute_strings_hash(&[&re_types_builder_hash, &definitions_hash]);
+
+    // Leave these be please, very useful when debugging.
+    eprintln!("re_types_builder_hash: {re_types_builder_hash:?}");
+    eprintln!("cur_hash: {cur_hash:?}");
+    eprintln!("definitions_hash: {definitions_hash:?}");
+    eprintln!("new_hash: {new_hash:?}");
+
+    if let Some(cur_hash) = cur_hash {
+        if cur_hash == new_hash {
+            // Neither the source of the code generator nor the IDL definitions have changed, no need
+            // to do anything at this point.
+            return;
+        }
+    }
+
+    let sh = Shell::new().unwrap();
+
+    re_types_builder::generate_rust_code(
+        DEFINITIONS_DIR_PATH,
+        RUST_OUTPUT_DIR_PATH,
+        "./definitions/rerun/archetypes.fbs",
+    );
+
+    // NOTE: We're purposefully ignoring the error here.
+    //
+    // In the very unlikely chance that the user doesn't have the `fmt` component installed,
+    // there's still no good reason to fail the build.
+    //
+    // The CI will catch the unformatted file at PR time and complain appropriately anyhow.
+    cmd!(sh, "cargo fmt").run().ok();
+
+    re_types_builder::generate_python_code(
+        DEFINITIONS_DIR_PATH,
+        PYTHON_OUTPUT_DIR_PATH,
+        "./definitions/rerun/archetypes.fbs",
+    );
+
+    // NOTE: This requires both `black` and `ruff` to be in $PATH, but only for contributors,
+    // not end users.
+    // Even for contributors, `black` and `ruff` won't be needed unless they edit some of the
+    // .fbs files... and even then, this won't crash if they are missing, it will just fail to pass
+    // the CI!
+
+    // NOTE: We're purposefully ignoring the error here.
+    //
+    // If the user doesn't have `black` in their $PATH, there's still no good reason to fail
+    // the build.
+    //
+    // The CI will catch the unformatted files at PR time and complain appropriately anyhow.
+    cmd!(sh, "black {PYTHON_OUTPUT_DIR_PATH}").run().ok();
+
+    // NOTE: We're purposefully ignoring the error here.
+    //
+    // If the user doesn't have `ruff` in their $PATH, there's still no good reason to fail
+    // the build.
+    //
+    // The CI will catch the unformatted files at PR time and complain appropriately anyhow.
+    cmd!(sh, "ruff --fix {PYTHON_OUTPUT_DIR_PATH}").run().ok();
+
+    write_versioning_hash(SOURCE_HASH_PATH, new_hash);
+}

crates/re_types/definitions/arrow/attributes.fbs

@@ -0,0 +1,17 @@
+namespace arrow;
+
+/// Marks a union as sparse, affecting its Arrow datatype.
+///
+/// This does _not_ affect the generated object structure in and of itself, it is a pure Arrow
+/// matter that only impacts (de)serialization.
+///
+/// Only applies to unions.
+attribute "arrow.attr.sparse_union";
+
+/// Marks a single-field object as transparent, affecting its Arrow datatype.
+///
+/// This does _not_ affect the generated object structure in and of itself, it is a pure Arrow
+/// matter that only impacts (de)serialization.
+///
+/// This is generally most useful for getting rid of extraneous `struct` layers.
+attribute "arrow.attr.transparent";

crates/re_types/definitions/fbs/attributes.fbs

@@ -0,0 +1,16 @@
+namespace fbs.attributes;
+
+/// Mandatory attribute that applies to all kinds of objects: structs, enums, unions and even the
+/// fields within.
+///
+/// This defines a stable order between objects of the same kind, e.g. the order in which fields of a
+/// struct should be laid out when generating code.
+/// This is always required since flatbuffers works entirely with unordered maps internally, which
+/// would result in flaky code generation.
+///
+/// In unions, this effectively defines the arrow tag of each variant, since the tag depends on the
+/// fields's order in the datatype!
+///
+/// NOTE: We do not use flatbuffers' builtin `id` attribute as it only works on `table`s, whereas we
+/// need a stable order for all kinds of things.
+attribute "order";

crates/re_types/definitions/fbs/scalars.fbs

@@ -0,0 +1,13 @@
+/// Unions cannot directly refer to scalar types, they need to be wrapped in a struct or table
+/// first.
+/// This package provides pre-wrapped scalars that will be automatically flattened down to their
+/// inner type by our parsers.
+///
+/// Look e.g. for `fbs.scalars.Float32` in `objects.rs` to see this flatenning in action.
+
+namespace fbs.scalars;
+
+/// Flattens down to a 32-bit float.
+struct Float32 {
+  v: float;
+}

crates/re_types/definitions/python/attributes.fbs

@@ -0,0 +1,16 @@
+namespace python.attributes;
+
+/// Marks a field as transparent, meaning its type will be replaced by the underlying type.
+///
+/// Only applies to fields whose type is an object with a single-field.
+attribute "python.attr.transparent";
+
+/// Defines the type aliases for a component, e.g. the types that make up `ComponentLike`.
+///
+/// Only applies to structs/unions that are components.
+attribute "python.attr.aliases";
+
+/// Defines the array type aliases for a component, e.g. the types that make up `ComponentArrayLike`.
+///
+/// Only applies to structs/unions that are components.
+attribute "python.attr.array_aliases";

crates/re_types/definitions/rerun/archetypes.fbs

@@ -0,0 +1 @@
+namespace rerun.archetypes;

crates/re_types/definitions/rust/attributes.fbs

@@ -0,0 +1,15 @@
+namespace rust.attributes;
+
+/// Apply to a struct or table object to generate a tuple struct.
+///
+/// The type definition of the target object must have exactly a single field.
+attribute "rust.attr.tuple_struct";
+
+/// Apply to any object to generate a #derive clause.
+///
+/// The value of the attribute will be trimmed out but otherwise left as-is.
+/// E.g. "rust.attr.derive": "Debug, Clone, Copy"`.
+attribute "rust.attr.derive";
+
+/// Apply to any object to generate a #repr clause with the specified value.
+attribute "rust.attr.repr";

crates/re_types/source_hash.txt

@@ -0,0 +1,4 @@
+# This is a sha256 hash for all direct and indirect dependencies of this crate's build script.
+# It can be safely removed at anytime to force the build script to run again.
+# Check out build.rs to see how it's computed.
+dae77f291d1698807cd865265cbb77731bd1aedf07c0968a6b0ac67c18f94590

crates/re_types/src/archetypes/mod.rs

@@ -0,0 +1 @@
+// NOTE: This file was autogenerated by re_types_builder; DO NOT EDIT.

crates/re_types/src/components/mod.rs

@@ -0,0 +1 @@
+// NOTE: This file was autogenerated by re_types_builder; DO NOT EDIT.

crates/re_types/src/datatypes/mod.rs

@@ -0,0 +1 @@
+// NOTE: This file was autogenerated by re_types_builder; DO NOT EDIT.