diff --git a/Cargo.lock b/Cargo.lock index 1558ece32be5..f6211650f901 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4497,6 +4497,22 @@ dependencies = [ "web-time", ] +[[package]] +name = "re_types" +version = "0.7.0-alpha.0" +dependencies = [ + "arrow2", + "bytemuck", + "document-features", + "ecolor", + "glam", + "itertools", + "macaw", + "re_build_tools", + "re_types_builder", + "xshell", +] + [[package]] name = "re_types_builder" version = "0.7.0-alpha.0" diff --git a/Cargo.toml b/Cargo.toml index 13b1df640125..f0a47151f6b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,6 +56,7 @@ re_tensor_ops = { path = "crates/re_tensor_ops", version = "0.7.0-alpha.0", defa re_time_panel = { path = "crates/re_time_panel", version = "=0.7.0-alpha.0", default-features = false } re_tracing = { path = "crates/re_tracing", version = "0.7.0-alpha.0", default-features = false } re_tuid = { path = "crates/re_tuid", version = "0.7.0-alpha.0", default-features = false } +re_types = { path = "crates/re_types", version = "=0.7.0-alpha.0", default-features = false } re_types_builder = { path = "crates/re_types_builder", version = "=0.7.0-alpha.0", default-features = false } re_ui = { path = "crates/re_ui", version = "0.7.0-alpha.0", default-features = false } re_viewer = { path = "crates/re_viewer", version = "0.7.0-alpha.0", default-features = false } diff --git a/crates/re_types/Cargo.toml b/crates/re_types/Cargo.toml new file mode 100644 index 000000000000..4da294c84c50 --- /dev/null +++ b/crates/re_types/Cargo.toml @@ -0,0 +1,61 @@ +[package] +name = "re_types" +authors.workspace = true +description = "The built-in Rerun data types, component types, and archetypes." +edition.workspace = true +homepage.workspace = true +include.workspace = true +license.workspace = true +publish = true +readme = "README.md" +repository.workspace = true +rust-version.workspace = true +version.workspace = true + + +[package.metadata.docs.rs] +all-features = true + + +[features] +default = [] + +## Enable color conversions. +ecolor = ["dep:ecolor"] + +## Add support for some math operations using [`glam`](https://crates.io/crates/glam/). +glam = ["dep:glam", "dep:macaw"] + + +[dependencies] + +# External +arrow2 = { workspace = true, features = [ + "io_ipc", + "io_print", + "compute_concatenate", +] } +bytemuck = { version = "1.11", features = ["derive", "extern_crate_alloc"] } +document-features = "0.2" + +# External (optional) +ecolor = { workspace = true, optional = true } +glam = { workspace = true, optional = true } +macaw = { workspace = true, optional = true } + + +[dev-dependencies] + +# External +glam.workspace = true +itertools.workspace = true + + +[build-dependencies] + +# Rerun +re_build_tools.workspace = true +re_types_builder.workspace = true + +# External +xshell = "0.2" diff --git a/crates/re_types/README.md b/crates/re_types/README.md new file mode 100644 index 000000000000..62c1ebddf06a --- /dev/null +++ b/crates/re_types/README.md @@ -0,0 +1,12 @@ +# re_types + +Part of the [`rerun`](https://github.com/rerun-io/rerun) family of crates. + +[![Latest version](https://img.shields.io/crates/v/re_types.svg)](https://crates.io/crates/re_types) +[![Documentation](https://docs.rs/re_types/badge.svg)](https://docs.rs/re_types) +![MIT](https://img.shields.io/badge/license-MIT-blue.svg) +![Apache](https://img.shields.io/badge/license-Apache-blue.svg) + +The standard Rerun data types, component types, and archetypes. + +This crate includes both the language-agnostic definitions (flatbuffers IDL) as well as the generated code. diff --git a/crates/re_types/build.rs b/crates/re_types/build.rs new file mode 100644 index 000000000000..82c328e0aa4b --- /dev/null +++ b/crates/re_types/build.rs @@ -0,0 +1,110 @@ +//! Generates Rust & Python code from flatbuffers definitions. + +use xshell::{cmd, Shell}; + +use re_build_tools::{ + compute_crate_hash, compute_dir_hash, compute_strings_hash, is_tracked_env_var_set, iter_dir, + read_versioning_hash, rerun_if_changed, rerun_if_changed_or_doesnt_exist, + write_versioning_hash, +}; + +// NOTE: Don't need to add extra context to xshell invocations, it does so on its own. + +// --- + +const SOURCE_HASH_PATH: &str = "./source_hash.txt"; +const DEFINITIONS_DIR_PATH: &str = "./definitions"; +const RUST_OUTPUT_DIR_PATH: &str = "."; +const PYTHON_OUTPUT_DIR_PATH: &str = "../../rerun_py/rerun_sdk/rerun2"; + +fn main() { + if std::env::var("CI").is_ok() { + // Don't run on CI! + // + // The code we're generating here is actual source code that gets committed into the + // repository. + return; + } + + if !is_tracked_env_var_set("IS_IN_RERUN_WORKSPACE") { + // Only run if we are in the rerun workspace, not on users machines. + return; + } + if is_tracked_env_var_set("RERUN_IS_PUBLISHING") { + // We don't need to rebuild - we should have done so beforehand! + // See `RELEASES.md` + return; + } + + rerun_if_changed_or_doesnt_exist(SOURCE_HASH_PATH); + for path in iter_dir(DEFINITIONS_DIR_PATH, Some(&[".fbs"])) { + rerun_if_changed(&path); + } + + // NOTE: We need to hash both the flatbuffers definitions as well as the source code of the + // code generator itself! + let cur_hash = read_versioning_hash(SOURCE_HASH_PATH); + let re_types_builder_hash = compute_crate_hash("re_types_builder"); + let definitions_hash = compute_dir_hash(DEFINITIONS_DIR_PATH, Some(&[".fbs"])); + let new_hash = compute_strings_hash(&[&re_types_builder_hash, &definitions_hash]); + + // Leave these be please, very useful when debugging. + eprintln!("re_types_builder_hash: {re_types_builder_hash:?}"); + eprintln!("cur_hash: {cur_hash:?}"); + eprintln!("definitions_hash: {definitions_hash:?}"); + eprintln!("new_hash: {new_hash:?}"); + + if let Some(cur_hash) = cur_hash { + if cur_hash == new_hash { + // Neither the source of the code generator nor the IDL definitions have changed, no need + // to do anything at this point. + return; + } + } + + let sh = Shell::new().unwrap(); + + re_types_builder::generate_rust_code( + DEFINITIONS_DIR_PATH, + RUST_OUTPUT_DIR_PATH, + "./definitions/rerun/archetypes.fbs", + ); + + // NOTE: We're purposefully ignoring the error here. + // + // In the very unlikely chance that the user doesn't have the `fmt` component installed, + // there's still no good reason to fail the build. + // + // The CI will catch the unformatted file at PR time and complain appropriately anyhow. + cmd!(sh, "cargo fmt").run().ok(); + + re_types_builder::generate_python_code( + DEFINITIONS_DIR_PATH, + PYTHON_OUTPUT_DIR_PATH, + "./definitions/rerun/archetypes.fbs", + ); + + // NOTE: This requires both `black` and `ruff` to be in $PATH, but only for contributors, + // not end users. + // Even for contributors, `black` and `ruff` won't be needed unless they edit some of the + // .fbs files... and even then, this won't crash if they are missing, it will just fail to pass + // the CI! + + // NOTE: We're purposefully ignoring the error here. + // + // If the user doesn't have `black` in their $PATH, there's still no good reason to fail + // the build. + // + // The CI will catch the unformatted files at PR time and complain appropriately anyhow. + cmd!(sh, "black {PYTHON_OUTPUT_DIR_PATH}").run().ok(); + + // NOTE: We're purposefully ignoring the error here. + // + // If the user doesn't have `ruff` in their $PATH, there's still no good reason to fail + // the build. + // + // The CI will catch the unformatted files at PR time and complain appropriately anyhow. + cmd!(sh, "ruff --fix {PYTHON_OUTPUT_DIR_PATH}").run().ok(); + + write_versioning_hash(SOURCE_HASH_PATH, new_hash); +} diff --git a/crates/re_types/definitions/arrow/attributes.fbs b/crates/re_types/definitions/arrow/attributes.fbs new file mode 100644 index 000000000000..f8e96e4d8bad --- /dev/null +++ b/crates/re_types/definitions/arrow/attributes.fbs @@ -0,0 +1,17 @@ +namespace arrow; + +/// Marks a union as sparse, affecting its Arrow datatype. +/// +/// This does _not_ affect the generated object structure in and of itself, it is a pure Arrow +/// matter that only impacts (de)serialization. +/// +/// Only applies to unions. +attribute "arrow.attr.sparse_union"; + +/// Marks a single-field object as transparent, affecting its Arrow datatype. +/// +/// This does _not_ affect the generated object structure in and of itself, it is a pure Arrow +/// matter that only impacts (de)serialization. +/// +/// This is generally most useful for getting rid of extraneous `struct` layers. +attribute "arrow.attr.transparent"; diff --git a/crates/re_types/definitions/fbs/attributes.fbs b/crates/re_types/definitions/fbs/attributes.fbs new file mode 100644 index 000000000000..d431691e8d96 --- /dev/null +++ b/crates/re_types/definitions/fbs/attributes.fbs @@ -0,0 +1,16 @@ +namespace fbs.attributes; + +/// Mandatory attribute that applies to all kinds of objects: structs, enums, unions and even the +/// fields within. +/// +/// This defines a stable order between objects of the same kind, e.g. the order in which fields of a +/// struct should be laid out when generating code. +/// This is always required since flatbuffers works entirely with unordered maps internally, which +/// would result in flaky code generation. +/// +/// In unions, this effectively defines the arrow tag of each variant, since the tag depends on the +/// fields's order in the datatype! +/// +/// NOTE: We do not use flatbuffers' builtin `id` attribute as it only works on `table`s, whereas we +/// need a stable order for all kinds of things. +attribute "order"; diff --git a/crates/re_types/definitions/fbs/scalars.fbs b/crates/re_types/definitions/fbs/scalars.fbs new file mode 100644 index 000000000000..085df42e26db --- /dev/null +++ b/crates/re_types/definitions/fbs/scalars.fbs @@ -0,0 +1,13 @@ +/// Unions cannot directly refer to scalar types, they need to be wrapped in a struct or table +/// first. +/// This package provides pre-wrapped scalars that will be automatically flattened down to their +/// inner type by our parsers. +/// +/// Look e.g. for `fbs.scalars.Float32` in `objects.rs` to see this flatenning in action. + +namespace fbs.scalars; + +/// Flattens down to a 32-bit float. +struct Float32 { + v: float; +} diff --git a/crates/re_types/definitions/python/attributes.fbs b/crates/re_types/definitions/python/attributes.fbs new file mode 100644 index 000000000000..4160bdc0c429 --- /dev/null +++ b/crates/re_types/definitions/python/attributes.fbs @@ -0,0 +1,16 @@ +namespace python.attributes; + +/// Marks a field as transparent, meaning its type will be replaced by the underlying type. +/// +/// Only applies to fields whose type is an object with a single-field. +attribute "python.attr.transparent"; + +/// Defines the type aliases for a component, e.g. the types that make up `ComponentLike`. +/// +/// Only applies to structs/unions that are components. +attribute "python.attr.aliases"; + +/// Defines the array type aliases for a component, e.g. the types that make up `ComponentArrayLike`. +/// +/// Only applies to structs/unions that are components. +attribute "python.attr.array_aliases"; diff --git a/crates/re_types/definitions/rerun/archetypes.fbs b/crates/re_types/definitions/rerun/archetypes.fbs new file mode 100644 index 000000000000..e8f07787a567 --- /dev/null +++ b/crates/re_types/definitions/rerun/archetypes.fbs @@ -0,0 +1 @@ +namespace rerun.archetypes; diff --git a/crates/re_types/definitions/rust/attributes.fbs b/crates/re_types/definitions/rust/attributes.fbs new file mode 100644 index 000000000000..58a8fac6b070 --- /dev/null +++ b/crates/re_types/definitions/rust/attributes.fbs @@ -0,0 +1,15 @@ +namespace rust.attributes; + +/// Apply to a struct or table object to generate a tuple struct. +/// +/// The type definition of the target object must have exactly a single field. +attribute "rust.attr.tuple_struct"; + +/// Apply to any object to generate a #derive clause. +/// +/// The value of the attribute will be trimmed out but otherwise left as-is. +/// E.g. "rust.attr.derive": "Debug, Clone, Copy"`. +attribute "rust.attr.derive"; + +/// Apply to any object to generate a #repr clause with the specified value. +attribute "rust.attr.repr"; diff --git a/crates/re_types/source_hash.txt b/crates/re_types/source_hash.txt new file mode 100644 index 000000000000..a5681461859d --- /dev/null +++ b/crates/re_types/source_hash.txt @@ -0,0 +1,4 @@ +# This is a sha256 hash for all direct and indirect dependencies of this crate's build script. +# It can be safely removed at anytime to force the build script to run again. +# Check out build.rs to see how it's computed. +dae77f291d1698807cd865265cbb77731bd1aedf07c0968a6b0ac67c18f94590 \ No newline at end of file diff --git a/crates/re_types/src/archetypes/mod.rs b/crates/re_types/src/archetypes/mod.rs new file mode 100644 index 000000000000..cf43c16c6a0e --- /dev/null +++ b/crates/re_types/src/archetypes/mod.rs @@ -0,0 +1 @@ +// NOTE: This file was autogenerated by re_types_builder; DO NOT EDIT. diff --git a/crates/re_types/src/components/mod.rs b/crates/re_types/src/components/mod.rs new file mode 100644 index 000000000000..cf43c16c6a0e --- /dev/null +++ b/crates/re_types/src/components/mod.rs @@ -0,0 +1 @@ +// NOTE: This file was autogenerated by re_types_builder; DO NOT EDIT. diff --git a/crates/re_types/src/datatypes/mod.rs b/crates/re_types/src/datatypes/mod.rs new file mode 100644 index 000000000000..cf43c16c6a0e --- /dev/null +++ b/crates/re_types/src/datatypes/mod.rs @@ -0,0 +1 @@ +// NOTE: This file was autogenerated by re_types_builder; DO NOT EDIT. diff --git a/crates/re_types/src/lib.rs b/crates/re_types/src/lib.rs new file mode 100644 index 000000000000..e6ddbfc3f92f --- /dev/null +++ b/crates/re_types/src/lib.rs @@ -0,0 +1,104 @@ +//! The standard Rerun data types, component types, and archetypes. +//! +//! This crate contains both the IDL definitions for Rerun types (flatbuffers) as well as the code +//! generated from those using `re_types_builder`. +//! +//! +//! ### Organization +//! +//! - `definitions/` contains IDL definitions for all Rerun types (data, components, archetypes). +//! - `src/` contains the code generated for Rust. +//! - `rerun_py/rerun2/` (at the root of this workspace) contains the code generated for Python. +//! +//! While most of the code in this crate is auto-generated, some manual extensions are littered +//! throughout: look for files ending in `_ext.rs` or `_ext.py` (also see the "Extensions" section +//! of this document). +//! +//! +//! ### Build cache +//! +//! Updating either the source code of the code generator itself (`re_types_builder`) or any of the +//! .fbs files should re-trigger the code generation process the next time `re_types` is built. +//! Manual extension files will be left untouched. +//! +//! Caching is controlled by a versioning hash that is stored in `store_hash.txt`. +//! If you suspect something is wrong with the caching mechanism and that your changes aren't taken +//! into account when they should, try and remove `source_hash.txt`. +//! If that fixes the issue, you've found a bug. +//! +//! +//! ### How-to: add a new datatype/component/archetype +//! +//! Create the appropriate .fbs file in the appropriate place, and make sure it gets included in +//! some way (most likely indirectly) by `archetypes.fbs`, which is the main entrypoint for +//! codegen. +//! Generally, the easiest thing to do is to add your new type to one of the centralized manifests, +//! e.g. for a new component, include it into `components.fbs`. +//! +//! Your file should get picked up automatically by the code generator. +//! Once the code for your new component has been generated, implement whatever extensions you need +//! and make sure to tests any custom constructors you add. +//! +//! +//! ### How-to: remove an existing datatype/component/archetype +//! +//! Simply get rid of the type in question and rebuild `re_types` to trigger codegen. +//! +//! Beware though: if you remove a whole definition file re-running codegen will not remove the +//! associated generated files, you'll have to do that yourself. +//! +//! +//! ### Extensions +//! +//! +//! #### Rust +//! +//! Generated Rust code can be manually extended by adding sibling files with the `_ext.rs` +//! prefix. E.g. to extend `vec2d.rs`, create a `vec2d_ext.rs`. +//! +//! Trigger the codegen (e.g. by removing `source_hash.txt`) to generate the right `mod` clauses +//! automatically. +//! +//! The simplest way to get started is to look at any of the existing examples. +//! +//! +//! #### Python +//! +//! Generated Python code can be manually extended by adding a sibling file with the `_ext.py` +//! prefix. E.g. to extend `vec2d.py`, create a `vec2d_ext.py`. +//! +//! This sibling file needs to implement an extension class that is mixed in with the +//! auto-generated class. +//! The simplest way to get started is to look at any of the existing examples. + +use std::borrow::Cow; + +/// A [`Datatype`] describes plain old data. +pub trait Datatype { + fn name() -> Cow<'static, str>; + + fn to_arrow_datatype() -> arrow2::datatypes::DataType; +} + +pub trait Component { + fn name() -> Cow<'static, str>; + + fn to_arrow_datatype() -> arrow2::datatypes::DataType; +} + +pub trait Archetype { + fn name() -> Cow<'static, str>; + + fn required_components() -> Vec>; + fn recommended_components() -> Vec>; + fn optional_components() -> Vec>; + + fn to_arrow_datatypes() -> Vec; +} + +/// Number of decimals shown for all vector display methods. +pub const DISPLAY_PRECISION: usize = 3; + +pub mod archetypes; +pub mod components; +pub mod datatypes;