Codegen/IDL 2: introduce re_types_builder (#2363)

`re_types_builder` provides the build-time tools to parse and inspect
flatbuffers schemas, and to generate Python & Rust code from those
definitions.

You can safely skip `crates/re_types_builder/definitions/reflection.fbs`
& `crates/re_types_builder/src/reflection.rs` which are just the
auto-generated flatbuffers code to provide the runtime reflection tools
(minus 3.2k LoCs, yay!).

This is by far the biggest PR of the bunch. It's also the least
important one in some ways, considering none of this code sees the light
of runtime.

The easiest way to make sense of `codegen/python.rs` and
`codegen/rust.rs` is to check out the output code:
- #2374 
- #2375  

⚠️ This now requires `flatc` to be in $PATH, but only for
contributors, not end users.
Even for contributors, `flatc` won't be needed unless they edit some of
the .fbs files.

---

Codegen/IDL PR series:
- #2362
- #2363
- #2369
- #2370 
- #2374 
- #2375 
- #2410
- #2432 

---

_Excerpt from the crate-level docs_

### Organization

The code generation process happens in 4 phases.

#### 1. Generate binary reflection data from flatbuffers definitions.

All this does is invoke the flatbuffers compiler (`flatc`) with the
right flags in order to
generate the binary dumps.

Look for `compile_binary_schemas` in the code.

#### 2. Run the semantic pass.

The semantic pass transforms the low-level raw reflection data generated
by the first phase
into higher level objects that are much easier to inspect/manipulate and
overall friendler
to work with.

Look for `objects.rs`.

#### 3. Fill the Arrow registry.

The Arrow registry keeps track of all type definitions and maps them to
Arrow datatypes.

Look for `arrow_registry.rs`.

#### 4. Run the actual codegen pass for a given language.

We currently have two different codegen passes implemented at the
moment: Python & Rust.

Codegen passes use the semantic objects from phase two and the registry
from phase three
in order to generate user-facing code for Rerun's SDKs.

These passes are intentionally implemented using a very low-tech
no-frills approach (stitch
strings together, make liberal use of `unimplemented`, etc) that keep
them flexible in the
face of ever changing needs in the generated code.

Look for `codegen/python.rs` and `codegen/rust.rs`.


### Error handling

Keep in mind: this is all _build-time_ code that will never see the
light of runtime.
There is therefore no need for fancy error handling in this crate: all
errors are fatal to the
build anyway.

Make sure to crash as soon as possible when something goes wrong and to
attach all the
appropriate/available context using `anyhow`'s `with_context` (e.g.
always include the
fully-qualified name of the faulty type/field) and you're good to go.


### Testing

Same comment as with error handling: this code becomes irrelevant at
runtime, and so testing it
brings very little value.

Make sure to test the behavior of its output though: `re_types`!

---------

Co-authored-by: Andreas Reich <r_andreas2@web.de>

.vscode/extensions.json

@@ -1,19 +1,20 @@
 {
-    // See https://go.microsoft.com/fwlink/?LinkId=827846
-    // for the documentation about the extensions.json format
-    "recommendations": [
-        "charliermarsh.ruff",
-        "github.vscode-github-actions",
-        "ms-python.python",
-        "ms-vsliveshare.vsliveshare",
-        "polymeilex.wgsl",
-        "rust-lang.rust-analyzer",
-        "serayuzgur.crates",
-        "streetsidesoftware.code-spell-checker",
-        "tamasfe.even-better-toml",
-        "vadimcn.vscode-lldb",
-        "wayou.vscode-todo-highlight",
-        "webfreak.debug",
-        "zxh404.vscode-proto3",
-    ]
+  // See https://go.microsoft.com/fwlink/?LinkId=827846
+  // for the documentation about the extensions.json format
+  "recommendations": [
+    "charliermarsh.ruff",
+    "github.vscode-github-actions",
+    "ms-python.python",
+    "ms-vsliveshare.vsliveshare",
+    "polymeilex.wgsl",
+    "rust-lang.rust-analyzer",
+    "serayuzgur.crates",
+    "streetsidesoftware.code-spell-checker",
+    "tamasfe.even-better-toml",
+    "vadimcn.vscode-lldb",
+    "wayou.vscode-todo-highlight",
+    "webfreak.debug",
+    "zxh404.vscode-proto3",
+    "gaborv.flatbuffers"
+  ]
 }

.vscode/settings.json

@@ -27,6 +27,7 @@
         "bindgroup",
         "colormap",
         "emath",
+        "flatbuffers",
         "framebuffer",
         "hoverable",
         "ilog",

Cargo.toml

@@ -56,6 +56,7 @@ re_tensor_ops = { path = "crates/re_tensor_ops", version = "0.7.0-alpha.0", defa
 re_time_panel = { path = "crates/re_time_panel", version = "=0.7.0-alpha.0", default-features = false }
 re_tracing = { path = "crates/re_tracing", version = "0.7.0-alpha.0", default-features = false }
 re_tuid = { path = "crates/re_tuid", version = "0.7.0-alpha.0", default-features = false }
+re_types_builder = { path = "crates/re_types_builder", version = "=0.7.0-alpha.0", default-features = false }
 re_ui = { path = "crates/re_ui", version = "0.7.0-alpha.0", default-features = false }
 re_viewer = { path = "crates/re_viewer", version = "0.7.0-alpha.0", default-features = false }
 re_viewer_context = { path = "crates/re_viewer_context", version = "0.7.0-alpha.0", default-features = false }
@@ -109,6 +110,7 @@ time = { version = "0.3", default-features = false, features = [
 ] }
 tinyvec = { version = "1.6", features = ["alloc", "rustc_1_55"] }
 tokio = { version = "1.24", default-features = false }
+unindent = "0.1"
 vec1 = "1.8"
 web-time = "0.2.0"
 wgpu = { version = "0.16.1" }

_typos.toml

@@ -13,6 +13,7 @@ extend-exclude = [
 [default.extend-words]
 lod = "lod" # level-of-detail
 teh = "teh" # part of @teh-cmc
+ND = "ND"   # np.NDArray
 
 # American English:
 grey = "gray"

crates/re_renderer/Cargo.toml

@@ -92,7 +92,7 @@ winit = "0.28.1"
 zip = { version = "0.6", default-features = false, features = ["deflate"] }
 
 # For tests:
-unindent = "0.1"
+unindent.workspace = true
 
 # native
 [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]

crates/re_types_builder/Cargo.toml

@@ -0,0 +1,38 @@
+[package]
+name = "re_types_builder"
+authors.workspace = true
+description = "Generates code for Rerun's SDKs from flatbuffers definitions."
+edition.workspace = true
+homepage.workspace = true
+include.workspace = true
+license.workspace = true
+publish = true
+readme = "README.md"
+repository.workspace = true
+rust-version.workspace = true
+version.workspace = true
+
+
+[package.metadata.docs.rs]
+all-features = true
+
+
+[dependencies]
+
+# External
+anyhow.workspace = true
+arrow2.workspace = true
+convert_case = "0.6"
+flatbuffers = "23.0"
+indent = "0.1"
+unindent.workspace = true
+xshell = "0.2"
+
+
+[build-dependencies]
+
+# Rerun
+re_build_tools.workspace = true
+
+# External
+xshell = "0.2"

crates/re_types_builder/README.md

@@ -0,0 +1,13 @@
+# re_types_builder
+
+Part of the [`rerun`](https://github.com/rerun-io/rerun) family of crates.
+
+[![Latest version](https://img.shields.io/crates/v/re_types_builder.svg)](https://crates.io/crates/re_types_builder)
+[![Documentation](https://docs.rs/re_types_builder/badge.svg)](https://docs.rs/re_types_builder)
+![MIT](https://img.shields.io/badge/license-MIT-blue.svg)
+![Apache](https://img.shields.io/badge/license-Apache-blue.svg)
+
+This crate implements Rerun's code generation tools.
+
+These tools translate language-agnostic IDL definitions (flatbuffers) into code.
+They are invoked from `re_types`' build script (`build.rs`).

crates/re_types_builder/build.rs

@@ -0,0 +1,76 @@
+//! Generates flatbuffers reflection code from `reflection.fbs`.
+
+use xshell::{cmd, Shell};
+
+use re_build_tools::{
+    compute_file_hash, is_tracked_env_var_set, read_versioning_hash, write_versioning_hash,
+};
+
+// ---
+
+// NOTE: Don't need to add extra context to xshell invocations, it does so on its own.
+
+const SOURCE_HASH_PATH: &str = "./source_hash.txt";
+const FBS_REFLECTION_DEFINITION_PATH: &str = "./definitions/reflection.fbs";
+
+fn main() {
+    if std::env::var("CI").is_ok() {
+        // Don't run on CI!
+        //
+        // The code we're generating here is actual source code that gets committed into the
+        // repository.
+        return;
+    }
+
+    if !is_tracked_env_var_set("IS_IN_RERUN_WORKSPACE") {
+        // Only run if we are in the rerun workspace, not on users machines.
+        return;
+    }
+    if is_tracked_env_var_set("RERUN_IS_PUBLISHING") {
+        // We don't need to rebuild - we should have done so beforehand!
+        // See `RELEASES.md`
+        return;
+    }
+
+    // We're building an actual build graph here, and Cargo has no idea about it.
+    //
+    // Worse: some nodes in our build graph actually output artifacts into the src/ directory,
+    // which Cargo always interprets as "need to rebuild everything ASAP", leading to an infinite
+    // feedback loop.
+    //
+    // For these reasons, we manually compute and track signature hashes for the graph nodes we
+    // depend on, and make sure to exit early if everything's already up to date.
+    let cur_hash = read_versioning_hash(SOURCE_HASH_PATH);
+    let new_hash = compute_file_hash(FBS_REFLECTION_DEFINITION_PATH);
+
+    // Leave these be please, very useful when debugging.
+    eprintln!("cur_hash: {cur_hash:?}");
+    eprintln!("new_hash: {new_hash:?}");
+
+    if let Some(cur_hash) = cur_hash {
+        if cur_hash == new_hash {
+            // Source definition hasn't changed, no need to do anything.
+            return;
+        }
+    }
+
+    // NOTE: This requires `flatc` to be in $PATH, but only for contributors, not end users.
+    // Even for contributors, `flatc` won't be needed unless they edit some of the .fbs files.
+    let sh = Shell::new().unwrap();
+    cmd!(
+        sh,
+        "flatc -o src/ --rust --gen-onefile --filename-suffix '' {FBS_REFLECTION_DEFINITION_PATH}"
+    )
+    .run()
+    .unwrap();
+
+    // NOTE: We're purposefully ignoring the error here.
+    //
+    // In the very unlikely chance that the user doesn't have the `fmt` component installed,
+    // there's still no good reason to fail the build.
+    //
+    // The CI will catch the unformatted file at PR time and complain appropriately anyhow.
+    cmd!(sh, "cargo fmt").run().ok();
+
+    write_versioning_hash(SOURCE_HASH_PATH, new_hash);
+}