C++ codegen (#2678)

Part of:
* #2647 

### What
Adds minimal code-gen for C++. It generates a `.hpp/.cpp` pair for each
type, plus a module file each for datatypes, components, and archetypes.

### Checklist
* [x] I have read and agree to [Contributor
Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and
the [Code of
Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
* [x] I've included a screenshot or gif (if applicable)
* [x] I have tested [demo.rerun.io](https://demo.rerun.io/pr/2678) (if
applicable)

- [PR Build Summary](https://build.rerun.io/pr/2678)
- [Docs preview](https://rerun.io/preview/pr%3Aemilk%2Fcpp-codegen/docs)
- [Examples
preview](https://rerun.io/preview/pr%3Aemilk%2Fcpp-codegen/examples)

---------

Co-authored-by: Andreas Reich <r_andreas2@web.de>

.gitignore

@@ -1,8 +1,14 @@
 .DS_Store
 
 # C++ and CMake stuff:
-**/CMakeFiles/
+*.bin
+*.o
 **/build/
+**/CMakeFiles/
+**/CMakeCache.txt
+**/Makefile
+**/cmake_install.cmake
+_deps
 
 # Rust compile target directory:
 **/target

.vscode/extensions.json

@@ -5,7 +5,10 @@
     "charliermarsh.ruff",
     "gaborv.flatbuffers",
     "github.vscode-github-actions",
+    "josetr.cmake-language-support-vscode",
     "ms-python.python",
+    "ms-vscode.cmake-tools",
+    "ms-vscode.cpptools-extension-pack",
     "ms-vsliveshare.vsliveshare",
     "polymeilex.wgsl",
     "rust-lang.rust-analyzer",
@@ -15,7 +18,7 @@
     "vadimcn.vscode-lldb",
     "wayou.vscode-todo-highlight",
     "webfreak.debug",
-    "xaver.clang-format", // C++ formatter
+    "xaver.clang-format",  // C++ formatter
     "zxh404.vscode-proto3",
   ]
 }

CMakeLists.txt

@@ -0,0 +1,6 @@
+cmake_minimum_required(VERSION 3.16)
+
+project(rerun_cpp_proj LANGUAGES CXX)
+
+add_subdirectory(rerun_cpp) # The Rerun C++ SDK library
+add_subdirectory(examples/cpp/minimal)

crates/re_types/Cargo.toml

@@ -62,4 +62,5 @@ re_build_tools.workspace = true
 re_types_builder.workspace = true
 
 # External
+rayon.workspace = true
 xshell = "0.2"

crates/re_types/build.rs

@@ -18,6 +18,7 @@ const SOURCE_HASH_PATH: &str = "./source_hash.txt";
 const DEFINITIONS_DIR_PATH: &str = "./definitions";
 const ENTRYPOINT_PATH: &str = "./definitions/rerun/archetypes.fbs";
 const DOC_EXAMPLES_DIR_PATH: &str = "../../docs/code-examples";
+const CPP_OUTPUT_DIR_PATH: &str = "../../rerun_cpp/src";
 const RUST_OUTPUT_DIR_PATH: &str = ".";
 const PYTHON_OUTPUT_DIR_PATH: &str = "../../rerun_py/rerun_sdk/rerun/_rerun2";
 
@@ -100,27 +101,24 @@ fn main() {
         panic!("re_types' fbs definitions and generated code are out-of-sync!");
     }
 
-    let sh = Shell::new().unwrap();
-
     // passes 1 through 3: bfbs, semantic, arrow registry
     let (objects, arrow_registry) =
         re_types_builder::generate_lang_agnostic(DEFINITIONS_DIR_PATH, ENTRYPOINT_PATH);
 
-    re_types_builder::generate_rust_code(RUST_OUTPUT_DIR_PATH, &objects, &arrow_registry);
-
-    // We need to run `cago fmt` several times because it is not idempotent!
-    // See https://github.com/rust-lang/rustfmt/issues/5824
-    for _ in 0..2 {
-        // NOTE: We're purposefully ignoring the error here.
-        //
-        // In the very unlikely chance that the user doesn't have the `fmt` component installed,
-        // there's still no good reason to fail the build.
-        //
-        // The CI will catch the unformatted file at PR time and complain appropriately anyhow.
-        cmd!(sh, "cargo fmt -p re_types").run().ok();
-    }
+    join3(
+        || re_types_builder::generate_cpp_code(CPP_OUTPUT_DIR_PATH, &objects, &arrow_registry),
+        || re_types_builder::generate_rust_code(RUST_OUTPUT_DIR_PATH, &objects, &arrow_registry),
+        || generate_and_format_python_code(&objects, &arrow_registry),
+    );
 
-    re_types_builder::generate_python_code(PYTHON_OUTPUT_DIR_PATH, &objects, &arrow_registry);
+    write_versioning_hash(SOURCE_HASH_PATH, new_hash);
+}
+
+fn generate_and_format_python_code(
+    objects: &re_types_builder::Objects,
+    arrow_registry: &re_types_builder::ArrowRegistry,
+) {
+    re_types_builder::generate_python_code(PYTHON_OUTPUT_DIR_PATH, objects, arrow_registry);
 
     let pyproject_path = PathBuf::from(PYTHON_OUTPUT_DIR_PATH)
         .parent()
@@ -133,6 +131,13 @@ fn main() {
         .to_string_lossy()
         .to_string();
 
+    // TODO(emilk): format the python code _before_ writing them to file instead,
+    // just like we do with C++ and Rust.
+    // This should be doable py piping the code of each file to black/ruff via stdin.
+    // Why? Right now the python code is written once, then changed, which means
+    // it is in flux while building, which creates weird phantom git diffs for a few seconds,
+    // and also update the modified file stamps.
+
     // NOTE: This requires both `black` and `ruff` to be in $PATH, but only for contributors,
     // not end users.
     // Even for contributors, `black` and `ruff` won't be needed unless they edit some of the
@@ -146,11 +151,15 @@ fn main() {
     // 2) Call ruff, which requires line-lengths to be correct
     // 3) Call black again to cleanup some whitespace issues ruff might introduce
 
+    let sh = Shell::new().unwrap();
     call_black(&sh, &pyproject_path);
     call_ruff(&sh, &pyproject_path);
     call_black(&sh, &pyproject_path);
+}
 
-    write_versioning_hash(SOURCE_HASH_PATH, new_hash);
+// Do 3 things in parallel
+fn join3(a: impl FnOnce() + Send, b: impl FnOnce() + Send, c: impl FnOnce() + Send) {
+    rayon::join(a, || rayon::join(b, c));
 }
 
 fn call_black(sh: &Shell, pyproject_path: &String) {

crates/re_types/source_hash.txt

@@ -1,4 +1,4 @@
 # This is a sha256 hash for all direct and indirect dependencies of this crate's build script.
 # It can be safely removed at anytime to force the build script to run again.
 # Check out build.rs to see how it's computed.
-7b07a9ead58634313a05cbe4384f177af85241e46701dea4e14fa46ac85916a3
+48b1929d5cb17125eaae7733df116017fed8e27f5202d2365146e68e9a1a5b16

crates/re_types_builder/Cargo.toml

@@ -23,12 +23,15 @@ all-features = true
 anyhow.workspace = true
 arrow2.workspace = true
 camino.workspace = true
+clang-format = "0.1"
 convert_case = "0.6"
 flatbuffers = "23.0"
 indent = "0.1"
 itertools.workspace = true
 proc-macro2 = { version = "1.0", default-features = false }
 quote = "1.0"
+rayon.workspace = true
+rust-format = "0.3"
 syn = "2.0"
 unindent.workspace = true
 xshell = "0.2"

crates/re_types_builder/src/codegen/cpp.rs

@@ -0,0 +1,175 @@
+use std::collections::BTreeSet;
+
+use anyhow::Context as _;
+use camino::{Utf8Path, Utf8PathBuf};
+use proc_macro2::TokenStream;
+use quote::{format_ident, quote};
+use rayon::prelude::*;
+
+use crate::{codegen::AUTOGEN_WARNING, ArrowRegistry, ObjectKind, Objects};
+
+const NEWLINE_TOKEN: &str = "RE_TOKEN_NEWLINE";
+
+pub struct CppCodeGenerator {
+    output_path: Utf8PathBuf,
+}
+
+impl CppCodeGenerator {
+    pub fn new(output_path: impl Into<Utf8PathBuf>) -> Self {
+        Self {
+            output_path: output_path.into(),
+        }
+    }
+
+    fn generate_folder(
+        &self,
+        objects: &Objects,
+        arrow_registry: &ArrowRegistry,
+        object_kind: ObjectKind,
+        folder_name: &str,
+    ) -> BTreeSet<Utf8PathBuf> {
+        let folder_path = self.output_path.join(folder_name);
+        std::fs::create_dir_all(&folder_path)
+            .with_context(|| format!("{folder_path:?}"))
+            .unwrap();
+
+        let mut filepaths = BTreeSet::default();
+
+        // Generate folder contents:
+        let ordered_objects = objects.ordered_objects(object_kind.into());
+        for &obj in &ordered_objects {
+            let filename = obj.snake_case_name();
+            let (hpp, cpp) = generate_hpp_cpp(objects, arrow_registry, obj);
+            for (extension, tokens) in [("hpp", hpp), ("cpp", cpp)] {
+                let string = string_from_token_stream(&tokens, obj.relative_filepath());
+                let filepath = folder_path.join(format!("{filename}.{extension}"));
+                write_file(&filepath, string);
+                let inserted = filepaths.insert(filepath);
+                assert!(
+                    inserted,
+                    "Multiple objects with the same name: {:?}",
+                    obj.name
+                );
+            }
+        }
+
+        {
+            // Generate module file that includes all the headers:
+            let hash = quote! { # };
+            let pragma_once = pragma_once();
+            let header_file_names = ordered_objects
+                .iter()
+                .map(|obj| format!("{folder_name}/{}.hpp", obj.snake_case_name()));
+            let tokens = quote! {
+                #pragma_once
+                #(#hash include #header_file_names "RE_TOKEN_NEWLINE")*
+            };
+            let filepath = folder_path
+                .parent()
+                .unwrap()
+                .join(format!("{folder_name}.hpp"));
+            let string = string_from_token_stream(&tokens, None);
+            write_file(&filepath, string);
+            filepaths.insert(filepath);
+        }
+
+        // Clean up old files:
+        for entry in std::fs::read_dir(folder_path).unwrap().flatten() {
+            let filepath = Utf8PathBuf::try_from(entry.path()).unwrap();
+            if !filepaths.contains(&filepath) {
+                std::fs::remove_file(filepath).ok();
+            }
+        }
+
+        filepaths
+    }
+}
+
+impl crate::CodeGenerator for CppCodeGenerator {
+    fn generate(
+        &mut self,
+        objects: &Objects,
+        arrow_registry: &ArrowRegistry,
+    ) -> BTreeSet<Utf8PathBuf> {
+        ObjectKind::ALL
+            .par_iter()
+            .map(|object_kind| {
+                let folder_name = object_kind.plural_snake_case();
+                self.generate_folder(objects, arrow_registry, *object_kind, folder_name)
+            })
+            .flatten()
+            .collect()
+    }
+}
+
+fn string_from_token_stream(token_stream: &TokenStream, source_path: Option<&Utf8Path>) -> String {
+    let mut code = String::new();
+    code.push_str(&format!("// {AUTOGEN_WARNING}\n"));
+    if let Some(source_path) = source_path {
+        code.push_str(&format!("// Based on {source_path:?}\n"));
+    }
+
+    code.push('\n');
+    code.push_str(
+        &token_stream
+            .to_string()
+            .replace(&format!("{NEWLINE_TOKEN:?}"), "\n"),
+    );
+    code.push('\n');
+
+    // clang_format has a bit of an ugly API: https://github.com/KDAB/clang-format-rs/issues/3
+    clang_format::CLANG_FORMAT_STYLE
+        .set(clang_format::ClangFormatStyle::File)
+        .ok();
+    code = clang_format::clang_format(&code).expect("Failed to run clang-format");
+
+    code
+}
+
+fn write_file(filepath: &Utf8PathBuf, code: String) {
+    if let Ok(existing) = std::fs::read_to_string(filepath) {
+        if existing == code {
+            // Don't touch the timestamp unnecessarily
+            return;
+        }
+    }
+
+    std::fs::write(filepath, code)
+        .with_context(|| format!("{filepath}"))
+        .unwrap();
+}
+
+fn generate_hpp_cpp(
+    _objects: &Objects,
+    _arrow_registry: &ArrowRegistry,
+    obj: &crate::Object,
+) -> (TokenStream, TokenStream) {
+    let obj_kind_ident = format_ident!("{}", obj.kind.plural_snake_case());
+
+    let pascal_case_name = &obj.name;
+    let pascal_case_ident = format_ident!("{pascal_case_name}");
+    let snake_case_name = obj.snake_case_name();
+
+    let hash = quote! { # };
+    let pragma_once = pragma_once();
+    let header_file_name = format!("{snake_case_name}.hpp");
+
+    let hpp = quote! {
+        #pragma_once
+        namespace rr {
+            namespace #obj_kind_ident {
+                struct #pascal_case_ident { };
+            }
+        }
+    };
+    let cpp = quote! { #hash include #header_file_name };
+
+    (hpp, cpp)
+}
+
+fn pragma_once() -> TokenStream {
+    let hash = quote! { # };
+    quote! {
+        #hash pragma once #NEWLINE_TOKEN #NEWLINE_TOKEN
+    }
+}

crates/re_types_builder/src/codegen/mod.rs

@@ -20,8 +20,10 @@ pub const AUTOGEN_WARNING: &str =
 mod common;
 use self::common::{get_documentation, StringExt};
 
+mod cpp;
 mod python;
 mod rust;
 
+pub use self::cpp::CppCodeGenerator;
 pub use self::python::PythonCodeGenerator;
 pub use self::rust::RustCodeGenerator;