From 7832e4cab533e25166a1dcbf52798347e65b8a07 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Wed, 26 Jul 2023 13:08:47 +0200 Subject: [PATCH] C++ codegen arrow serialize non-union components/datatypes without nested rerun types (#2820) * Part of #2647 * Next step after: #2766 ### What Generates serialization for all non-union components and datatypes except for datatypes nested in structs & unions. Uses new serialization methods for 2D & 3D point logging. Ergonomics are pretty poor at this point, handling this will be a follow-up. Commit history got a bit messy at some point - **not** recommending commit by commit reviewing ;) Next Steps: * work towards Point2D roundtrip * builder methods on archetype * make logging on Recording stream nice'ish * basic testing * serialize unions * serialize datatypes nested in unions, structs and lists * more testing & roundtripping ### Checklist * [x] I have read and agree to [Contributor Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md) * [x] I've included a screenshot or gif (if applicable) * [x] I have tested [demo.rerun.io](https://demo.rerun.io/pr/2820) (if applicable) - [PR Build Summary](https://build.rerun.io/pr/2820) - [Docs preview](https://rerun.io/preview/pr%3Aandreas%2Fcpp-codegen%2Farrow-serialize-structs/docs) - [Examples preview](https://rerun.io/preview/pr%3Aandreas%2Fcpp-codegen%2Farrow-serialize-structs/examples) --- .clang-format | 4 +- crates/re_types/source_hash.txt | 2 +- .../src/codegen/cpp/forward_decl.rs | 1 + .../src/codegen/cpp/method.rs | 6 +- .../re_types_builder/src/codegen/cpp/mod.rs | 537 ++++++++++++++++-- crates/rerun_c/src/rerun.h | 5 +- examples/cpp/minimal/main.cpp | 62 +- rerun_cpp/src/archetypes/affix_fuzzer1.hpp | 1 + rerun_cpp/src/archetypes/points2d.hpp | 1 + rerun_cpp/src/archetypes/points3d.hpp | 1 + rerun_cpp/src/archetypes/transform3d.hpp | 1 + rerun_cpp/src/components/affix_fuzzer1.cpp | 32 ++ rerun_cpp/src/components/affix_fuzzer1.hpp | 16 +- rerun_cpp/src/components/affix_fuzzer10.cpp | 33 ++ rerun_cpp/src/components/affix_fuzzer10.hpp | 16 +- rerun_cpp/src/components/affix_fuzzer11.cpp | 44 ++ rerun_cpp/src/components/affix_fuzzer11.hpp | 16 +- rerun_cpp/src/components/affix_fuzzer12.cpp | 39 ++ rerun_cpp/src/components/affix_fuzzer12.hpp | 16 +- rerun_cpp/src/components/affix_fuzzer13.cpp | 43 ++ rerun_cpp/src/components/affix_fuzzer13.hpp | 16 +- rerun_cpp/src/components/affix_fuzzer14.cpp | 31 + rerun_cpp/src/components/affix_fuzzer14.hpp | 17 +- rerun_cpp/src/components/affix_fuzzer15.cpp | 26 + rerun_cpp/src/components/affix_fuzzer15.hpp | 17 +- rerun_cpp/src/components/affix_fuzzer16.cpp | 36 +- rerun_cpp/src/components/affix_fuzzer16.hpp | 16 +- rerun_cpp/src/components/affix_fuzzer17.cpp | 36 +- rerun_cpp/src/components/affix_fuzzer17.hpp | 19 +- rerun_cpp/src/components/affix_fuzzer18.cpp | 36 +- rerun_cpp/src/components/affix_fuzzer18.hpp | 19 +- rerun_cpp/src/components/affix_fuzzer19.cpp | 32 ++ rerun_cpp/src/components/affix_fuzzer19.hpp | 16 +- rerun_cpp/src/components/affix_fuzzer2.cpp | 32 ++ rerun_cpp/src/components/affix_fuzzer2.hpp | 16 +- rerun_cpp/src/components/affix_fuzzer3.cpp | 32 ++ rerun_cpp/src/components/affix_fuzzer3.hpp | 16 +- rerun_cpp/src/components/affix_fuzzer4.cpp | 27 + rerun_cpp/src/components/affix_fuzzer4.hpp | 16 +- rerun_cpp/src/components/affix_fuzzer5.cpp | 27 + rerun_cpp/src/components/affix_fuzzer5.hpp | 16 +- rerun_cpp/src/components/affix_fuzzer6.cpp | 27 + rerun_cpp/src/components/affix_fuzzer6.hpp | 16 +- rerun_cpp/src/components/affix_fuzzer7.cpp | 36 +- rerun_cpp/src/components/affix_fuzzer7.hpp | 16 +- rerun_cpp/src/components/affix_fuzzer8.cpp | 33 ++ rerun_cpp/src/components/affix_fuzzer8.hpp | 16 +- rerun_cpp/src/components/affix_fuzzer9.cpp | 28 + rerun_cpp/src/components/affix_fuzzer9.hpp | 16 +- rerun_cpp/src/components/class_id.cpp | 26 + rerun_cpp/src/components/class_id.hpp | 16 +- rerun_cpp/src/components/color.cpp | 26 + rerun_cpp/src/components/color.hpp | 16 +- rerun_cpp/src/components/draw_order.cpp | 26 + rerun_cpp/src/components/draw_order.hpp | 16 +- rerun_cpp/src/components/instance_key.cpp | 26 + rerun_cpp/src/components/instance_key.hpp | 16 +- rerun_cpp/src/components/keypoint_id.cpp | 26 + rerun_cpp/src/components/keypoint_id.hpp | 16 +- rerun_cpp/src/components/label.cpp | 28 + rerun_cpp/src/components/label.hpp | 16 +- rerun_cpp/src/components/point2d.cpp | 32 ++ rerun_cpp/src/components/point2d.hpp | 16 +- rerun_cpp/src/components/point3d.cpp | 32 ++ rerun_cpp/src/components/point3d.hpp | 16 +- rerun_cpp/src/components/radius.cpp | 26 + rerun_cpp/src/components/radius.hpp | 16 +- rerun_cpp/src/components/transform3d.cpp | 31 + rerun_cpp/src/components/transform3d.hpp | 16 +- rerun_cpp/src/datatypes/affix_fuzzer1.cpp | 163 +++++- rerun_cpp/src/datatypes/affix_fuzzer1.hpp | 16 +- rerun_cpp/src/datatypes/affix_fuzzer2.cpp | 33 ++ rerun_cpp/src/datatypes/affix_fuzzer2.hpp | 16 +- rerun_cpp/src/datatypes/affix_fuzzer3.cpp | 68 ++- rerun_cpp/src/datatypes/affix_fuzzer3.hpp | 12 +- rerun_cpp/src/datatypes/affix_fuzzer4.cpp | 72 ++- rerun_cpp/src/datatypes/affix_fuzzer4.hpp | 19 +- rerun_cpp/src/datatypes/affix_fuzzer5.cpp | 44 +- rerun_cpp/src/datatypes/affix_fuzzer5.hpp | 16 +- rerun_cpp/src/datatypes/angle.cpp | 36 ++ rerun_cpp/src/datatypes/angle.hpp | 12 +- rerun_cpp/src/datatypes/flattened_scalar.cpp | 38 ++ rerun_cpp/src/datatypes/flattened_scalar.hpp | 16 +- rerun_cpp/src/datatypes/mat3x3.cpp | 41 +- rerun_cpp/src/datatypes/mat3x3.hpp | 15 +- rerun_cpp/src/datatypes/mat4x4.cpp | 41 +- rerun_cpp/src/datatypes/mat4x4.hpp | 15 +- rerun_cpp/src/datatypes/point2d.cpp | 48 ++ rerun_cpp/src/datatypes/point2d.hpp | 16 +- rerun_cpp/src/datatypes/point3d.cpp | 57 ++ rerun_cpp/src/datatypes/point3d.hpp | 16 +- rerun_cpp/src/datatypes/quaternion.cpp | 40 +- rerun_cpp/src/datatypes/quaternion.hpp | 16 +- rerun_cpp/src/datatypes/rotation3d.cpp | 52 +- rerun_cpp/src/datatypes/rotation3d.hpp | 12 +- .../src/datatypes/rotation_axis_angle.cpp | 37 ++ .../src/datatypes/rotation_axis_angle.hpp | 17 +- rerun_cpp/src/datatypes/scale3d.cpp | 36 ++ rerun_cpp/src/datatypes/scale3d.hpp | 12 +- rerun_cpp/src/datatypes/transform3d.cpp | 57 +- rerun_cpp/src/datatypes/transform3d.hpp | 18 +- .../src/datatypes/translation_and_mat3x3.cpp | 52 +- .../src/datatypes/translation_and_mat3x3.hpp | 17 +- .../translation_rotation_scale3d.cpp | 63 +- .../translation_rotation_scale3d.hpp | 17 +- rerun_cpp/src/datatypes/vec2d.cpp | 41 +- rerun_cpp/src/datatypes/vec2d.hpp | 15 +- rerun_cpp/src/datatypes/vec3d.cpp | 41 +- rerun_cpp/src/datatypes/vec3d.hpp | 15 +- rerun_cpp/src/datatypes/vec4d.cpp | 41 +- rerun_cpp/src/datatypes/vec4d.hpp | 15 +- rerun_cpp/src/recording_stream.cpp | 6 +- rerun_cpp/src/recording_stream.hpp | 11 +- rerun_cpp/src/rerun.cpp | 28 +- 114 files changed, 3096 insertions(+), 375 deletions(-) diff --git a/.clang-format b/.clang-format index 5ae127526eed..23ca7b00fcd8 100644 --- a/.clang-format +++ b/.clang-format @@ -3,6 +3,8 @@ BasedOnStyle: Google # Make it slightly more similar to Rust. # Based loosely on https://gist.github.com/YodaEmbedding/c2c77dc693d11f3734d78489f9a6eea4 AccessModifierOffset: -2 +AlignAfterOpenBracket: BlockIndent +AllowAllArgumentsOnNextLine: false AllowShortBlocksOnASingleLine: false AllowShortCaseLabelsOnASingleLine: false AllowShortFunctionsOnASingleLine: Empty @@ -19,9 +21,9 @@ InsertTrailingCommas: Wrapped MaxEmptyLinesToKeep: 1 NamespaceIndentation: All PointerAlignment: Left +ReflowComments: true SeparateDefinitionBlocks: Always SpacesBeforeTrailingComments: 1 -ReflowComments: true # Don't change include blocks, we want to control this manually. # Sorting headers however is allowed as all our headers should be standalone. diff --git a/crates/re_types/source_hash.txt b/crates/re_types/source_hash.txt index e7a5b43275d8..90386b12beae 100644 --- a/crates/re_types/source_hash.txt +++ b/crates/re_types/source_hash.txt @@ -1,4 +1,4 @@ # This is a sha256 hash for all direct and indirect dependencies of this crate's build script. # It can be safely removed at anytime to force the build script to run again. # Check out build.rs to see how it's computed. -b60f0b918bc9d49fcebd79c19baeda65279d8c205bbc1367d270b92fd3395848 +360fb26536a5ab8641566344660cc6df17d791db89059100018fd7f995da9cdc diff --git a/crates/re_types_builder/src/codegen/cpp/forward_decl.rs b/crates/re_types_builder/src/codegen/cpp/forward_decl.rs index 8498e27c3e1f..6757828c7b14 100644 --- a/crates/re_types_builder/src/codegen/cpp/forward_decl.rs +++ b/crates/re_types_builder/src/codegen/cpp/forward_decl.rs @@ -37,6 +37,7 @@ pub struct ForwardDecls { } impl ForwardDecls { + #[allow(dead_code)] pub fn insert(&mut self, namespace: impl Into, decl: ForwardDecl) { self.declarations_per_namespace .entry(namespace.into()) diff --git a/crates/re_types_builder/src/codegen/cpp/method.rs b/crates/re_types_builder/src/codegen/cpp/method.rs index b036452a85b7..1f34eaf3526b 100644 --- a/crates/re_types_builder/src/codegen/cpp/method.rs +++ b/crates/re_types_builder/src/codegen/cpp/method.rs @@ -3,7 +3,7 @@ use quote::quote; use crate::Docs; -use super::{doc_comment, quote_docstrings, NEWLINE_TOKEN}; +use super::{quote_doc_comment, quote_docstrings, NEWLINE_TOKEN}; #[derive(Default)] pub struct MethodDeclaration { @@ -82,7 +82,7 @@ impl quote::ToTokens for MethodDocumentation { match self { Self::None => {} Self::String(s) => { - tokens.extend(doc_comment(s)); + tokens.extend(quote_doc_comment(s)); } Self::Docs(docs) => tokens.extend(quote_docstrings(docs)), } @@ -125,12 +125,14 @@ impl Method { #declaration { #definition_body } + #NEWLINE_TOKEN } } else { quote! { #NEWLINE_TOKEN #docs #declaration; + #NEWLINE_TOKEN } } } diff --git a/crates/re_types_builder/src/codegen/cpp/mod.rs b/crates/re_types_builder/src/codegen/cpp/mod.rs index 0f7a35387243..27f922ac3f93 100644 --- a/crates/re_types_builder/src/codegen/cpp/mod.rs +++ b/crates/re_types_builder/src/codegen/cpp/mod.rs @@ -4,6 +4,7 @@ mod method; use std::collections::BTreeSet; +use anyhow::Context; use arrow2::datatypes::DataType; use camino::{Utf8Path, Utf8PathBuf}; use itertools::Itertools; @@ -12,12 +13,13 @@ use quote::{format_ident, quote}; use rayon::prelude::*; use crate::codegen::common::write_file; +use crate::Object; use crate::{ codegen::AUTOGEN_WARNING, ArrowRegistry, Docs, ElementType, ObjectField, ObjectKind, Objects, Type, }; -use self::forward_decl::{ForwardDecl, ForwardDecls}; +use self::forward_decl::ForwardDecls; use self::includes::Includes; use self::method::{Method, MethodDeclaration}; @@ -34,11 +36,11 @@ const SYS_INCLUDE_PATH_PREFIX_TOKEN: &str = "SYS_INCLUDE_PATH_PREFIX_TOKEN"; const SYS_INCLUDE_PATH_SUFFIX_TOKEN: &str = "SYS_INCLUDE_PATH_SUFFIX_TOKEN"; const TODO_TOKEN: &str = "TODO_TOKEN"; -fn comment(text: &str) -> TokenStream { +fn quote_comment(text: &str) -> TokenStream { quote! { #NORMAL_COMMENT_PREFIX_TOKEN #text #NORMAL_COMMENT_SUFFIX_TOKEN } } -fn doc_comment(text: &str) -> TokenStream { +fn quote_doc_comment(text: &str) -> TokenStream { quote! { #DOC_COMMENT_PREFIX_TOKEN #text #DOC_COMMENT_SUFFIX_TOKEN } } @@ -164,7 +166,7 @@ impl crate::CodeGenerator for CppCodeGenerator { fn generate_hpp_cpp( objects: &Objects, arrow_registry: &ArrowRegistry, - obj: &crate::Object, + obj: &Object, ) -> (TokenStream, TokenStream) { let QuotedObject { hpp, cpp } = QuotedObject::new(arrow_registry, objects, obj); let snake_case_name = obj.snake_case_name(); @@ -197,7 +199,7 @@ struct QuotedObject { } impl QuotedObject { - pub fn new(arrow_registry: &ArrowRegistry, objects: &Objects, obj: &crate::Object) -> Self { + pub fn new(arrow_registry: &ArrowRegistry, objects: &Objects, obj: &Object) -> Self { match obj.specifics { crate::ObjectSpecifics::Struct => Self::from_struct(arrow_registry, objects, obj), crate::ObjectSpecifics::Union { .. } => Self::from_union(arrow_registry, objects, obj), @@ -207,7 +209,7 @@ impl QuotedObject { fn from_struct( arrow_registry: &ArrowRegistry, _objects: &Objects, - obj: &crate::Object, + obj: &Object, ) -> QuotedObject { let namespace_ident = format_ident!("{}", obj.kind.plural_snake_case()); // `datatypes`, `components`, or `archetypes` let pascal_case_name = &obj.name; @@ -216,7 +218,12 @@ impl QuotedObject { let mut hpp_includes = Includes::default(); hpp_includes.system.insert("cstdint".to_owned()); // we use `uint32_t` etc everywhere. + + // Doing our own forward declarations doesn't get us super far since some arrow types like `FloatBuilder` are type aliases. + // TODO(andreas): This drags in arrow headers into the public api though. We probably should try harder with forward declarations. + hpp_includes.system.insert("arrow/type_fwd.h".to_owned()); let mut cpp_includes = Includes::default(); + #[allow(unused)] let mut hpp_declarations = ForwardDecls::default(); let field_declarations = obj @@ -259,16 +266,22 @@ impl QuotedObject { } }; + let datatype = arrow_registry.get(&obj.fqname); + match obj.kind { ObjectKind::Datatype | ObjectKind::Component => { - methods.push(arrow_data_type_method( - &arrow_registry.get(&obj.fqname), - &mut hpp_includes, + methods.push(arrow_data_type_method(&datatype, &mut cpp_includes)); + methods.push(new_arrow_array_builder_method(&datatype, &mut cpp_includes)); + methods.push(fill_arrow_array_builder_method( + &datatype, + obj, + &pascal_case_ident, &mut cpp_includes, - &mut hpp_declarations, )); } - ObjectKind::Archetype => {} + ObjectKind::Archetype => { + // TODO(andreas): Should also be convertible to arrow? + } }; let hpp_method_section = if methods.is_empty() { @@ -282,6 +295,7 @@ impl QuotedObject { }; let hpp = quote! { #hpp_includes + #hpp_declarations namespace rr { @@ -309,11 +323,7 @@ impl QuotedObject { Self { hpp, cpp } } - fn from_union( - arrow_registry: &ArrowRegistry, - objects: &Objects, - obj: &crate::Object, - ) -> QuotedObject { + fn from_union(arrow_registry: &ArrowRegistry, objects: &Objects, obj: &Object) -> QuotedObject { // We implement sum-types as tagged unions; // Putting non-POD types in a union requires C++11. // @@ -342,7 +352,7 @@ impl QuotedObject { let data_typename = format_ident!("{pascal_case_name}Data"); let tag_fields = std::iter::once({ - let comment = doc_comment( + let comment = quote_doc_comment( "Having a special empty state makes it possible to implement move-semantics. \ We need to be able to leave the object in a state which we can run the destructor on."); let tag_name = format_ident!("NONE"); @@ -364,7 +374,13 @@ impl QuotedObject { hpp_includes.system.insert("cstdint".to_owned()); // we use `uint32_t` etc everywhere. hpp_includes.system.insert("utility".to_owned()); // std::move hpp_includes.system.insert("cstring".to_owned()); // std::memcpy + + // Doing our own forward declarations doesn't get us super far since some arrow types like `FloatBuilder` are type aliases. + // TODO(andreas): This drags in arrow headers into the public api though. We probably should try harder with forward declarations. + hpp_includes.system.insert("arrow/type_fwd.h".to_owned()); + let mut cpp_includes = Includes::default(); + #[allow(unused)] let mut hpp_declarations = ForwardDecls::default(); let enum_data_declarations = obj @@ -415,11 +431,14 @@ impl QuotedObject { // `enum Angle { Radians(f32), Degrees(f32) };` }; - methods.push(arrow_data_type_method( - &arrow_registry.get(&obj.fqname), - &mut hpp_includes, + let datatype = arrow_registry.get(&obj.fqname); + methods.push(arrow_data_type_method(&datatype, &mut cpp_includes)); + methods.push(new_arrow_array_builder_method(&datatype, &mut cpp_includes)); + methods.push(fill_arrow_array_builder_method( + &datatype, + obj, + &pascal_case_ident, &mut cpp_includes, - &mut hpp_declarations, )); let destructor = if obj.has_default_destructor(objects) { @@ -427,7 +446,7 @@ impl QuotedObject { quote! {} } else { let destructor_match_arms = std::iter::once({ - let comment = comment("Nothing to destroy"); + let comment = quote_comment("Nothing to destroy"); quote! { case detail::#tag_typename::NONE: { break; #comment @@ -439,7 +458,7 @@ impl QuotedObject { let field_ident = format_ident!("{}", crate::to_snake_case(&obj_field.name)); if obj_field.typ.has_default_destructor(objects) { - let comment = comment("has a trivial destructor"); + let comment = quote_comment("has a trivial destructor"); quote! { case detail::#tag_typename::#tag_ident: { break; #comment @@ -485,12 +504,14 @@ impl QuotedObject { } }; - let swap_comment = comment("This bitwise swap would fail for self-referential types, but we don't have any of those."); + let swap_comment = quote_comment("This bitwise swap would fail for self-referential types, but we don't have any of those."); let hpp_methods = methods.iter().map(|m| m.to_hpp_tokens()); let hpp = quote! { #hpp_includes + #hpp_declarations + namespace rr { namespace #namespace_ident { namespace detail { @@ -571,15 +592,8 @@ impl QuotedObject { } } -fn arrow_data_type_method( - datatype: &DataType, - hpp_includes: &mut Includes, - cpp_includes: &mut Includes, - hpp_declarations: &mut ForwardDecls, -) -> Method { - hpp_declarations.insert("arrow", ForwardDecl::Class("DataType".to_owned())); +fn arrow_data_type_method(datatype: &DataType, cpp_includes: &mut Includes) -> Method { cpp_includes.system.insert("arrow/api.h".to_owned()); - hpp_includes.system.insert("memory".to_owned()); // std::shared_ptr let quoted_datatype = quote_arrow_data_type(datatype, cpp_includes, true); @@ -595,6 +609,458 @@ fn arrow_data_type_method( } } +fn new_arrow_array_builder_method(datatype: &DataType, cpp_includes: &mut Includes) -> Method { + let arrow_builder_type = arrow_array_builder_type(datatype); + let arrow_builder_type = format_ident!("{arrow_builder_type}"); + let builder_instantiation = + quote_arrow_array_builder_type_instantiation(datatype, cpp_includes, true); + + Method { + docs: "Creates a new array builder with an array of this type.".into(), + declaration: MethodDeclaration { + is_static: true, + return_type: quote! { arrow::Result> }, + name_and_parameters: quote!(new_arrow_array_builder(arrow::MemoryPool * memory_pool)), + }, + definition_body: quote! { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + #NEWLINE_TOKEN + #NEWLINE_TOKEN + return arrow::Result(#builder_instantiation); + }, + inline: false, + } +} + +fn fill_arrow_array_builder_method( + datatype: &DataType, + obj: &Object, + pascal_case_ident: &Ident, + cpp_includes: &mut Includes, +) -> Method { + let DataType::Extension(_fqname, logical_datatype, _metadata) = datatype else { + panic!("Can only generate arrow serialization code for extension types. {}", obj.fqname); + }; + + let builder = format_ident!("builder"); + let arrow_builder_type = arrow_array_builder_type(datatype); + let arrow_builder_type = format_ident!("{arrow_builder_type}"); + + let fill_builder = quote_fill_arrow_array_builder( + pascal_case_ident, + logical_datatype, + &obj.fields, + &builder, + cpp_includes, + ) + .context(format!("Generating serialization for {}", obj.fqname)) + .unwrap(); + + Method { + docs: "Fills an arrow array builder with an array of this type.".into(), + declaration: MethodDeclaration { + is_static: true, + return_type: quote! { arrow::Status }, + // TODO(andreas): Pass in validity map. + name_and_parameters: quote! { + fill_arrow_array_builder(arrow::#arrow_builder_type* #builder, const #pascal_case_ident* elements, size_t num_elements) + }, + }, + definition_body: quote! { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + #NEWLINE_TOKEN + #NEWLINE_TOKEN + #fill_builder + #NEWLINE_TOKEN + #NEWLINE_TOKEN + return arrow::Status::OK(); + }, + inline: false, + } +} + +fn quote_fill_arrow_array_builder( + pascal_case_ident: &Ident, + datatype: &DataType, + fields: &[ObjectField], + builder: &Ident, + cpp_includes: &mut Includes, +) -> anyhow::Result { + let tokens = match datatype { + DataType::Boolean + | DataType::Int8 + | DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::UInt8 + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 + | DataType::Float16 + | DataType::Float32 + | DataType::Float64 + | DataType::Binary + | DataType::LargeBinary + | DataType::Utf8 + | DataType::LargeUtf8 => { + anyhow::ensure!( + fields.len() == 1, + "Expected exactly one field for primitive type {:?}", + datatype + ); + let object_field = &fields[0]; + quote_append_elements_to_builder(object_field, datatype, builder, true) + } + DataType::List(field) | DataType::FixedSizeList(field, _) => { + anyhow::ensure!( + fields.len() == 1, + "Expected exactly one field for list type {:?}", + datatype + ); + let object_field = &fields[0]; + + if matches!(field.data_type(), DataType::Extension { .. }) { + return Ok(quote!(return arrow::Status::NotImplemented( + "TODO(andreas): custom data types in lists/fixedsizelist are not yet implemented" + );)); + } + + let value_builder_type = arrow_array_builder_type(field.data_type()); + let value_builder_type = format_ident!("{value_builder_type}"); + let field_name = format_ident!("{}", &object_field.name); + let (num_items_per_element, reserve_factor) = match datatype { + DataType::List(..) => { + if field.is_nullable { + (quote!(element.#field_name.value().size()), quote_integer(1)) + } else { + (quote!(element.#field_name.size()), quote_integer(2)) + } + } + DataType::FixedSizeList(_, length) => { + let length = quote_integer(length); + (length.clone(), length) + } + _ => unreachable!(), + }; + + let setup = quote! { + auto value_builder = static_cast(#builder->value_builder()); + ARROW_RETURN_NOT_OK(#builder->Reserve(num_elements)); + ARROW_RETURN_NOT_OK(value_builder->Reserve(num_elements * #reserve_factor)); + #NEWLINE_TOKEN #NEWLINE_TOKEN + }; + + if field.is_nullable { + let item_append = if trivial_batch_append(field.data_type()) { + // `&expression[0]` is not pretty but works on both arrays and vectors! + quote! { + ARROW_RETURN_NOT_OK(value_builder->AppendValues(&element.#field_name.value()[0], #num_items_per_element, nullptr)); + } + } else { + quote! { + for (auto item_idx = 0; item_idx < #num_items_per_element; item_idx += 1) { + value_builder->Append(element.#field_name.value()[item_idx]); + } + } + }; + quote! { + #setup + for (auto elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto& element = elements[elem_idx]; + if (element.#field_name.has_value()) { + #item_append + ARROW_RETURN_NOT_OK(#builder->Append()); + } else { + ARROW_RETURN_NOT_OK(#builder->AppendNull()); + } + } + } + } else if matches!(datatype, DataType::FixedSizeList(..)) + && trivial_batch_append(field.data_type()) + { + // Optimize common case: Trivial batch of transparent fixed size elements. + quote! { + auto value_builder = static_cast(#builder->value_builder()); + #NEWLINE_TOKEN #NEWLINE_TOKEN + static_assert(sizeof(elements[0].#field_name) == sizeof(elements[0])); + ARROW_RETURN_NOT_OK(value_builder->AppendValues(elements[0].#field_name, num_elements * #num_items_per_element, nullptr)); + ARROW_RETURN_NOT_OK(#builder->AppendValues(num_elements)); + } + } else { + quote! { + #setup + for (auto elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto& element = elements[elem_idx]; + for (auto item_idx = 0; item_idx < #num_items_per_element; item_idx += 1) { + value_builder->Append(element.#field_name[item_idx]); + } + ARROW_RETURN_NOT_OK(#builder->Append()); + } + } + } + } + DataType::Struct(field_datatypes) => { + let fill_fields = fields.iter().zip(field_datatypes).enumerate().map( + |(i, (field, arrow_field))| { + let builder_index = quote_integer(i); + match arrow_field.data_type() { + DataType::FixedSizeList(..) | DataType::List(..) => { + quote!( + return arrow::Status::NotImplemented( + "TODO(andreas): lists in structs are not yet supported" + ); + ) + } + DataType::Extension(_fqname, _, _) => { + quote!( + return arrow::Status::NotImplemented( + "TODO(andreas): extensions in structs are not yet supported" + ); + ) + } + _ => { + let element_builder = format_ident!("element_builder"); + let element_builder_type = arrow_array_builder_type(arrow_field.data_type()); + let element_builder_type = format_ident!("{element_builder_type}"); + let field_append = quote_append_elements_to_builder(field, datatype, &element_builder, false); + quote! { + { + auto #element_builder = static_cast(builder->field_builder(#builder_index)); + #field_append + } + } + } + } + }, + ); + + quote! { + #(#fill_fields)* + #NEWLINE_TOKEN + ARROW_RETURN_NOT_OK(builder->AppendValues(num_elements, nullptr)); + } + } + DataType::Union(_, _, _) => { + quote! { + #NEWLINE_TOKEN + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto& element = elements[elem_idx]; + } + return arrow::Status::NotImplemented( + "TODO(andreas): unions are not yet implemented" + ); + } + } + DataType::Extension(fqname, _datatype, _) => { + assert_eq!(fields.len(), 1); + if fields[0].is_nullable { + // Idea: pass in a tagged union for both optional and non-optional arrays to all fill_arrow_array_builder methods? + quote!(return arrow::Status::NotImplemented(("TODO(andreas) Handle nullable extensions"));) + } else { + let quoted_fqname = quote_fqname_as_type_path(cpp_includes, fqname); + quote! { + static_assert(sizeof(#quoted_fqname) == sizeof(#pascal_case_ident)); + ARROW_RETURN_NOT_OK(#quoted_fqname::fill_arrow_array_builder( + builder, reinterpret_cast(elements), num_elements + )); + } + } + } + _ => anyhow::bail!( + "Arrow serialization for type {:?} not implemented", + datatype + ), + }; + + Ok(tokens) +} + +fn trivial_batch_append(datatype: &DataType) -> bool { + matches!( + datatype, + DataType::Null + | DataType::Boolean + | DataType::Int8 + | DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::UInt8 + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 + | DataType::Float16 + | DataType::Float32 + | DataType::Float64 + ) +} + +fn quote_append_elements_to_builder( + field: &ObjectField, + datatype: &DataType, + builder: &Ident, + is_transparent: bool, +) -> TokenStream { + let field_name = format_ident!("{}", field.name); + if field.is_nullable { + quote! { + ARROW_RETURN_NOT_OK(#builder->Reserve(num_elements)); + for (auto elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto& element = elements[elem_idx]; + if (element.#field_name.has_value()) { + ARROW_RETURN_NOT_OK(#builder->Append(element.#field_name.value())); + } else { + ARROW_RETURN_NOT_OK(#builder->AppendNull()); + } + } + } + } else if is_transparent && trivial_batch_append(datatype) { + // Trivial optimization: If this is the only field of this type and it's a trivial field (not array/string/blob), + // we can just pass the whole array as-is! + quote! { + static_assert(sizeof(*elements) == sizeof(elements->#field_name)); + ARROW_RETURN_NOT_OK(#builder->AppendValues(&elements->#field_name, num_elements)); + } + } else { + quote! { + ARROW_RETURN_NOT_OK(#builder->Reserve(num_elements)); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + ARROW_RETURN_NOT_OK(#builder->Append(elements[elem_idx].#field_name)); + } + } + } +} + +fn arrow_array_builder_type(datatype: &DataType) -> &'static str { + match datatype.to_logical_type() { + DataType::Boolean => "BooleanBuilder", + DataType::Int8 => "Int8Builder", + DataType::Int16 => "Int16Builder", + DataType::Int32 => "Int32Builder", + DataType::Int64 => "Int64Builder", + DataType::UInt8 => "UInt8Builder", + DataType::UInt16 => "UInt16Builder", + DataType::UInt32 => "UInt32Builder", + DataType::UInt64 => "UInt64Builder", + DataType::Float16 => "HalfFloatBuilder", + DataType::Float32 => "FloatBuilder", + DataType::Float64 => "DoubleBuilder", + DataType::Binary => "BinaryBuilder", + DataType::LargeBinary => "LargeBinaryBuilder", + DataType::Utf8 => "StringBuilder", + DataType::LargeUtf8 => "LargeStringBuilder", + DataType::FixedSizeList(..) => "FixedSizeListBuilder", + DataType::List(..) => "ListBuilder", + DataType::Struct(..) => "StructBuilder", + DataType::Null => "NullBuilder", + DataType::Union(_, _, mode) => match mode { + arrow2::datatypes::UnionMode::Dense => "DenseUnionBuilder", + arrow2::datatypes::UnionMode::Sparse => "SparseUnionBuilder", + }, + DataType::Extension(_, _, _metadata) => { + unreachable!("Logical type can't be an extension type.") + } + _ => unimplemented!( + "Arrow serialization for type {:?} not implemented", + datatype + ), + } +} + +fn quote_arrow_array_builder_type_instantiation( + datatype: &DataType, + cpp_includes: &mut Includes, + is_top_level_type: bool, +) -> TokenStream { + let builder_type = arrow_array_builder_type(datatype); + let builder_type = format_ident!("{builder_type}"); + + match datatype { + DataType::Boolean + | DataType::Int8 + | DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::UInt8 + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 + | DataType::Float16 + | DataType::Float32 + | DataType::Float64 + | DataType::Binary + | DataType::LargeBinary + | DataType::Utf8 + | DataType::LargeUtf8 + | DataType::Null => { + quote!(std::make_shared(memory_pool)) + } + DataType::List(field) => { + let element_builder = quote_arrow_array_builder_type_instantiation( + field.data_type(), + cpp_includes, + false, + ); + quote!(std::make_shared(memory_pool, #element_builder)) + } + DataType::FixedSizeList(field, length) => { + let quoted_length = quote_integer(length); + let element_builder = quote_arrow_array_builder_type_instantiation( + field.data_type(), + cpp_includes, + false, + ); + quote!(std::make_shared(memory_pool, #element_builder, #quoted_length)) + } + DataType::Struct(fields) => { + let field_builders = fields.iter().map(|field| { + quote_arrow_array_builder_type_instantiation(field.data_type(), cpp_includes, false) + }); + quote! { + std::make_shared( + to_arrow_datatype(), + memory_pool, + std::vector>({ #(#field_builders,)* }) + ) + } + } + DataType::Union(fields, _, _) => { + let field_builders = fields.iter().map(|field| { + quote_arrow_array_builder_type_instantiation(field.data_type(), cpp_includes, false) + }); + quote! { + std::make_shared( + memory_pool, + std::vector>({ #(#field_builders,)* }), + to_arrow_datatype() + ) + } + } + DataType::Extension(fqname, datatype, _metadata) => { + if is_top_level_type { + quote_arrow_array_builder_type_instantiation(datatype.as_ref(), cpp_includes, false) + } else { + // Propagating error here is hard since we're in a nested context. + // But also not that important since we *know* that this only fails for null pools and we already checked that now. + let quoted_fqname = quote_fqname_as_type_path(cpp_includes, fqname); + quote! { + #quoted_fqname::new_arrow_array_builder(memory_pool).ValueOrDie() + } + } + } + _ => unimplemented!( + "Arrow serialization for type {:?} not implemented", + datatype + ), + } +} + /// e.g. `static Angle radians(float radians);` -> `auto angle = Angle::radians(radians);` fn static_constructor_for_enum_type( objects: &Objects, @@ -823,7 +1289,7 @@ fn quote_fqname_as_type_path(includes: &mut Includes, fqname: &str) -> TokenStre fn quote_docstrings(docs: &Docs) -> TokenStream { let lines = crate::codegen::get_documentation(docs, &["cpp", "c++"]); - let quoted_lines = lines.iter().map(|docstring| doc_comment(docstring)); + let quoted_lines = lines.iter().map(|docstring| quote_doc_comment(docstring)); quote! { #NEWLINE_TOKEN #(#quoted_lines)* @@ -902,8 +1368,9 @@ fn quote_arrow_data_type( // In the future we'll add the extension type here to the schema. quote_arrow_data_type(datatype, includes, false) } else { - let fqname_use = quote_fqname_as_type_path(includes, fqname); - quote! { #fqname_use::to_arrow_datatype() } + // TODO(andreas): remove unnecessary namespacing. + let quoted_fqname = quote_fqname_as_type_path(includes, fqname); + quote! { #quoted_fqname::to_arrow_datatype() } } } diff --git a/crates/rerun_c/src/rerun.h b/crates/rerun_c/src/rerun.h index 6ee2220e6394..c7efa3040265 100644 --- a/crates/rerun_c/src/rerun.h +++ b/crates/rerun_c/src/rerun.h @@ -82,8 +82,9 @@ extern const char* rr_version_string(void); /// Usually you only have one recording stream, so you can call /// `rr_recording_stream_new` once, ignore its return value, and use /// `RERUN_REC_STREAM_DEFAULT` everywhere in your code. -extern rr_recording_stream rr_recording_stream_new(const struct rr_store_info* store_info, - const char* tcp_addr); +extern rr_recording_stream rr_recording_stream_new( + const struct rr_store_info* store_info, const char* tcp_addr +); /// Free the given recording stream. The handle will be invalid after this. extern void rr_recording_stream_free(rr_recording_stream stream); diff --git a/examples/cpp/minimal/main.cpp b/examples/cpp/minimal/main.cpp index dd8e246b152a..1a7e24b33c12 100644 --- a/examples/cpp/minimal/main.cpp +++ b/examples/cpp/minimal/main.cpp @@ -1,6 +1,29 @@ #include #include +#include + +arrow::Result> points2(size_t num_points, const float* xy) { + arrow::MemoryPool* pool = arrow::default_memory_pool(); + + ARROW_ASSIGN_OR_RAISE(auto builder, rr::components::Point2D::new_arrow_array_builder(pool)); + ARROW_RETURN_NOT_OK(rr::components::Point2D::fill_arrow_array_builder( + builder.get(), + (const rr::components::Point2D*) + xy, // TODO(andreas): Hack to get Points2D C-style array in an easy fashion + num_points + )); + + std::shared_ptr array; + ARROW_RETURN_NOT_OK(builder->Finish(&array)); + + auto name = "points"; // Unused, but should be the name of the field in the archetype + auto schema = + arrow::schema({arrow::field(name, rr::components::Point2D::to_arrow_datatype(), false)}); + + return arrow::Table::Make(schema, {array}); +} + int main(int argc, char** argv) { loguru::g_preamble_uptime = false; loguru::g_preamble_thread = false; @@ -10,18 +33,37 @@ int main(int argc, char** argv) { auto rr_stream = rr::RecordingStream{"c-example-app", "127.0.0.1:9876"}; - float xyz[9] = {0.0, 0.0, 0.0, 1.0, 2.0, 3.0, 5.0, 5.0, 5.0}; - auto points = rr::points3(3, xyz).ValueOrDie(); - auto buffer = rr::ipc_from_table(*points).ValueOrDie(); + // Points3D. + { + float xyz[9] = {0.0, 0.0, 0.0, 1.0, 2.0, 3.0, 5.0, 5.0, 5.0}; + auto points = rr::points3(3, xyz).ValueOrDie(); // TODO(andreas): phase this out. + auto buffer = rr::ipc_from_table(*points).ValueOrDie(); + + const rr::DataCell data_cells[1] = {rr::DataCell{ + .component_name = "rerun.point3d", + .num_bytes = static_cast(buffer->size()), + .bytes = buffer->data(), + }}; + + uint32_t num_instances = 3; + rr_stream.log_data_row("3d/points", num_instances, 1, data_cells); + } + + // Points2D. + { + float xy[6] = {0.0, 0.0, 1.0, 3.0, 5.0, 5.0}; + auto points = points2(3, xy).ValueOrDie(); + auto buffer = rr::ipc_from_table(*points).ValueOrDie(); - const rr::DataCell data_cells[1] = {rr::DataCell{ - .component_name = "rerun.point3d", - .num_bytes = static_cast(buffer->size()), - .bytes = buffer->data(), - }}; + const rr::DataCell data_cells[1] = {rr::DataCell{ + .component_name = "rerun.point2d", + .num_bytes = static_cast(buffer->size()), + .bytes = buffer->data(), + }}; - uint32_t num_instances = 3; - rr_stream.log_data_row("points", num_instances, 1, data_cells); + uint32_t num_instances = 3; + rr_stream.log_data_row("2d/points", num_instances, 1, data_cells); + } // Test some type instantiation auto tls = rr::datatypes::TranslationRotationScale3D{}; diff --git a/rerun_cpp/src/archetypes/affix_fuzzer1.hpp b/rerun_cpp/src/archetypes/affix_fuzzer1.hpp index 027fb9a2a2d7..a6396d0d2d0a 100644 --- a/rerun_cpp/src/archetypes/affix_fuzzer1.hpp +++ b/rerun_cpp/src/archetypes/affix_fuzzer1.hpp @@ -23,6 +23,7 @@ #include "../components/affix_fuzzer8.hpp" #include "../components/affix_fuzzer9.hpp" +#include #include #include #include diff --git a/rerun_cpp/src/archetypes/points2d.hpp b/rerun_cpp/src/archetypes/points2d.hpp index 66fb2798a798..39fbbbd547c1 100644 --- a/rerun_cpp/src/archetypes/points2d.hpp +++ b/rerun_cpp/src/archetypes/points2d.hpp @@ -12,6 +12,7 @@ #include "../components/point2d.hpp" #include "../components/radius.hpp" +#include #include #include #include diff --git a/rerun_cpp/src/archetypes/points3d.hpp b/rerun_cpp/src/archetypes/points3d.hpp index 841b6a0d709b..6fe3b44b3fd0 100644 --- a/rerun_cpp/src/archetypes/points3d.hpp +++ b/rerun_cpp/src/archetypes/points3d.hpp @@ -12,6 +12,7 @@ #include "../components/point3d.hpp" #include "../components/radius.hpp" +#include #include #include #include diff --git a/rerun_cpp/src/archetypes/transform3d.hpp b/rerun_cpp/src/archetypes/transform3d.hpp index cb678d92f1d7..82dda3cc1ed0 100644 --- a/rerun_cpp/src/archetypes/transform3d.hpp +++ b/rerun_cpp/src/archetypes/transform3d.hpp @@ -5,6 +5,7 @@ #include "../components/transform3d.hpp" +#include #include #include diff --git a/rerun_cpp/src/components/affix_fuzzer1.cpp b/rerun_cpp/src/components/affix_fuzzer1.cpp index 14a27aa3c059..5518fea03a42 100644 --- a/rerun_cpp/src/components/affix_fuzzer1.cpp +++ b/rerun_cpp/src/components/affix_fuzzer1.cpp @@ -12,5 +12,37 @@ namespace rr { std::shared_ptr AffixFuzzer1::to_arrow_datatype() { return rr::datatypes::AffixFuzzer1::to_arrow_datatype(); } + + arrow::Result> AffixFuzzer1::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result( + rr::datatypes::AffixFuzzer1::new_arrow_array_builder(memory_pool).ValueOrDie() + ); + } + + arrow::Status AffixFuzzer1::fill_arrow_array_builder( + arrow::StructBuilder *builder, const AffixFuzzer1 *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + static_assert(sizeof(rr::datatypes::AffixFuzzer1) == sizeof(AffixFuzzer1)); + ARROW_RETURN_NOT_OK(rr::datatypes::AffixFuzzer1::fill_arrow_array_builder( + builder, + reinterpret_cast(elements), + num_elements + )); + + return arrow::Status::OK(); + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer1.hpp b/rerun_cpp/src/components/affix_fuzzer1.hpp index 35e7ce40be07..a749a1ea4bb5 100644 --- a/rerun_cpp/src/components/affix_fuzzer1.hpp +++ b/rerun_cpp/src/components/affix_fuzzer1.hpp @@ -5,14 +5,10 @@ #include "../datatypes/affix_fuzzer1.hpp" +#include #include -#include #include -namespace arrow { - class DataType; -} - namespace rr { namespace components { struct AffixFuzzer1 { @@ -24,6 +20,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const AffixFuzzer1* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer10.cpp b/rerun_cpp/src/components/affix_fuzzer10.cpp index 81a99b9b8215..0c3a13d23e85 100644 --- a/rerun_cpp/src/components/affix_fuzzer10.cpp +++ b/rerun_cpp/src/components/affix_fuzzer10.cpp @@ -10,5 +10,38 @@ namespace rr { std::shared_ptr AffixFuzzer10::to_arrow_datatype() { return arrow::utf8(); } + + arrow::Result> AffixFuzzer10::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared(memory_pool)); + } + + arrow::Status AffixFuzzer10::fill_arrow_array_builder( + arrow::StringBuilder* builder, const AffixFuzzer10* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + ARROW_RETURN_NOT_OK(builder->Reserve(num_elements)); + for (auto elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto& element = elements[elem_idx]; + if (element.single_string_optional.has_value()) { + ARROW_RETURN_NOT_OK(builder->Append(element.single_string_optional.value())); + } else { + ARROW_RETURN_NOT_OK(builder->AppendNull()); + } + } + + return arrow::Status::OK(); + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer10.hpp b/rerun_cpp/src/components/affix_fuzzer10.hpp index 5293a168d2b9..1c3c951d9711 100644 --- a/rerun_cpp/src/components/affix_fuzzer10.hpp +++ b/rerun_cpp/src/components/affix_fuzzer10.hpp @@ -3,16 +3,12 @@ #pragma once +#include #include -#include #include #include #include -namespace arrow { - class DataType; -} - namespace rr { namespace components { struct AffixFuzzer10 { @@ -24,6 +20,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StringBuilder* builder, const AffixFuzzer10* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer11.cpp b/rerun_cpp/src/components/affix_fuzzer11.cpp index a71618858bf4..4e82609e9bca 100644 --- a/rerun_cpp/src/components/affix_fuzzer11.cpp +++ b/rerun_cpp/src/components/affix_fuzzer11.cpp @@ -10,5 +10,49 @@ namespace rr { std::shared_ptr AffixFuzzer11::to_arrow_datatype() { return arrow::list(arrow::field("item", arrow::float32(), true, nullptr)); } + + arrow::Result> AffixFuzzer11::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + memory_pool, + std::make_shared(memory_pool) + )); + } + + arrow::Status AffixFuzzer11::fill_arrow_array_builder( + arrow::ListBuilder *builder, const AffixFuzzer11 *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto value_builder = static_cast(builder->value_builder()); + ARROW_RETURN_NOT_OK(builder->Reserve(num_elements)); + ARROW_RETURN_NOT_OK(value_builder->Reserve(num_elements * 1)); + + for (auto elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto &element = elements[elem_idx]; + if (element.many_floats_optional.has_value()) { + ARROW_RETURN_NOT_OK(value_builder->AppendValues( + &element.many_floats_optional.value()[0], + element.many_floats_optional.value().size(), + nullptr + )); + ARROW_RETURN_NOT_OK(builder->Append()); + } else { + ARROW_RETURN_NOT_OK(builder->AppendNull()); + } + } + + return arrow::Status::OK(); + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer11.hpp b/rerun_cpp/src/components/affix_fuzzer11.hpp index d0cd2ae0858f..ad40ffa4036b 100644 --- a/rerun_cpp/src/components/affix_fuzzer11.hpp +++ b/rerun_cpp/src/components/affix_fuzzer11.hpp @@ -3,16 +3,12 @@ #pragma once +#include #include -#include #include #include #include -namespace arrow { - class DataType; -} - namespace rr { namespace components { struct AffixFuzzer11 { @@ -24,6 +20,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::ListBuilder* builder, const AffixFuzzer11* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer12.cpp b/rerun_cpp/src/components/affix_fuzzer12.cpp index 253eec0c8844..a37316693711 100644 --- a/rerun_cpp/src/components/affix_fuzzer12.cpp +++ b/rerun_cpp/src/components/affix_fuzzer12.cpp @@ -10,5 +10,44 @@ namespace rr { std::shared_ptr AffixFuzzer12::to_arrow_datatype() { return arrow::list(arrow::field("item", arrow::utf8(), false, nullptr)); } + + arrow::Result> AffixFuzzer12::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + memory_pool, + std::make_shared(memory_pool) + )); + } + + arrow::Status AffixFuzzer12::fill_arrow_array_builder( + arrow::ListBuilder *builder, const AffixFuzzer12 *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto value_builder = static_cast(builder->value_builder()); + ARROW_RETURN_NOT_OK(builder->Reserve(num_elements)); + ARROW_RETURN_NOT_OK(value_builder->Reserve(num_elements * 2)); + + for (auto elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto &element = elements[elem_idx]; + for (auto item_idx = 0; item_idx < element.many_strings_required.size(); + item_idx += 1) { + value_builder->Append(element.many_strings_required[item_idx]); + } + ARROW_RETURN_NOT_OK(builder->Append()); + } + + return arrow::Status::OK(); + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer12.hpp b/rerun_cpp/src/components/affix_fuzzer12.hpp index d553f5109506..d6e2761dc424 100644 --- a/rerun_cpp/src/components/affix_fuzzer12.hpp +++ b/rerun_cpp/src/components/affix_fuzzer12.hpp @@ -3,16 +3,12 @@ #pragma once +#include #include -#include #include #include #include -namespace arrow { - class DataType; -} - namespace rr { namespace components { struct AffixFuzzer12 { @@ -24,6 +20,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::ListBuilder* builder, const AffixFuzzer12* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer13.cpp b/rerun_cpp/src/components/affix_fuzzer13.cpp index 4fa2852ef5a3..c2bdb9baa824 100644 --- a/rerun_cpp/src/components/affix_fuzzer13.cpp +++ b/rerun_cpp/src/components/affix_fuzzer13.cpp @@ -10,5 +10,48 @@ namespace rr { std::shared_ptr AffixFuzzer13::to_arrow_datatype() { return arrow::list(arrow::field("item", arrow::utf8(), true, nullptr)); } + + arrow::Result> AffixFuzzer13::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + memory_pool, + std::make_shared(memory_pool) + )); + } + + arrow::Status AffixFuzzer13::fill_arrow_array_builder( + arrow::ListBuilder *builder, const AffixFuzzer13 *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto value_builder = static_cast(builder->value_builder()); + ARROW_RETURN_NOT_OK(builder->Reserve(num_elements)); + ARROW_RETURN_NOT_OK(value_builder->Reserve(num_elements * 1)); + + for (auto elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto &element = elements[elem_idx]; + if (element.many_strings_optional.has_value()) { + for (auto item_idx = 0; item_idx < element.many_strings_optional.value().size(); + item_idx += 1) { + value_builder->Append(element.many_strings_optional.value()[item_idx]); + } + ARROW_RETURN_NOT_OK(builder->Append()); + } else { + ARROW_RETURN_NOT_OK(builder->AppendNull()); + } + } + + return arrow::Status::OK(); + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer13.hpp b/rerun_cpp/src/components/affix_fuzzer13.hpp index f6afa2733905..5ddd66ff404c 100644 --- a/rerun_cpp/src/components/affix_fuzzer13.hpp +++ b/rerun_cpp/src/components/affix_fuzzer13.hpp @@ -3,17 +3,13 @@ #pragma once +#include #include -#include #include #include #include #include -namespace arrow { - class DataType; -} - namespace rr { namespace components { struct AffixFuzzer13 { @@ -25,6 +21,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::ListBuilder* builder, const AffixFuzzer13* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer14.cpp b/rerun_cpp/src/components/affix_fuzzer14.cpp index e37954f9bc6a..3a97ebed9e83 100644 --- a/rerun_cpp/src/components/affix_fuzzer14.cpp +++ b/rerun_cpp/src/components/affix_fuzzer14.cpp @@ -12,5 +12,36 @@ namespace rr { std::shared_ptr AffixFuzzer14::to_arrow_datatype() { return rr::datatypes::AffixFuzzer3::to_arrow_datatype(); } + + arrow::Result> + AffixFuzzer14::new_arrow_array_builder(arrow::MemoryPool *memory_pool) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result( + rr::datatypes::AffixFuzzer3::new_arrow_array_builder(memory_pool).ValueOrDie() + ); + } + + arrow::Status AffixFuzzer14::fill_arrow_array_builder( + arrow::DenseUnionBuilder *builder, const AffixFuzzer14 *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + static_assert(sizeof(rr::datatypes::AffixFuzzer3) == sizeof(AffixFuzzer14)); + ARROW_RETURN_NOT_OK(rr::datatypes::AffixFuzzer3::fill_arrow_array_builder( + builder, + reinterpret_cast(elements), + num_elements + )); + + return arrow::Status::OK(); + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer14.hpp b/rerun_cpp/src/components/affix_fuzzer14.hpp index a74dca7d43d9..c7d4e8d345d6 100644 --- a/rerun_cpp/src/components/affix_fuzzer14.hpp +++ b/rerun_cpp/src/components/affix_fuzzer14.hpp @@ -5,14 +5,10 @@ #include "../datatypes/affix_fuzzer3.hpp" +#include #include -#include #include -namespace arrow { - class DataType; -} - namespace rr { namespace components { struct AffixFuzzer14 { @@ -24,6 +20,17 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::DenseUnionBuilder* builder, const AffixFuzzer14* elements, + size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer15.cpp b/rerun_cpp/src/components/affix_fuzzer15.cpp index 45d1a31b6e7a..c80eb365c0a6 100644 --- a/rerun_cpp/src/components/affix_fuzzer15.cpp +++ b/rerun_cpp/src/components/affix_fuzzer15.cpp @@ -12,5 +12,31 @@ namespace rr { std::shared_ptr AffixFuzzer15::to_arrow_datatype() { return rr::datatypes::AffixFuzzer3::to_arrow_datatype(); } + + arrow::Result> + AffixFuzzer15::new_arrow_array_builder(arrow::MemoryPool* memory_pool) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result( + rr::datatypes::AffixFuzzer3::new_arrow_array_builder(memory_pool).ValueOrDie() + ); + } + + arrow::Status AffixFuzzer15::fill_arrow_array_builder( + arrow::DenseUnionBuilder* builder, const AffixFuzzer15* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + return arrow::Status::NotImplemented(("TODO(andreas) Handle nullable extensions")); + + return arrow::Status::OK(); + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer15.hpp b/rerun_cpp/src/components/affix_fuzzer15.hpp index 6c95217a2f20..3fdb1a83796e 100644 --- a/rerun_cpp/src/components/affix_fuzzer15.hpp +++ b/rerun_cpp/src/components/affix_fuzzer15.hpp @@ -5,15 +5,11 @@ #include "../datatypes/affix_fuzzer3.hpp" +#include #include -#include #include #include -namespace arrow { - class DataType; -} - namespace rr { namespace components { struct AffixFuzzer15 { @@ -25,6 +21,17 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::DenseUnionBuilder* builder, const AffixFuzzer15* elements, + size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer16.cpp b/rerun_cpp/src/components/affix_fuzzer16.cpp index a257a5981278..10d40df328e6 100644 --- a/rerun_cpp/src/components/affix_fuzzer16.cpp +++ b/rerun_cpp/src/components/affix_fuzzer16.cpp @@ -11,7 +11,41 @@ namespace rr { namespace components { std::shared_ptr AffixFuzzer16::to_arrow_datatype() { return arrow::list(arrow::field( - "item", rr::datatypes::AffixFuzzer3::to_arrow_datatype(), false, nullptr)); + "item", + rr::datatypes::AffixFuzzer3::to_arrow_datatype(), + false, + nullptr + )); + } + + arrow::Result> AffixFuzzer16::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + memory_pool, + rr::datatypes::AffixFuzzer3::new_arrow_array_builder(memory_pool).ValueOrDie() + )); + } + + arrow::Status AffixFuzzer16::fill_arrow_array_builder( + arrow::ListBuilder* builder, const AffixFuzzer16* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + return arrow::Status::NotImplemented( + "TODO(andreas): custom data types in lists/fixedsizelist are not yet implemented" + ); + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer16.hpp b/rerun_cpp/src/components/affix_fuzzer16.hpp index dcb477706185..973b3d502cb8 100644 --- a/rerun_cpp/src/components/affix_fuzzer16.hpp +++ b/rerun_cpp/src/components/affix_fuzzer16.hpp @@ -5,15 +5,11 @@ #include "../datatypes/affix_fuzzer3.hpp" +#include #include -#include #include #include -namespace arrow { - class DataType; -} - namespace rr { namespace components { struct AffixFuzzer16 { @@ -25,6 +21,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::ListBuilder* builder, const AffixFuzzer16* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer17.cpp b/rerun_cpp/src/components/affix_fuzzer17.cpp index 2b9d7cb30041..8e478c4e812d 100644 --- a/rerun_cpp/src/components/affix_fuzzer17.cpp +++ b/rerun_cpp/src/components/affix_fuzzer17.cpp @@ -11,7 +11,41 @@ namespace rr { namespace components { std::shared_ptr AffixFuzzer17::to_arrow_datatype() { return arrow::list(arrow::field( - "item", rr::datatypes::AffixFuzzer3::to_arrow_datatype(), true, nullptr)); + "item", + rr::datatypes::AffixFuzzer3::to_arrow_datatype(), + true, + nullptr + )); + } + + arrow::Result> AffixFuzzer17::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + memory_pool, + rr::datatypes::AffixFuzzer3::new_arrow_array_builder(memory_pool).ValueOrDie() + )); + } + + arrow::Status AffixFuzzer17::fill_arrow_array_builder( + arrow::ListBuilder* builder, const AffixFuzzer17* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + return arrow::Status::NotImplemented( + "TODO(andreas): custom data types in lists/fixedsizelist are not yet implemented" + ); + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer17.hpp b/rerun_cpp/src/components/affix_fuzzer17.hpp index 89b27364f836..849774b2cf96 100644 --- a/rerun_cpp/src/components/affix_fuzzer17.hpp +++ b/rerun_cpp/src/components/affix_fuzzer17.hpp @@ -5,16 +5,12 @@ #include "../datatypes/affix_fuzzer3.hpp" +#include #include -#include #include #include #include -namespace arrow { - class DataType; -} - namespace rr { namespace components { struct AffixFuzzer17 { @@ -22,11 +18,22 @@ namespace rr { public: AffixFuzzer17( - std::optional> many_optional_unions) + std::optional> many_optional_unions + ) : many_optional_unions(std::move(many_optional_unions)) {} /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::ListBuilder* builder, const AffixFuzzer17* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer18.cpp b/rerun_cpp/src/components/affix_fuzzer18.cpp index e6c172976e91..088de4597696 100644 --- a/rerun_cpp/src/components/affix_fuzzer18.cpp +++ b/rerun_cpp/src/components/affix_fuzzer18.cpp @@ -11,7 +11,41 @@ namespace rr { namespace components { std::shared_ptr AffixFuzzer18::to_arrow_datatype() { return arrow::list(arrow::field( - "item", rr::datatypes::AffixFuzzer4::to_arrow_datatype(), true, nullptr)); + "item", + rr::datatypes::AffixFuzzer4::to_arrow_datatype(), + true, + nullptr + )); + } + + arrow::Result> AffixFuzzer18::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + memory_pool, + rr::datatypes::AffixFuzzer4::new_arrow_array_builder(memory_pool).ValueOrDie() + )); + } + + arrow::Status AffixFuzzer18::fill_arrow_array_builder( + arrow::ListBuilder* builder, const AffixFuzzer18* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + return arrow::Status::NotImplemented( + "TODO(andreas): custom data types in lists/fixedsizelist are not yet implemented" + ); + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer18.hpp b/rerun_cpp/src/components/affix_fuzzer18.hpp index 599f893187d2..825e29082b9e 100644 --- a/rerun_cpp/src/components/affix_fuzzer18.hpp +++ b/rerun_cpp/src/components/affix_fuzzer18.hpp @@ -5,16 +5,12 @@ #include "../datatypes/affix_fuzzer4.hpp" +#include #include -#include #include #include #include -namespace arrow { - class DataType; -} - namespace rr { namespace components { struct AffixFuzzer18 { @@ -22,11 +18,22 @@ namespace rr { public: AffixFuzzer18( - std::optional> many_optional_unions) + std::optional> many_optional_unions + ) : many_optional_unions(std::move(many_optional_unions)) {} /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::ListBuilder* builder, const AffixFuzzer18* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer19.cpp b/rerun_cpp/src/components/affix_fuzzer19.cpp index 8d375e459b22..b1630b37e408 100644 --- a/rerun_cpp/src/components/affix_fuzzer19.cpp +++ b/rerun_cpp/src/components/affix_fuzzer19.cpp @@ -12,5 +12,37 @@ namespace rr { std::shared_ptr AffixFuzzer19::to_arrow_datatype() { return rr::datatypes::AffixFuzzer5::to_arrow_datatype(); } + + arrow::Result> AffixFuzzer19::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result( + rr::datatypes::AffixFuzzer5::new_arrow_array_builder(memory_pool).ValueOrDie() + ); + } + + arrow::Status AffixFuzzer19::fill_arrow_array_builder( + arrow::StructBuilder *builder, const AffixFuzzer19 *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + static_assert(sizeof(rr::datatypes::AffixFuzzer5) == sizeof(AffixFuzzer19)); + ARROW_RETURN_NOT_OK(rr::datatypes::AffixFuzzer5::fill_arrow_array_builder( + builder, + reinterpret_cast(elements), + num_elements + )); + + return arrow::Status::OK(); + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer19.hpp b/rerun_cpp/src/components/affix_fuzzer19.hpp index 234201301457..089daef4598f 100644 --- a/rerun_cpp/src/components/affix_fuzzer19.hpp +++ b/rerun_cpp/src/components/affix_fuzzer19.hpp @@ -5,14 +5,10 @@ #include "../datatypes/affix_fuzzer5.hpp" +#include #include -#include #include -namespace arrow { - class DataType; -} - namespace rr { namespace components { struct AffixFuzzer19 { @@ -24,6 +20,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const AffixFuzzer19* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer2.cpp b/rerun_cpp/src/components/affix_fuzzer2.cpp index cb49f0249c94..083d72529def 100644 --- a/rerun_cpp/src/components/affix_fuzzer2.cpp +++ b/rerun_cpp/src/components/affix_fuzzer2.cpp @@ -12,5 +12,37 @@ namespace rr { std::shared_ptr AffixFuzzer2::to_arrow_datatype() { return rr::datatypes::AffixFuzzer1::to_arrow_datatype(); } + + arrow::Result> AffixFuzzer2::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result( + rr::datatypes::AffixFuzzer1::new_arrow_array_builder(memory_pool).ValueOrDie() + ); + } + + arrow::Status AffixFuzzer2::fill_arrow_array_builder( + arrow::StructBuilder *builder, const AffixFuzzer2 *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + static_assert(sizeof(rr::datatypes::AffixFuzzer1) == sizeof(AffixFuzzer2)); + ARROW_RETURN_NOT_OK(rr::datatypes::AffixFuzzer1::fill_arrow_array_builder( + builder, + reinterpret_cast(elements), + num_elements + )); + + return arrow::Status::OK(); + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer2.hpp b/rerun_cpp/src/components/affix_fuzzer2.hpp index e1e2d868cf82..4db325c8df87 100644 --- a/rerun_cpp/src/components/affix_fuzzer2.hpp +++ b/rerun_cpp/src/components/affix_fuzzer2.hpp @@ -5,14 +5,10 @@ #include "../datatypes/affix_fuzzer1.hpp" +#include #include -#include #include -namespace arrow { - class DataType; -} - namespace rr { namespace components { struct AffixFuzzer2 { @@ -24,6 +20,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const AffixFuzzer2* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer3.cpp b/rerun_cpp/src/components/affix_fuzzer3.cpp index 727da20b00c4..9a9e36eb4e58 100644 --- a/rerun_cpp/src/components/affix_fuzzer3.cpp +++ b/rerun_cpp/src/components/affix_fuzzer3.cpp @@ -12,5 +12,37 @@ namespace rr { std::shared_ptr AffixFuzzer3::to_arrow_datatype() { return rr::datatypes::AffixFuzzer1::to_arrow_datatype(); } + + arrow::Result> AffixFuzzer3::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result( + rr::datatypes::AffixFuzzer1::new_arrow_array_builder(memory_pool).ValueOrDie() + ); + } + + arrow::Status AffixFuzzer3::fill_arrow_array_builder( + arrow::StructBuilder *builder, const AffixFuzzer3 *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + static_assert(sizeof(rr::datatypes::AffixFuzzer1) == sizeof(AffixFuzzer3)); + ARROW_RETURN_NOT_OK(rr::datatypes::AffixFuzzer1::fill_arrow_array_builder( + builder, + reinterpret_cast(elements), + num_elements + )); + + return arrow::Status::OK(); + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer3.hpp b/rerun_cpp/src/components/affix_fuzzer3.hpp index b124479821b8..d0443ea19ad1 100644 --- a/rerun_cpp/src/components/affix_fuzzer3.hpp +++ b/rerun_cpp/src/components/affix_fuzzer3.hpp @@ -5,14 +5,10 @@ #include "../datatypes/affix_fuzzer1.hpp" +#include #include -#include #include -namespace arrow { - class DataType; -} - namespace rr { namespace components { struct AffixFuzzer3 { @@ -24,6 +20,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const AffixFuzzer3* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer4.cpp b/rerun_cpp/src/components/affix_fuzzer4.cpp index 7b5c4ab0f69e..cb4c8d7a4438 100644 --- a/rerun_cpp/src/components/affix_fuzzer4.cpp +++ b/rerun_cpp/src/components/affix_fuzzer4.cpp @@ -12,5 +12,32 @@ namespace rr { std::shared_ptr AffixFuzzer4::to_arrow_datatype() { return rr::datatypes::AffixFuzzer1::to_arrow_datatype(); } + + arrow::Result> AffixFuzzer4::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result( + rr::datatypes::AffixFuzzer1::new_arrow_array_builder(memory_pool).ValueOrDie() + ); + } + + arrow::Status AffixFuzzer4::fill_arrow_array_builder( + arrow::StructBuilder* builder, const AffixFuzzer4* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + return arrow::Status::NotImplemented(("TODO(andreas) Handle nullable extensions")); + + return arrow::Status::OK(); + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer4.hpp b/rerun_cpp/src/components/affix_fuzzer4.hpp index bd0f478a2bad..5ab17b0629be 100644 --- a/rerun_cpp/src/components/affix_fuzzer4.hpp +++ b/rerun_cpp/src/components/affix_fuzzer4.hpp @@ -5,15 +5,11 @@ #include "../datatypes/affix_fuzzer1.hpp" +#include #include -#include #include #include -namespace arrow { - class DataType; -} - namespace rr { namespace components { struct AffixFuzzer4 { @@ -25,6 +21,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const AffixFuzzer4* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer5.cpp b/rerun_cpp/src/components/affix_fuzzer5.cpp index ba18315c31c9..f6080fafc63d 100644 --- a/rerun_cpp/src/components/affix_fuzzer5.cpp +++ b/rerun_cpp/src/components/affix_fuzzer5.cpp @@ -12,5 +12,32 @@ namespace rr { std::shared_ptr AffixFuzzer5::to_arrow_datatype() { return rr::datatypes::AffixFuzzer1::to_arrow_datatype(); } + + arrow::Result> AffixFuzzer5::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result( + rr::datatypes::AffixFuzzer1::new_arrow_array_builder(memory_pool).ValueOrDie() + ); + } + + arrow::Status AffixFuzzer5::fill_arrow_array_builder( + arrow::StructBuilder* builder, const AffixFuzzer5* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + return arrow::Status::NotImplemented(("TODO(andreas) Handle nullable extensions")); + + return arrow::Status::OK(); + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer5.hpp b/rerun_cpp/src/components/affix_fuzzer5.hpp index c376bd5bff54..f6ed7ebb2bbd 100644 --- a/rerun_cpp/src/components/affix_fuzzer5.hpp +++ b/rerun_cpp/src/components/affix_fuzzer5.hpp @@ -5,15 +5,11 @@ #include "../datatypes/affix_fuzzer1.hpp" +#include #include -#include #include #include -namespace arrow { - class DataType; -} - namespace rr { namespace components { struct AffixFuzzer5 { @@ -25,6 +21,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const AffixFuzzer5* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer6.cpp b/rerun_cpp/src/components/affix_fuzzer6.cpp index 03699aeee233..f6eb36be291f 100644 --- a/rerun_cpp/src/components/affix_fuzzer6.cpp +++ b/rerun_cpp/src/components/affix_fuzzer6.cpp @@ -12,5 +12,32 @@ namespace rr { std::shared_ptr AffixFuzzer6::to_arrow_datatype() { return rr::datatypes::AffixFuzzer1::to_arrow_datatype(); } + + arrow::Result> AffixFuzzer6::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result( + rr::datatypes::AffixFuzzer1::new_arrow_array_builder(memory_pool).ValueOrDie() + ); + } + + arrow::Status AffixFuzzer6::fill_arrow_array_builder( + arrow::StructBuilder* builder, const AffixFuzzer6* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + return arrow::Status::NotImplemented(("TODO(andreas) Handle nullable extensions")); + + return arrow::Status::OK(); + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer6.hpp b/rerun_cpp/src/components/affix_fuzzer6.hpp index 0de4f39d7170..78f27128c35f 100644 --- a/rerun_cpp/src/components/affix_fuzzer6.hpp +++ b/rerun_cpp/src/components/affix_fuzzer6.hpp @@ -5,15 +5,11 @@ #include "../datatypes/affix_fuzzer1.hpp" +#include #include -#include #include #include -namespace arrow { - class DataType; -} - namespace rr { namespace components { struct AffixFuzzer6 { @@ -25,6 +21,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const AffixFuzzer6* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer7.cpp b/rerun_cpp/src/components/affix_fuzzer7.cpp index 5b820f97cc2c..30d4bb200b15 100644 --- a/rerun_cpp/src/components/affix_fuzzer7.cpp +++ b/rerun_cpp/src/components/affix_fuzzer7.cpp @@ -11,7 +11,41 @@ namespace rr { namespace components { std::shared_ptr AffixFuzzer7::to_arrow_datatype() { return arrow::list(arrow::field( - "item", rr::datatypes::AffixFuzzer1::to_arrow_datatype(), true, nullptr)); + "item", + rr::datatypes::AffixFuzzer1::to_arrow_datatype(), + true, + nullptr + )); + } + + arrow::Result> AffixFuzzer7::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + memory_pool, + rr::datatypes::AffixFuzzer1::new_arrow_array_builder(memory_pool).ValueOrDie() + )); + } + + arrow::Status AffixFuzzer7::fill_arrow_array_builder( + arrow::ListBuilder* builder, const AffixFuzzer7* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + return arrow::Status::NotImplemented( + "TODO(andreas): custom data types in lists/fixedsizelist are not yet implemented" + ); + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer7.hpp b/rerun_cpp/src/components/affix_fuzzer7.hpp index 33de99180505..1e9cd32bef98 100644 --- a/rerun_cpp/src/components/affix_fuzzer7.hpp +++ b/rerun_cpp/src/components/affix_fuzzer7.hpp @@ -5,16 +5,12 @@ #include "../datatypes/affix_fuzzer1.hpp" +#include #include -#include #include #include #include -namespace arrow { - class DataType; -} - namespace rr { namespace components { struct AffixFuzzer7 { @@ -26,6 +22,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::ListBuilder* builder, const AffixFuzzer7* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer8.cpp b/rerun_cpp/src/components/affix_fuzzer8.cpp index 1be2d22a9c4c..2ac5850087c3 100644 --- a/rerun_cpp/src/components/affix_fuzzer8.cpp +++ b/rerun_cpp/src/components/affix_fuzzer8.cpp @@ -10,5 +10,38 @@ namespace rr { std::shared_ptr AffixFuzzer8::to_arrow_datatype() { return arrow::float32(); } + + arrow::Result> AffixFuzzer8::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared(memory_pool)); + } + + arrow::Status AffixFuzzer8::fill_arrow_array_builder( + arrow::FloatBuilder* builder, const AffixFuzzer8* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + ARROW_RETURN_NOT_OK(builder->Reserve(num_elements)); + for (auto elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto& element = elements[elem_idx]; + if (element.single_float_optional.has_value()) { + ARROW_RETURN_NOT_OK(builder->Append(element.single_float_optional.value())); + } else { + ARROW_RETURN_NOT_OK(builder->AppendNull()); + } + } + + return arrow::Status::OK(); + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer8.hpp b/rerun_cpp/src/components/affix_fuzzer8.hpp index 48defb1b7a94..63a0925591f3 100644 --- a/rerun_cpp/src/components/affix_fuzzer8.hpp +++ b/rerun_cpp/src/components/affix_fuzzer8.hpp @@ -3,15 +3,11 @@ #pragma once +#include #include -#include #include #include -namespace arrow { - class DataType; -} - namespace rr { namespace components { struct AffixFuzzer8 { @@ -23,6 +19,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::FloatBuilder* builder, const AffixFuzzer8* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer9.cpp b/rerun_cpp/src/components/affix_fuzzer9.cpp index 54d23bf742fc..815fb20a74dc 100644 --- a/rerun_cpp/src/components/affix_fuzzer9.cpp +++ b/rerun_cpp/src/components/affix_fuzzer9.cpp @@ -10,5 +10,33 @@ namespace rr { std::shared_ptr AffixFuzzer9::to_arrow_datatype() { return arrow::utf8(); } + + arrow::Result> AffixFuzzer9::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared(memory_pool)); + } + + arrow::Status AffixFuzzer9::fill_arrow_array_builder( + arrow::StringBuilder* builder, const AffixFuzzer9* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + ARROW_RETURN_NOT_OK(builder->Reserve(num_elements)); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + ARROW_RETURN_NOT_OK(builder->Append(elements[elem_idx].single_string_required)); + } + + return arrow::Status::OK(); + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer9.hpp b/rerun_cpp/src/components/affix_fuzzer9.hpp index 8a89f340411f..c65fb7d5f5a3 100644 --- a/rerun_cpp/src/components/affix_fuzzer9.hpp +++ b/rerun_cpp/src/components/affix_fuzzer9.hpp @@ -3,15 +3,11 @@ #pragma once +#include #include -#include #include #include -namespace arrow { - class DataType; -} - namespace rr { namespace components { struct AffixFuzzer9 { @@ -23,6 +19,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StringBuilder* builder, const AffixFuzzer9* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/class_id.cpp b/rerun_cpp/src/components/class_id.cpp index 96bc2887d7cc..91e5835cfc5f 100644 --- a/rerun_cpp/src/components/class_id.cpp +++ b/rerun_cpp/src/components/class_id.cpp @@ -10,5 +10,31 @@ namespace rr { std::shared_ptr ClassId::to_arrow_datatype() { return arrow::uint16(); } + + arrow::Result> ClassId::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared(memory_pool)); + } + + arrow::Status ClassId::fill_arrow_array_builder( + arrow::UInt16Builder* builder, const ClassId* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + static_assert(sizeof(*elements) == sizeof(elements->id)); + ARROW_RETURN_NOT_OK(builder->AppendValues(&elements->id, num_elements)); + + return arrow::Status::OK(); + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/class_id.hpp b/rerun_cpp/src/components/class_id.hpp index e8663e0b93a7..7836c98f0077 100644 --- a/rerun_cpp/src/components/class_id.hpp +++ b/rerun_cpp/src/components/class_id.hpp @@ -3,14 +3,10 @@ #pragma once +#include #include -#include #include -namespace arrow { - class DataType; -} - namespace rr { namespace components { /// A 16-bit ID representing a type of semantic class. @@ -22,6 +18,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::UInt16Builder* builder, const ClassId* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/color.cpp b/rerun_cpp/src/components/color.cpp index a8a219d507d4..a40b178607e2 100644 --- a/rerun_cpp/src/components/color.cpp +++ b/rerun_cpp/src/components/color.cpp @@ -10,5 +10,31 @@ namespace rr { std::shared_ptr Color::to_arrow_datatype() { return arrow::uint32(); } + + arrow::Result> Color::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared(memory_pool)); + } + + arrow::Status Color::fill_arrow_array_builder( + arrow::UInt32Builder* builder, const Color* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + static_assert(sizeof(*elements) == sizeof(elements->rgba)); + ARROW_RETURN_NOT_OK(builder->AppendValues(&elements->rgba, num_elements)); + + return arrow::Status::OK(); + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/color.hpp b/rerun_cpp/src/components/color.hpp index 22c487e46286..0210c902283a 100644 --- a/rerun_cpp/src/components/color.hpp +++ b/rerun_cpp/src/components/color.hpp @@ -3,14 +3,10 @@ #pragma once +#include #include -#include #include -namespace arrow { - class DataType; -} - namespace rr { namespace components { /// An RGBA color tuple with unmultiplied/separate alpha, in sRGB gamma space with linear @@ -23,6 +19,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::UInt32Builder* builder, const Color* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/draw_order.cpp b/rerun_cpp/src/components/draw_order.cpp index 25c1f3d88ef5..fdd4b8d7922b 100644 --- a/rerun_cpp/src/components/draw_order.cpp +++ b/rerun_cpp/src/components/draw_order.cpp @@ -10,5 +10,31 @@ namespace rr { std::shared_ptr DrawOrder::to_arrow_datatype() { return arrow::float32(); } + + arrow::Result> DrawOrder::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared(memory_pool)); + } + + arrow::Status DrawOrder::fill_arrow_array_builder( + arrow::FloatBuilder* builder, const DrawOrder* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + static_assert(sizeof(*elements) == sizeof(elements->value)); + ARROW_RETURN_NOT_OK(builder->AppendValues(&elements->value, num_elements)); + + return arrow::Status::OK(); + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/draw_order.hpp b/rerun_cpp/src/components/draw_order.hpp index 8da0c3efface..417d4aad965d 100644 --- a/rerun_cpp/src/components/draw_order.hpp +++ b/rerun_cpp/src/components/draw_order.hpp @@ -3,14 +3,10 @@ #pragma once +#include #include -#include #include -namespace arrow { - class DataType; -} - namespace rr { namespace components { /// Draw order used for the display order of 2D elements. @@ -28,6 +24,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::FloatBuilder* builder, const DrawOrder* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/instance_key.cpp b/rerun_cpp/src/components/instance_key.cpp index 3ebf77f59f00..1c428100087f 100644 --- a/rerun_cpp/src/components/instance_key.cpp +++ b/rerun_cpp/src/components/instance_key.cpp @@ -10,5 +10,31 @@ namespace rr { std::shared_ptr InstanceKey::to_arrow_datatype() { return arrow::uint64(); } + + arrow::Result> InstanceKey::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared(memory_pool)); + } + + arrow::Status InstanceKey::fill_arrow_array_builder( + arrow::UInt64Builder* builder, const InstanceKey* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + static_assert(sizeof(*elements) == sizeof(elements->value)); + ARROW_RETURN_NOT_OK(builder->AppendValues(&elements->value, num_elements)); + + return arrow::Status::OK(); + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/instance_key.hpp b/rerun_cpp/src/components/instance_key.hpp index 561b9d753acc..fcae04055fed 100644 --- a/rerun_cpp/src/components/instance_key.hpp +++ b/rerun_cpp/src/components/instance_key.hpp @@ -3,14 +3,10 @@ #pragma once +#include #include -#include #include -namespace arrow { - class DataType; -} - namespace rr { namespace components { /// A unique numeric identifier for each individual instance within a batch. @@ -22,6 +18,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::UInt64Builder* builder, const InstanceKey* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/keypoint_id.cpp b/rerun_cpp/src/components/keypoint_id.cpp index c8bcd72a6b63..6871f4989368 100644 --- a/rerun_cpp/src/components/keypoint_id.cpp +++ b/rerun_cpp/src/components/keypoint_id.cpp @@ -10,5 +10,31 @@ namespace rr { std::shared_ptr KeypointId::to_arrow_datatype() { return arrow::uint16(); } + + arrow::Result> KeypointId::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared(memory_pool)); + } + + arrow::Status KeypointId::fill_arrow_array_builder( + arrow::UInt16Builder* builder, const KeypointId* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + static_assert(sizeof(*elements) == sizeof(elements->id)); + ARROW_RETURN_NOT_OK(builder->AppendValues(&elements->id, num_elements)); + + return arrow::Status::OK(); + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/keypoint_id.hpp b/rerun_cpp/src/components/keypoint_id.hpp index b2c8a985aeb6..ff29000da457 100644 --- a/rerun_cpp/src/components/keypoint_id.hpp +++ b/rerun_cpp/src/components/keypoint_id.hpp @@ -3,14 +3,10 @@ #pragma once +#include #include -#include #include -namespace arrow { - class DataType; -} - namespace rr { namespace components { /// A 16-bit ID representing a type of semantic keypoint within a class. @@ -22,6 +18,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::UInt16Builder* builder, const KeypointId* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/label.cpp b/rerun_cpp/src/components/label.cpp index af2d9bffdac8..f2c1fb409f78 100644 --- a/rerun_cpp/src/components/label.cpp +++ b/rerun_cpp/src/components/label.cpp @@ -10,5 +10,33 @@ namespace rr { std::shared_ptr Label::to_arrow_datatype() { return arrow::utf8(); } + + arrow::Result> Label::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared(memory_pool)); + } + + arrow::Status Label::fill_arrow_array_builder( + arrow::StringBuilder* builder, const Label* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + ARROW_RETURN_NOT_OK(builder->Reserve(num_elements)); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + ARROW_RETURN_NOT_OK(builder->Append(elements[elem_idx].value)); + } + + return arrow::Status::OK(); + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/label.hpp b/rerun_cpp/src/components/label.hpp index f2b51c7efb30..297343f68a6d 100644 --- a/rerun_cpp/src/components/label.hpp +++ b/rerun_cpp/src/components/label.hpp @@ -3,15 +3,11 @@ #pragma once +#include #include -#include #include #include -namespace arrow { - class DataType; -} - namespace rr { namespace components { /// A String label component. @@ -23,6 +19,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StringBuilder* builder, const Label* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/point2d.cpp b/rerun_cpp/src/components/point2d.cpp index 0e53d5d09b09..d55ee3798564 100644 --- a/rerun_cpp/src/components/point2d.cpp +++ b/rerun_cpp/src/components/point2d.cpp @@ -12,5 +12,37 @@ namespace rr { std::shared_ptr Point2D::to_arrow_datatype() { return rr::datatypes::Point2D::to_arrow_datatype(); } + + arrow::Result> Point2D::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result( + rr::datatypes::Point2D::new_arrow_array_builder(memory_pool).ValueOrDie() + ); + } + + arrow::Status Point2D::fill_arrow_array_builder( + arrow::StructBuilder *builder, const Point2D *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + static_assert(sizeof(rr::datatypes::Point2D) == sizeof(Point2D)); + ARROW_RETURN_NOT_OK(rr::datatypes::Point2D::fill_arrow_array_builder( + builder, + reinterpret_cast(elements), + num_elements + )); + + return arrow::Status::OK(); + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/point2d.hpp b/rerun_cpp/src/components/point2d.hpp index af611cf8e760..c729d8478b76 100644 --- a/rerun_cpp/src/components/point2d.hpp +++ b/rerun_cpp/src/components/point2d.hpp @@ -5,14 +5,10 @@ #include "../datatypes/point2d.hpp" +#include #include -#include #include -namespace arrow { - class DataType; -} - namespace rr { namespace components { /// A point in 2D space. @@ -24,6 +20,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const Point2D* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/point3d.cpp b/rerun_cpp/src/components/point3d.cpp index ff4b363341bb..3566bac48ae0 100644 --- a/rerun_cpp/src/components/point3d.cpp +++ b/rerun_cpp/src/components/point3d.cpp @@ -12,5 +12,37 @@ namespace rr { std::shared_ptr Point3D::to_arrow_datatype() { return rr::datatypes::Point3D::to_arrow_datatype(); } + + arrow::Result> Point3D::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result( + rr::datatypes::Point3D::new_arrow_array_builder(memory_pool).ValueOrDie() + ); + } + + arrow::Status Point3D::fill_arrow_array_builder( + arrow::StructBuilder *builder, const Point3D *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + static_assert(sizeof(rr::datatypes::Point3D) == sizeof(Point3D)); + ARROW_RETURN_NOT_OK(rr::datatypes::Point3D::fill_arrow_array_builder( + builder, + reinterpret_cast(elements), + num_elements + )); + + return arrow::Status::OK(); + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/point3d.hpp b/rerun_cpp/src/components/point3d.hpp index f3f28c0f0891..72e196ff194a 100644 --- a/rerun_cpp/src/components/point3d.hpp +++ b/rerun_cpp/src/components/point3d.hpp @@ -5,14 +5,10 @@ #include "../datatypes/point3d.hpp" +#include #include -#include #include -namespace arrow { - class DataType; -} - namespace rr { namespace components { /// A point in 3D space. @@ -24,6 +20,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const Point3D* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/radius.cpp b/rerun_cpp/src/components/radius.cpp index 0ebee94ca034..30980e6119a6 100644 --- a/rerun_cpp/src/components/radius.cpp +++ b/rerun_cpp/src/components/radius.cpp @@ -10,5 +10,31 @@ namespace rr { std::shared_ptr Radius::to_arrow_datatype() { return arrow::float32(); } + + arrow::Result> Radius::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared(memory_pool)); + } + + arrow::Status Radius::fill_arrow_array_builder( + arrow::FloatBuilder* builder, const Radius* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + static_assert(sizeof(*elements) == sizeof(elements->value)); + ARROW_RETURN_NOT_OK(builder->AppendValues(&elements->value, num_elements)); + + return arrow::Status::OK(); + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/radius.hpp b/rerun_cpp/src/components/radius.hpp index 9fc1aafde961..12841b6c5240 100644 --- a/rerun_cpp/src/components/radius.hpp +++ b/rerun_cpp/src/components/radius.hpp @@ -3,14 +3,10 @@ #pragma once +#include #include -#include #include -namespace arrow { - class DataType; -} - namespace rr { namespace components { /// A Radius component. @@ -22,6 +18,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::FloatBuilder* builder, const Radius* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/transform3d.cpp b/rerun_cpp/src/components/transform3d.cpp index 022fa1c2668e..8678683c86a1 100644 --- a/rerun_cpp/src/components/transform3d.cpp +++ b/rerun_cpp/src/components/transform3d.cpp @@ -12,5 +12,36 @@ namespace rr { std::shared_ptr Transform3D::to_arrow_datatype() { return rr::datatypes::Transform3D::to_arrow_datatype(); } + + arrow::Result> + Transform3D::new_arrow_array_builder(arrow::MemoryPool *memory_pool) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result( + rr::datatypes::Transform3D::new_arrow_array_builder(memory_pool).ValueOrDie() + ); + } + + arrow::Status Transform3D::fill_arrow_array_builder( + arrow::DenseUnionBuilder *builder, const Transform3D *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + static_assert(sizeof(rr::datatypes::Transform3D) == sizeof(Transform3D)); + ARROW_RETURN_NOT_OK(rr::datatypes::Transform3D::fill_arrow_array_builder( + builder, + reinterpret_cast(elements), + num_elements + )); + + return arrow::Status::OK(); + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/transform3d.hpp b/rerun_cpp/src/components/transform3d.hpp index 9d049d24358e..0853d1bc7b2d 100644 --- a/rerun_cpp/src/components/transform3d.hpp +++ b/rerun_cpp/src/components/transform3d.hpp @@ -5,14 +5,10 @@ #include "../datatypes/transform3d.hpp" +#include #include -#include #include -namespace arrow { - class DataType; -} - namespace rr { namespace components { /// An affine transform between two 3D spaces, represented in a given direction. @@ -25,6 +21,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::DenseUnionBuilder* builder, const Transform3D* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/datatypes/affix_fuzzer1.cpp b/rerun_cpp/src/datatypes/affix_fuzzer1.cpp index ddcc8c816f3c..c5423b18c8bc 100644 --- a/rerun_cpp/src/datatypes/affix_fuzzer1.cpp +++ b/rerun_cpp/src/datatypes/affix_fuzzer1.cpp @@ -14,25 +14,156 @@ namespace rr { arrow::field("single_float_optional", arrow::float32(), true, nullptr), arrow::field("single_string_required", arrow::utf8(), false, nullptr), arrow::field("single_string_optional", arrow::utf8(), true, nullptr), - arrow::field("many_floats_optional", - arrow::list(arrow::field("item", arrow::float32(), true, nullptr)), - true, - nullptr), - arrow::field("many_strings_required", - arrow::list(arrow::field("item", arrow::utf8(), false, nullptr)), - false, - nullptr), - arrow::field("many_strings_optional", - arrow::list(arrow::field("item", arrow::utf8(), true, nullptr)), - true, - nullptr), + arrow::field( + "many_floats_optional", + arrow::list(arrow::field("item", arrow::float32(), true, nullptr)), + true, + nullptr + ), + arrow::field( + "many_strings_required", + arrow::list(arrow::field("item", arrow::utf8(), false, nullptr)), + false, + nullptr + ), + arrow::field( + "many_strings_optional", + arrow::list(arrow::field("item", arrow::utf8(), true, nullptr)), + true, + nullptr + ), arrow::field("flattened_scalar", arrow::float32(), false, nullptr), - arrow::field("almost_flattened_scalar", - rr::datatypes::FlattenedScalar::to_arrow_datatype(), - false, - nullptr), + arrow::field( + "almost_flattened_scalar", + rr::datatypes::FlattenedScalar::to_arrow_datatype(), + false, + nullptr + ), arrow::field("from_parent", arrow::boolean(), true, nullptr), }); } + + arrow::Result> AffixFuzzer1::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + to_arrow_datatype(), + memory_pool, + std::vector>({ + std::make_shared(memory_pool), + std::make_shared(memory_pool), + std::make_shared(memory_pool), + std::make_shared( + memory_pool, + std::make_shared(memory_pool) + ), + std::make_shared( + memory_pool, + std::make_shared(memory_pool) + ), + std::make_shared( + memory_pool, + std::make_shared(memory_pool) + ), + std::make_shared(memory_pool), + rr::datatypes::FlattenedScalar::new_arrow_array_builder(memory_pool) + .ValueOrDie(), + std::make_shared(memory_pool), + }) + )); + } + + arrow::Status AffixFuzzer1::fill_arrow_array_builder( + arrow::StructBuilder *builder, const AffixFuzzer1 *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + { + auto element_builder = + static_cast(builder->field_builder(0)); + ARROW_RETURN_NOT_OK(element_builder->Reserve(num_elements)); + for (auto elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto &element = elements[elem_idx]; + if (element.single_float_optional.has_value()) { + ARROW_RETURN_NOT_OK( + element_builder->Append(element.single_float_optional.value()) + ); + } else { + ARROW_RETURN_NOT_OK(element_builder->AppendNull()); + } + } + } + { + auto element_builder = + static_cast(builder->field_builder(1)); + ARROW_RETURN_NOT_OK(element_builder->Reserve(num_elements)); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + ARROW_RETURN_NOT_OK( + element_builder->Append(elements[elem_idx].single_string_required) + ); + } + } + { + auto element_builder = + static_cast(builder->field_builder(2)); + ARROW_RETURN_NOT_OK(element_builder->Reserve(num_elements)); + for (auto elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto &element = elements[elem_idx]; + if (element.single_string_optional.has_value()) { + ARROW_RETURN_NOT_OK( + element_builder->Append(element.single_string_optional.value()) + ); + } else { + ARROW_RETURN_NOT_OK(element_builder->AppendNull()); + } + } + } + return arrow::Status::NotImplemented( + "TODO(andreas): lists in structs are not yet supported" + ); + return arrow::Status::NotImplemented( + "TODO(andreas): lists in structs are not yet supported" + ); + return arrow::Status::NotImplemented( + "TODO(andreas): lists in structs are not yet supported" + ); + { + auto element_builder = + static_cast(builder->field_builder(6)); + ARROW_RETURN_NOT_OK(element_builder->Reserve(num_elements)); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + ARROW_RETURN_NOT_OK(element_builder->Append(elements[elem_idx].flattened_scalar) + ); + } + } + return arrow::Status::NotImplemented( + "TODO(andreas): extensions in structs are not yet supported" + ); + { + auto element_builder = + static_cast(builder->field_builder(8)); + ARROW_RETURN_NOT_OK(element_builder->Reserve(num_elements)); + for (auto elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto &element = elements[elem_idx]; + if (element.from_parent.has_value()) { + ARROW_RETURN_NOT_OK(element_builder->Append(element.from_parent.value())); + } else { + ARROW_RETURN_NOT_OK(element_builder->AppendNull()); + } + } + } + ARROW_RETURN_NOT_OK(builder->AppendValues(num_elements, nullptr)); + + return arrow::Status::OK(); + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/affix_fuzzer1.hpp b/rerun_cpp/src/datatypes/affix_fuzzer1.hpp index 922ff3042a81..6626e3314f8b 100644 --- a/rerun_cpp/src/datatypes/affix_fuzzer1.hpp +++ b/rerun_cpp/src/datatypes/affix_fuzzer1.hpp @@ -5,16 +5,12 @@ #include "../datatypes/flattened_scalar.hpp" +#include #include -#include #include #include #include -namespace arrow { - class DataType; -} - namespace rr { namespace datatypes { struct AffixFuzzer1 { @@ -39,6 +35,16 @@ namespace rr { public: /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const AffixFuzzer1* elements, size_t num_elements + ); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/affix_fuzzer2.cpp b/rerun_cpp/src/datatypes/affix_fuzzer2.cpp index 7b83c7618ccd..20a26bd384b8 100644 --- a/rerun_cpp/src/datatypes/affix_fuzzer2.cpp +++ b/rerun_cpp/src/datatypes/affix_fuzzer2.cpp @@ -10,5 +10,38 @@ namespace rr { std::shared_ptr AffixFuzzer2::to_arrow_datatype() { return arrow::float32(); } + + arrow::Result> AffixFuzzer2::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared(memory_pool)); + } + + arrow::Status AffixFuzzer2::fill_arrow_array_builder( + arrow::FloatBuilder* builder, const AffixFuzzer2* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + ARROW_RETURN_NOT_OK(builder->Reserve(num_elements)); + for (auto elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto& element = elements[elem_idx]; + if (element.single_float_optional.has_value()) { + ARROW_RETURN_NOT_OK(builder->Append(element.single_float_optional.value())); + } else { + ARROW_RETURN_NOT_OK(builder->AppendNull()); + } + } + + return arrow::Status::OK(); + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/affix_fuzzer2.hpp b/rerun_cpp/src/datatypes/affix_fuzzer2.hpp index 3db3f9b54b7b..b0c9d0ca3dd2 100644 --- a/rerun_cpp/src/datatypes/affix_fuzzer2.hpp +++ b/rerun_cpp/src/datatypes/affix_fuzzer2.hpp @@ -3,15 +3,11 @@ #pragma once +#include #include -#include #include #include -namespace arrow { - class DataType; -} - namespace rr { namespace datatypes { struct AffixFuzzer2 { @@ -23,6 +19,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::FloatBuilder* builder, const AffixFuzzer2* elements, size_t num_elements + ); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/affix_fuzzer3.cpp b/rerun_cpp/src/datatypes/affix_fuzzer3.cpp index f84a05cbc457..d76727c7be3c 100644 --- a/rerun_cpp/src/datatypes/affix_fuzzer3.cpp +++ b/rerun_cpp/src/datatypes/affix_fuzzer3.cpp @@ -17,15 +17,69 @@ namespace rr { arrow::field( "craziness", arrow::list(arrow::field( - "item", rr::datatypes::AffixFuzzer1::to_arrow_datatype(), false, nullptr)), + "item", + rr::datatypes::AffixFuzzer1::to_arrow_datatype(), + false, + nullptr + )), false, - nullptr), - arrow::field("fixed_size_shenanigans", - arrow::fixed_size_list( - arrow::field("item", arrow::float32(), false, nullptr), 3), - false, - nullptr), + nullptr + ), + arrow::field( + "fixed_size_shenanigans", + arrow::fixed_size_list( + arrow::field("item", arrow::float32(), false, nullptr), + 3 + ), + false, + nullptr + ), }); } + + arrow::Result> + AffixFuzzer3::new_arrow_array_builder(arrow::MemoryPool* memory_pool) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + memory_pool, + std::vector>({ + std::make_shared(memory_pool), + std::make_shared(memory_pool), + std::make_shared(memory_pool), + std::make_shared( + memory_pool, + rr::datatypes::AffixFuzzer1::new_arrow_array_builder(memory_pool) + .ValueOrDie() + ), + std::make_shared( + memory_pool, + std::make_shared(memory_pool), + 3 + ), + }), + to_arrow_datatype() + )); + } + + arrow::Status AffixFuzzer3::fill_arrow_array_builder( + arrow::DenseUnionBuilder* builder, const AffixFuzzer3* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto& element = elements[elem_idx]; + } + return arrow::Status::NotImplemented("TODO(andreas): unions are not yet implemented"); + + return arrow::Status::OK(); + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/affix_fuzzer3.hpp b/rerun_cpp/src/datatypes/affix_fuzzer3.hpp index e5cbabd1f772..fd12d4917c96 100644 --- a/rerun_cpp/src/datatypes/affix_fuzzer3.hpp +++ b/rerun_cpp/src/datatypes/affix_fuzzer3.hpp @@ -5,9 +5,9 @@ #include "../datatypes/affix_fuzzer1.hpp" +#include #include #include -#include #include #include #include @@ -125,6 +125,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::DenseUnionBuilder* builder, const AffixFuzzer3* elements, size_t num_elements + ); + void swap(AffixFuzzer3& other) noexcept { auto tag_temp = this->_tag; this->_tag = other._tag; diff --git a/rerun_cpp/src/datatypes/affix_fuzzer4.cpp b/rerun_cpp/src/datatypes/affix_fuzzer4.cpp index ca241ac777a3..e39e3fbd3a96 100644 --- a/rerun_cpp/src/datatypes/affix_fuzzer4.cpp +++ b/rerun_cpp/src/datatypes/affix_fuzzer4.cpp @@ -12,23 +12,79 @@ namespace rr { std::shared_ptr AffixFuzzer4::to_arrow_datatype() { return arrow::dense_union({ arrow::field("_null_markers", arrow::null(), true, nullptr), - arrow::field("single_required", - rr::datatypes::AffixFuzzer3::to_arrow_datatype(), - false, - nullptr), + arrow::field( + "single_required", + rr::datatypes::AffixFuzzer3::to_arrow_datatype(), + false, + nullptr + ), arrow::field( "many_required", arrow::list(arrow::field( - "item", rr::datatypes::AffixFuzzer3::to_arrow_datatype(), false, nullptr)), + "item", + rr::datatypes::AffixFuzzer3::to_arrow_datatype(), + false, + nullptr + )), false, - nullptr), + nullptr + ), arrow::field( "many_optional", arrow::list(arrow::field( - "item", rr::datatypes::AffixFuzzer3::to_arrow_datatype(), true, nullptr)), + "item", + rr::datatypes::AffixFuzzer3::to_arrow_datatype(), + true, + nullptr + )), false, - nullptr), + nullptr + ), }); } + + arrow::Result> + AffixFuzzer4::new_arrow_array_builder(arrow::MemoryPool* memory_pool) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + memory_pool, + std::vector>({ + std::make_shared(memory_pool), + rr::datatypes::AffixFuzzer3::new_arrow_array_builder(memory_pool).ValueOrDie(), + std::make_shared( + memory_pool, + rr::datatypes::AffixFuzzer3::new_arrow_array_builder(memory_pool) + .ValueOrDie() + ), + std::make_shared( + memory_pool, + rr::datatypes::AffixFuzzer3::new_arrow_array_builder(memory_pool) + .ValueOrDie() + ), + }), + to_arrow_datatype() + )); + } + + arrow::Status AffixFuzzer4::fill_arrow_array_builder( + arrow::DenseUnionBuilder* builder, const AffixFuzzer4* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto& element = elements[elem_idx]; + } + return arrow::Status::NotImplemented("TODO(andreas): unions are not yet implemented"); + + return arrow::Status::OK(); + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/affix_fuzzer4.hpp b/rerun_cpp/src/datatypes/affix_fuzzer4.hpp index cb60e5e9a36e..2c1fc11f799e 100644 --- a/rerun_cpp/src/datatypes/affix_fuzzer4.hpp +++ b/rerun_cpp/src/datatypes/affix_fuzzer4.hpp @@ -5,9 +5,9 @@ #include "../datatypes/affix_fuzzer3.hpp" +#include #include #include -#include #include #include #include @@ -96,8 +96,8 @@ namespace rr { return std::move(self); } - static AffixFuzzer4 many_required( - std::vector many_required) { + static AffixFuzzer4 many_required(std::vector many_required + ) { typedef std::vector TypeAlias; AffixFuzzer4 self; self._tag = detail::AffixFuzzer4Tag::many_required; @@ -106,7 +106,8 @@ namespace rr { } static AffixFuzzer4 many_optional( - std::optional> many_optional) { + std::optional> many_optional + ) { typedef std::optional> TypeAlias; AffixFuzzer4 self; self._tag = detail::AffixFuzzer4Tag::many_optional; @@ -117,6 +118,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::DenseUnionBuilder* builder, const AffixFuzzer4* elements, size_t num_elements + ); + void swap(AffixFuzzer4& other) noexcept { auto tag_temp = this->_tag; this->_tag = other._tag; diff --git a/rerun_cpp/src/datatypes/affix_fuzzer5.cpp b/rerun_cpp/src/datatypes/affix_fuzzer5.cpp index 7211ae386353..bcf69f78142b 100644 --- a/rerun_cpp/src/datatypes/affix_fuzzer5.cpp +++ b/rerun_cpp/src/datatypes/affix_fuzzer5.cpp @@ -11,11 +11,47 @@ namespace rr { namespace datatypes { std::shared_ptr AffixFuzzer5::to_arrow_datatype() { return arrow::struct_({ - arrow::field("single_optional_union", - rr::datatypes::AffixFuzzer4::to_arrow_datatype(), - true, - nullptr), + arrow::field( + "single_optional_union", + rr::datatypes::AffixFuzzer4::to_arrow_datatype(), + true, + nullptr + ), }); } + + arrow::Result> AffixFuzzer5::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + to_arrow_datatype(), + memory_pool, + std::vector>({ + rr::datatypes::AffixFuzzer4::new_arrow_array_builder(memory_pool).ValueOrDie(), + }) + )); + } + + arrow::Status AffixFuzzer5::fill_arrow_array_builder( + arrow::StructBuilder* builder, const AffixFuzzer5* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + return arrow::Status::NotImplemented( + "TODO(andreas): extensions in structs are not yet supported" + ); + ARROW_RETURN_NOT_OK(builder->AppendValues(num_elements, nullptr)); + + return arrow::Status::OK(); + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/affix_fuzzer5.hpp b/rerun_cpp/src/datatypes/affix_fuzzer5.hpp index bff009a92ce3..b68bcf488589 100644 --- a/rerun_cpp/src/datatypes/affix_fuzzer5.hpp +++ b/rerun_cpp/src/datatypes/affix_fuzzer5.hpp @@ -5,15 +5,11 @@ #include "../datatypes/affix_fuzzer4.hpp" +#include #include -#include #include #include -namespace arrow { - class DataType; -} - namespace rr { namespace datatypes { struct AffixFuzzer5 { @@ -25,6 +21,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const AffixFuzzer5* elements, size_t num_elements + ); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/angle.cpp b/rerun_cpp/src/datatypes/angle.cpp index 25517f5efcb3..7371cebbc935 100644 --- a/rerun_cpp/src/datatypes/angle.cpp +++ b/rerun_cpp/src/datatypes/angle.cpp @@ -14,5 +14,41 @@ namespace rr { arrow::field("Degrees", arrow::float32(), false, nullptr), }); } + + arrow::Result> Angle::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + memory_pool, + std::vector>({ + std::make_shared(memory_pool), + std::make_shared(memory_pool), + std::make_shared(memory_pool), + }), + to_arrow_datatype() + )); + } + + arrow::Status Angle::fill_arrow_array_builder( + arrow::DenseUnionBuilder* builder, const Angle* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto& element = elements[elem_idx]; + } + return arrow::Status::NotImplemented("TODO(andreas): unions are not yet implemented"); + + return arrow::Status::OK(); + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/angle.hpp b/rerun_cpp/src/datatypes/angle.hpp index 26d880318fec..59bf5a92cd63 100644 --- a/rerun_cpp/src/datatypes/angle.hpp +++ b/rerun_cpp/src/datatypes/angle.hpp @@ -3,9 +3,9 @@ #pragma once +#include #include #include -#include #include namespace rr { @@ -75,6 +75,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::DenseUnionBuilder* builder, const Angle* elements, size_t num_elements + ); + void swap(Angle& other) noexcept { auto tag_temp = this->_tag; this->_tag = other._tag; diff --git a/rerun_cpp/src/datatypes/flattened_scalar.cpp b/rerun_cpp/src/datatypes/flattened_scalar.cpp index f54aff725814..1fa7c7b7e034 100644 --- a/rerun_cpp/src/datatypes/flattened_scalar.cpp +++ b/rerun_cpp/src/datatypes/flattened_scalar.cpp @@ -12,5 +12,43 @@ namespace rr { arrow::field("value", arrow::float32(), false, nullptr), }); } + + arrow::Result> + FlattenedScalar::new_arrow_array_builder(arrow::MemoryPool *memory_pool) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + to_arrow_datatype(), + memory_pool, + std::vector>({ + std::make_shared(memory_pool), + }) + )); + } + + arrow::Status FlattenedScalar::fill_arrow_array_builder( + arrow::StructBuilder *builder, const FlattenedScalar *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + { + auto element_builder = + static_cast(builder->field_builder(0)); + ARROW_RETURN_NOT_OK(element_builder->Reserve(num_elements)); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + ARROW_RETURN_NOT_OK(element_builder->Append(elements[elem_idx].value)); + } + } + ARROW_RETURN_NOT_OK(builder->AppendValues(num_elements, nullptr)); + + return arrow::Status::OK(); + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/flattened_scalar.hpp b/rerun_cpp/src/datatypes/flattened_scalar.hpp index 0e5ce80b7690..6dcd047a807c 100644 --- a/rerun_cpp/src/datatypes/flattened_scalar.hpp +++ b/rerun_cpp/src/datatypes/flattened_scalar.hpp @@ -3,14 +3,10 @@ #pragma once +#include #include -#include #include -namespace arrow { - class DataType; -} - namespace rr { namespace datatypes { struct FlattenedScalar { @@ -21,6 +17,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const FlattenedScalar* elements, size_t num_elements + ); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/mat3x3.cpp b/rerun_cpp/src/datatypes/mat3x3.cpp index 02c917d274a3..4ef61f626513 100644 --- a/rerun_cpp/src/datatypes/mat3x3.cpp +++ b/rerun_cpp/src/datatypes/mat3x3.cpp @@ -8,8 +8,45 @@ namespace rr { namespace datatypes { std::shared_ptr Mat3x3::to_arrow_datatype() { - return arrow::fixed_size_list(arrow::field("item", arrow::float32(), false, nullptr), - 9); + return arrow::fixed_size_list( + arrow::field("item", arrow::float32(), false, nullptr), + 9 + ); + } + + arrow::Result> Mat3x3::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + memory_pool, + std::make_shared(memory_pool), + 9 + )); + } + + arrow::Status Mat3x3::fill_arrow_array_builder( + arrow::FixedSizeListBuilder *builder, const Mat3x3 *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto value_builder = static_cast(builder->value_builder()); + + static_assert(sizeof(elements[0].coeffs) == sizeof(elements[0])); + ARROW_RETURN_NOT_OK( + value_builder->AppendValues(elements[0].coeffs, num_elements * 9, nullptr) + ); + ARROW_RETURN_NOT_OK(builder->AppendValues(num_elements)); + + return arrow::Status::OK(); } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/mat3x3.hpp b/rerun_cpp/src/datatypes/mat3x3.hpp index 9e605c364223..bc3ecee49130 100644 --- a/rerun_cpp/src/datatypes/mat3x3.hpp +++ b/rerun_cpp/src/datatypes/mat3x3.hpp @@ -3,12 +3,8 @@ #pragma once +#include #include -#include - -namespace arrow { - class DataType; -} namespace rr { namespace datatypes { @@ -19,6 +15,15 @@ namespace rr { public: /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> + new_arrow_array_builder(arrow::MemoryPool* memory_pool); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::FixedSizeListBuilder* builder, const Mat3x3* elements, size_t num_elements + ); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/mat4x4.cpp b/rerun_cpp/src/datatypes/mat4x4.cpp index 212fbfe5f87c..7d8454e3b430 100644 --- a/rerun_cpp/src/datatypes/mat4x4.cpp +++ b/rerun_cpp/src/datatypes/mat4x4.cpp @@ -8,8 +8,45 @@ namespace rr { namespace datatypes { std::shared_ptr Mat4x4::to_arrow_datatype() { - return arrow::fixed_size_list(arrow::field("item", arrow::float32(), false, nullptr), - 16); + return arrow::fixed_size_list( + arrow::field("item", arrow::float32(), false, nullptr), + 16 + ); + } + + arrow::Result> Mat4x4::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + memory_pool, + std::make_shared(memory_pool), + 16 + )); + } + + arrow::Status Mat4x4::fill_arrow_array_builder( + arrow::FixedSizeListBuilder *builder, const Mat4x4 *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto value_builder = static_cast(builder->value_builder()); + + static_assert(sizeof(elements[0].coeffs) == sizeof(elements[0])); + ARROW_RETURN_NOT_OK( + value_builder->AppendValues(elements[0].coeffs, num_elements * 16, nullptr) + ); + ARROW_RETURN_NOT_OK(builder->AppendValues(num_elements)); + + return arrow::Status::OK(); } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/mat4x4.hpp b/rerun_cpp/src/datatypes/mat4x4.hpp index 2793f21f4be7..43519801a0b9 100644 --- a/rerun_cpp/src/datatypes/mat4x4.hpp +++ b/rerun_cpp/src/datatypes/mat4x4.hpp @@ -3,12 +3,8 @@ #pragma once +#include #include -#include - -namespace arrow { - class DataType; -} namespace rr { namespace datatypes { @@ -19,6 +15,15 @@ namespace rr { public: /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> + new_arrow_array_builder(arrow::MemoryPool* memory_pool); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::FixedSizeListBuilder* builder, const Mat4x4* elements, size_t num_elements + ); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/point2d.cpp b/rerun_cpp/src/datatypes/point2d.cpp index da68dbd6b781..2835da471d52 100644 --- a/rerun_cpp/src/datatypes/point2d.cpp +++ b/rerun_cpp/src/datatypes/point2d.cpp @@ -13,5 +13,53 @@ namespace rr { arrow::field("y", arrow::float32(), false, nullptr), }); } + + arrow::Result> Point2D::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + to_arrow_datatype(), + memory_pool, + std::vector>({ + std::make_shared(memory_pool), + std::make_shared(memory_pool), + }) + )); + } + + arrow::Status Point2D::fill_arrow_array_builder( + arrow::StructBuilder *builder, const Point2D *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + { + auto element_builder = + static_cast(builder->field_builder(0)); + ARROW_RETURN_NOT_OK(element_builder->Reserve(num_elements)); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + ARROW_RETURN_NOT_OK(element_builder->Append(elements[elem_idx].x)); + } + } + { + auto element_builder = + static_cast(builder->field_builder(1)); + ARROW_RETURN_NOT_OK(element_builder->Reserve(num_elements)); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + ARROW_RETURN_NOT_OK(element_builder->Append(elements[elem_idx].y)); + } + } + ARROW_RETURN_NOT_OK(builder->AppendValues(num_elements, nullptr)); + + return arrow::Status::OK(); + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/point2d.hpp b/rerun_cpp/src/datatypes/point2d.hpp index 9d1cb74e9d7b..0cf76ab2062e 100644 --- a/rerun_cpp/src/datatypes/point2d.hpp +++ b/rerun_cpp/src/datatypes/point2d.hpp @@ -3,12 +3,8 @@ #pragma once +#include #include -#include - -namespace arrow { - class DataType; -} namespace rr { namespace datatypes { @@ -21,6 +17,16 @@ namespace rr { public: /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const Point2D* elements, size_t num_elements + ); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/point3d.cpp b/rerun_cpp/src/datatypes/point3d.cpp index f6bae5a0411b..bc24c2203cc1 100644 --- a/rerun_cpp/src/datatypes/point3d.cpp +++ b/rerun_cpp/src/datatypes/point3d.cpp @@ -14,5 +14,62 @@ namespace rr { arrow::field("z", arrow::float32(), false, nullptr), }); } + + arrow::Result> Point3D::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + to_arrow_datatype(), + memory_pool, + std::vector>({ + std::make_shared(memory_pool), + std::make_shared(memory_pool), + std::make_shared(memory_pool), + }) + )); + } + + arrow::Status Point3D::fill_arrow_array_builder( + arrow::StructBuilder *builder, const Point3D *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + { + auto element_builder = + static_cast(builder->field_builder(0)); + ARROW_RETURN_NOT_OK(element_builder->Reserve(num_elements)); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + ARROW_RETURN_NOT_OK(element_builder->Append(elements[elem_idx].x)); + } + } + { + auto element_builder = + static_cast(builder->field_builder(1)); + ARROW_RETURN_NOT_OK(element_builder->Reserve(num_elements)); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + ARROW_RETURN_NOT_OK(element_builder->Append(elements[elem_idx].y)); + } + } + { + auto element_builder = + static_cast(builder->field_builder(2)); + ARROW_RETURN_NOT_OK(element_builder->Reserve(num_elements)); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + ARROW_RETURN_NOT_OK(element_builder->Append(elements[elem_idx].z)); + } + } + ARROW_RETURN_NOT_OK(builder->AppendValues(num_elements, nullptr)); + + return arrow::Status::OK(); + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/point3d.hpp b/rerun_cpp/src/datatypes/point3d.hpp index 78b6e9b04e0b..437c60ed84ea 100644 --- a/rerun_cpp/src/datatypes/point3d.hpp +++ b/rerun_cpp/src/datatypes/point3d.hpp @@ -3,12 +3,8 @@ #pragma once +#include #include -#include - -namespace arrow { - class DataType; -} namespace rr { namespace datatypes { @@ -23,6 +19,16 @@ namespace rr { public: /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const Point3D* elements, size_t num_elements + ); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/quaternion.cpp b/rerun_cpp/src/datatypes/quaternion.cpp index f206392e3c29..6f2f7bd2d395 100644 --- a/rerun_cpp/src/datatypes/quaternion.cpp +++ b/rerun_cpp/src/datatypes/quaternion.cpp @@ -8,8 +8,44 @@ namespace rr { namespace datatypes { std::shared_ptr Quaternion::to_arrow_datatype() { - return arrow::fixed_size_list(arrow::field("item", arrow::float32(), false, nullptr), - 4); + return arrow::fixed_size_list( + arrow::field("item", arrow::float32(), false, nullptr), + 4 + ); + } + + arrow::Result> + Quaternion::new_arrow_array_builder(arrow::MemoryPool *memory_pool) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + memory_pool, + std::make_shared(memory_pool), + 4 + )); + } + + arrow::Status Quaternion::fill_arrow_array_builder( + arrow::FixedSizeListBuilder *builder, const Quaternion *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto value_builder = static_cast(builder->value_builder()); + + static_assert(sizeof(elements[0].xyzw) == sizeof(elements[0])); + ARROW_RETURN_NOT_OK( + value_builder->AppendValues(elements[0].xyzw, num_elements * 4, nullptr) + ); + ARROW_RETURN_NOT_OK(builder->AppendValues(num_elements)); + + return arrow::Status::OK(); } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/quaternion.hpp b/rerun_cpp/src/datatypes/quaternion.hpp index 11abeed97f9a..40a4587ec56d 100644 --- a/rerun_cpp/src/datatypes/quaternion.hpp +++ b/rerun_cpp/src/datatypes/quaternion.hpp @@ -3,12 +3,8 @@ #pragma once +#include #include -#include - -namespace arrow { - class DataType; -} namespace rr { namespace datatypes { @@ -19,6 +15,16 @@ namespace rr { public: /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> + new_arrow_array_builder(arrow::MemoryPool* memory_pool); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::FixedSizeListBuilder* builder, const Quaternion* elements, + size_t num_elements + ); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/rotation3d.cpp b/rerun_cpp/src/datatypes/rotation3d.cpp index 72075a72c98b..86980103a742 100644 --- a/rerun_cpp/src/datatypes/rotation3d.cpp +++ b/rerun_cpp/src/datatypes/rotation3d.cpp @@ -14,12 +14,54 @@ namespace rr { return arrow::dense_union({ arrow::field("_null_markers", arrow::null(), true, nullptr), arrow::field( - "Quaternion", rr::datatypes::Quaternion::to_arrow_datatype(), false, nullptr), - arrow::field("AxisAngle", - rr::datatypes::RotationAxisAngle::to_arrow_datatype(), - false, - nullptr), + "Quaternion", + rr::datatypes::Quaternion::to_arrow_datatype(), + false, + nullptr + ), + arrow::field( + "AxisAngle", + rr::datatypes::RotationAxisAngle::to_arrow_datatype(), + false, + nullptr + ), }); } + + arrow::Result> + Rotation3D::new_arrow_array_builder(arrow::MemoryPool* memory_pool) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + memory_pool, + std::vector>({ + std::make_shared(memory_pool), + rr::datatypes::Quaternion::new_arrow_array_builder(memory_pool).ValueOrDie(), + rr::datatypes::RotationAxisAngle::new_arrow_array_builder(memory_pool) + .ValueOrDie(), + }), + to_arrow_datatype() + )); + } + + arrow::Status Rotation3D::fill_arrow_array_builder( + arrow::DenseUnionBuilder* builder, const Rotation3D* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto& element = elements[elem_idx]; + } + return arrow::Status::NotImplemented("TODO(andreas): unions are not yet implemented"); + + return arrow::Status::OK(); + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/rotation3d.hpp b/rerun_cpp/src/datatypes/rotation3d.hpp index b0617a856285..1852cdbdac80 100644 --- a/rerun_cpp/src/datatypes/rotation3d.hpp +++ b/rerun_cpp/src/datatypes/rotation3d.hpp @@ -6,9 +6,9 @@ #include "../datatypes/quaternion.hpp" #include "../datatypes/rotation_axis_angle.hpp" +#include #include #include -#include #include namespace rr { @@ -92,6 +92,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::DenseUnionBuilder* builder, const Rotation3D* elements, size_t num_elements + ); + void swap(Rotation3D& other) noexcept { auto tag_temp = this->_tag; this->_tag = other._tag; diff --git a/rerun_cpp/src/datatypes/rotation_axis_angle.cpp b/rerun_cpp/src/datatypes/rotation_axis_angle.cpp index 98c460ff92db..f46caad3761a 100644 --- a/rerun_cpp/src/datatypes/rotation_axis_angle.cpp +++ b/rerun_cpp/src/datatypes/rotation_axis_angle.cpp @@ -16,5 +16,42 @@ namespace rr { arrow::field("angle", rr::datatypes::Angle::to_arrow_datatype(), false, nullptr), }); } + + arrow::Result> + RotationAxisAngle::new_arrow_array_builder(arrow::MemoryPool* memory_pool) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + to_arrow_datatype(), + memory_pool, + std::vector>({ + rr::datatypes::Vec3D::new_arrow_array_builder(memory_pool).ValueOrDie(), + rr::datatypes::Angle::new_arrow_array_builder(memory_pool).ValueOrDie(), + }) + )); + } + + arrow::Status RotationAxisAngle::fill_arrow_array_builder( + arrow::StructBuilder* builder, const RotationAxisAngle* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + return arrow::Status::NotImplemented( + "TODO(andreas): extensions in structs are not yet supported" + ); + return arrow::Status::NotImplemented( + "TODO(andreas): extensions in structs are not yet supported" + ); + ARROW_RETURN_NOT_OK(builder->AppendValues(num_elements, nullptr)); + + return arrow::Status::OK(); + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/rotation_axis_angle.hpp b/rerun_cpp/src/datatypes/rotation_axis_angle.hpp index 894140bc5667..b80e9870e573 100644 --- a/rerun_cpp/src/datatypes/rotation_axis_angle.hpp +++ b/rerun_cpp/src/datatypes/rotation_axis_angle.hpp @@ -6,12 +6,8 @@ #include "../datatypes/angle.hpp" #include "../datatypes/vec3d.hpp" +#include #include -#include - -namespace arrow { - class DataType; -} namespace rr { namespace datatypes { @@ -30,6 +26,17 @@ namespace rr { public: /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const RotationAxisAngle* elements, + size_t num_elements + ); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/scale3d.cpp b/rerun_cpp/src/datatypes/scale3d.cpp index 6398dbcb5a74..0c839ea1189c 100644 --- a/rerun_cpp/src/datatypes/scale3d.cpp +++ b/rerun_cpp/src/datatypes/scale3d.cpp @@ -16,5 +16,41 @@ namespace rr { arrow::field("Uniform", arrow::float32(), false, nullptr), }); } + + arrow::Result> Scale3D::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + memory_pool, + std::vector>({ + std::make_shared(memory_pool), + rr::datatypes::Vec3D::new_arrow_array_builder(memory_pool).ValueOrDie(), + std::make_shared(memory_pool), + }), + to_arrow_datatype() + )); + } + + arrow::Status Scale3D::fill_arrow_array_builder( + arrow::DenseUnionBuilder* builder, const Scale3D* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto& element = elements[elem_idx]; + } + return arrow::Status::NotImplemented("TODO(andreas): unions are not yet implemented"); + + return arrow::Status::OK(); + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/scale3d.hpp b/rerun_cpp/src/datatypes/scale3d.hpp index 2d1be4ca6729..d18ba6f6c016 100644 --- a/rerun_cpp/src/datatypes/scale3d.hpp +++ b/rerun_cpp/src/datatypes/scale3d.hpp @@ -5,9 +5,9 @@ #include "../datatypes/vec3d.hpp" +#include #include #include -#include #include namespace rr { @@ -91,6 +91,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::DenseUnionBuilder* builder, const Scale3D* elements, size_t num_elements + ); + void swap(Scale3D& other) noexcept { auto tag_temp = this->_tag; this->_tag = other._tag; diff --git a/rerun_cpp/src/datatypes/transform3d.cpp b/rerun_cpp/src/datatypes/transform3d.cpp index 1c24cda5390f..4943990e83dc 100644 --- a/rerun_cpp/src/datatypes/transform3d.cpp +++ b/rerun_cpp/src/datatypes/transform3d.cpp @@ -13,15 +13,56 @@ namespace rr { std::shared_ptr Transform3D::to_arrow_datatype() { return arrow::dense_union({ arrow::field("_null_markers", arrow::null(), true, nullptr), - arrow::field("TranslationAndMat3x3", - rr::datatypes::TranslationAndMat3x3::to_arrow_datatype(), - false, - nullptr), - arrow::field("TranslationRotationScale", - rr::datatypes::TranslationRotationScale3D::to_arrow_datatype(), - false, - nullptr), + arrow::field( + "TranslationAndMat3x3", + rr::datatypes::TranslationAndMat3x3::to_arrow_datatype(), + false, + nullptr + ), + arrow::field( + "TranslationRotationScale", + rr::datatypes::TranslationRotationScale3D::to_arrow_datatype(), + false, + nullptr + ), }); } + + arrow::Result> + Transform3D::new_arrow_array_builder(arrow::MemoryPool* memory_pool) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + memory_pool, + std::vector>({ + std::make_shared(memory_pool), + rr::datatypes::TranslationAndMat3x3::new_arrow_array_builder(memory_pool) + .ValueOrDie(), + rr::datatypes::TranslationRotationScale3D::new_arrow_array_builder(memory_pool) + .ValueOrDie(), + }), + to_arrow_datatype() + )); + } + + arrow::Status Transform3D::fill_arrow_array_builder( + arrow::DenseUnionBuilder* builder, const Transform3D* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto& element = elements[elem_idx]; + } + return arrow::Status::NotImplemented("TODO(andreas): unions are not yet implemented"); + + return arrow::Status::OK(); + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/transform3d.hpp b/rerun_cpp/src/datatypes/transform3d.hpp index fe61b8a8025f..302e366b116f 100644 --- a/rerun_cpp/src/datatypes/transform3d.hpp +++ b/rerun_cpp/src/datatypes/transform3d.hpp @@ -6,9 +6,9 @@ #include "../datatypes/translation_and_mat3x3.hpp" #include "../datatypes/translation_rotation_scale3d.hpp" +#include #include #include -#include #include namespace rr { @@ -62,7 +62,8 @@ namespace rr { } static Transform3D translation_and_mat3x3( - rr::datatypes::TranslationAndMat3x3 translation_and_mat3x3) { + rr::datatypes::TranslationAndMat3x3 translation_and_mat3x3 + ) { Transform3D self; self._tag = detail::Transform3DTag::TranslationAndMat3x3; self._data.translation_and_mat3x3 = std::move(translation_and_mat3x3); @@ -70,7 +71,8 @@ namespace rr { } static Transform3D translation_rotation_scale( - rr::datatypes::TranslationRotationScale3D translation_rotation_scale) { + rr::datatypes::TranslationRotationScale3D translation_rotation_scale + ) { Transform3D self; self._tag = detail::Transform3DTag::TranslationRotationScale; self._data.translation_rotation_scale = std::move(translation_rotation_scale); @@ -89,6 +91,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::DenseUnionBuilder* builder, const Transform3D* elements, size_t num_elements + ); + void swap(Transform3D& other) noexcept { auto tag_temp = this->_tag; this->_tag = other._tag; diff --git a/rerun_cpp/src/datatypes/translation_and_mat3x3.cpp b/rerun_cpp/src/datatypes/translation_and_mat3x3.cpp index c94faa6e9bad..31b8eee2c8aa 100644 --- a/rerun_cpp/src/datatypes/translation_and_mat3x3.cpp +++ b/rerun_cpp/src/datatypes/translation_and_mat3x3.cpp @@ -13,10 +13,60 @@ namespace rr { std::shared_ptr TranslationAndMat3x3::to_arrow_datatype() { return arrow::struct_({ arrow::field( - "translation", rr::datatypes::Vec3D::to_arrow_datatype(), true, nullptr), + "translation", + rr::datatypes::Vec3D::to_arrow_datatype(), + true, + nullptr + ), arrow::field("matrix", rr::datatypes::Mat3x3::to_arrow_datatype(), true, nullptr), arrow::field("from_parent", arrow::boolean(), false, nullptr), }); } + + arrow::Result> + TranslationAndMat3x3::new_arrow_array_builder(arrow::MemoryPool *memory_pool) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + to_arrow_datatype(), + memory_pool, + std::vector>({ + rr::datatypes::Vec3D::new_arrow_array_builder(memory_pool).ValueOrDie(), + rr::datatypes::Mat3x3::new_arrow_array_builder(memory_pool).ValueOrDie(), + std::make_shared(memory_pool), + }) + )); + } + + arrow::Status TranslationAndMat3x3::fill_arrow_array_builder( + arrow::StructBuilder *builder, const TranslationAndMat3x3 *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + return arrow::Status::NotImplemented( + "TODO(andreas): extensions in structs are not yet supported" + ); + return arrow::Status::NotImplemented( + "TODO(andreas): extensions in structs are not yet supported" + ); + { + auto element_builder = + static_cast(builder->field_builder(2)); + ARROW_RETURN_NOT_OK(element_builder->Reserve(num_elements)); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + ARROW_RETURN_NOT_OK(element_builder->Append(elements[elem_idx].from_parent)); + } + } + ARROW_RETURN_NOT_OK(builder->AppendValues(num_elements, nullptr)); + + return arrow::Status::OK(); + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/translation_and_mat3x3.hpp b/rerun_cpp/src/datatypes/translation_and_mat3x3.hpp index b56858e24443..0abf367df8d9 100644 --- a/rerun_cpp/src/datatypes/translation_and_mat3x3.hpp +++ b/rerun_cpp/src/datatypes/translation_and_mat3x3.hpp @@ -6,14 +6,10 @@ #include "../datatypes/mat3x3.hpp" #include "../datatypes/vec3d.hpp" +#include #include -#include #include -namespace arrow { - class DataType; -} - namespace rr { namespace datatypes { /// Representation of an affine transform via a 3x3 affine matrix paired with a translation. @@ -33,6 +29,17 @@ namespace rr { public: /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const TranslationAndMat3x3* elements, + size_t num_elements + ); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/translation_rotation_scale3d.cpp b/rerun_cpp/src/datatypes/translation_rotation_scale3d.cpp index 9b09dd315da9..3ba54c30e625 100644 --- a/rerun_cpp/src/datatypes/translation_rotation_scale3d.cpp +++ b/rerun_cpp/src/datatypes/translation_rotation_scale3d.cpp @@ -14,12 +14,71 @@ namespace rr { std::shared_ptr TranslationRotationScale3D::to_arrow_datatype() { return arrow::struct_({ arrow::field( - "translation", rr::datatypes::Vec3D::to_arrow_datatype(), true, nullptr), + "translation", + rr::datatypes::Vec3D::to_arrow_datatype(), + true, + nullptr + ), arrow::field( - "rotation", rr::datatypes::Rotation3D::to_arrow_datatype(), true, nullptr), + "rotation", + rr::datatypes::Rotation3D::to_arrow_datatype(), + true, + nullptr + ), arrow::field("scale", rr::datatypes::Scale3D::to_arrow_datatype(), true, nullptr), arrow::field("from_parent", arrow::boolean(), false, nullptr), }); } + + arrow::Result> + TranslationRotationScale3D::new_arrow_array_builder(arrow::MemoryPool *memory_pool) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + to_arrow_datatype(), + memory_pool, + std::vector>({ + rr::datatypes::Vec3D::new_arrow_array_builder(memory_pool).ValueOrDie(), + rr::datatypes::Rotation3D::new_arrow_array_builder(memory_pool).ValueOrDie(), + rr::datatypes::Scale3D::new_arrow_array_builder(memory_pool).ValueOrDie(), + std::make_shared(memory_pool), + }) + )); + } + + arrow::Status TranslationRotationScale3D::fill_arrow_array_builder( + arrow::StructBuilder *builder, const TranslationRotationScale3D *elements, + size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + return arrow::Status::NotImplemented( + "TODO(andreas): extensions in structs are not yet supported" + ); + return arrow::Status::NotImplemented( + "TODO(andreas): extensions in structs are not yet supported" + ); + return arrow::Status::NotImplemented( + "TODO(andreas): extensions in structs are not yet supported" + ); + { + auto element_builder = + static_cast(builder->field_builder(3)); + ARROW_RETURN_NOT_OK(element_builder->Reserve(num_elements)); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + ARROW_RETURN_NOT_OK(element_builder->Append(elements[elem_idx].from_parent)); + } + } + ARROW_RETURN_NOT_OK(builder->AppendValues(num_elements, nullptr)); + + return arrow::Status::OK(); + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/translation_rotation_scale3d.hpp b/rerun_cpp/src/datatypes/translation_rotation_scale3d.hpp index dfca120794c2..8a8e11cadca9 100644 --- a/rerun_cpp/src/datatypes/translation_rotation_scale3d.hpp +++ b/rerun_cpp/src/datatypes/translation_rotation_scale3d.hpp @@ -7,14 +7,10 @@ #include "../datatypes/scale3d.hpp" #include "../datatypes/vec3d.hpp" +#include #include -#include #include -namespace arrow { - class DataType; -} - namespace rr { namespace datatypes { /// Representation of an affine transform via separate translation, rotation & scale. @@ -35,6 +31,17 @@ namespace rr { public: /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const TranslationRotationScale3D* elements, + size_t num_elements + ); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/vec2d.cpp b/rerun_cpp/src/datatypes/vec2d.cpp index d07bbc291717..7af078821fd4 100644 --- a/rerun_cpp/src/datatypes/vec2d.cpp +++ b/rerun_cpp/src/datatypes/vec2d.cpp @@ -8,8 +8,45 @@ namespace rr { namespace datatypes { std::shared_ptr Vec2D::to_arrow_datatype() { - return arrow::fixed_size_list(arrow::field("item", arrow::float32(), false, nullptr), - 2); + return arrow::fixed_size_list( + arrow::field("item", arrow::float32(), false, nullptr), + 2 + ); + } + + arrow::Result> Vec2D::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + memory_pool, + std::make_shared(memory_pool), + 2 + )); + } + + arrow::Status Vec2D::fill_arrow_array_builder( + arrow::FixedSizeListBuilder *builder, const Vec2D *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto value_builder = static_cast(builder->value_builder()); + + static_assert(sizeof(elements[0].xy) == sizeof(elements[0])); + ARROW_RETURN_NOT_OK( + value_builder->AppendValues(elements[0].xy, num_elements * 2, nullptr) + ); + ARROW_RETURN_NOT_OK(builder->AppendValues(num_elements)); + + return arrow::Status::OK(); } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/vec2d.hpp b/rerun_cpp/src/datatypes/vec2d.hpp index 50bea24f4e0a..6f3438d30242 100644 --- a/rerun_cpp/src/datatypes/vec2d.hpp +++ b/rerun_cpp/src/datatypes/vec2d.hpp @@ -3,12 +3,8 @@ #pragma once +#include #include -#include - -namespace arrow { - class DataType; -} namespace rr { namespace datatypes { @@ -19,6 +15,15 @@ namespace rr { public: /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> + new_arrow_array_builder(arrow::MemoryPool* memory_pool); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::FixedSizeListBuilder* builder, const Vec2D* elements, size_t num_elements + ); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/vec3d.cpp b/rerun_cpp/src/datatypes/vec3d.cpp index e7baacc0f235..be277e718291 100644 --- a/rerun_cpp/src/datatypes/vec3d.cpp +++ b/rerun_cpp/src/datatypes/vec3d.cpp @@ -8,8 +8,45 @@ namespace rr { namespace datatypes { std::shared_ptr Vec3D::to_arrow_datatype() { - return arrow::fixed_size_list(arrow::field("item", arrow::float32(), false, nullptr), - 3); + return arrow::fixed_size_list( + arrow::field("item", arrow::float32(), false, nullptr), + 3 + ); + } + + arrow::Result> Vec3D::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + memory_pool, + std::make_shared(memory_pool), + 3 + )); + } + + arrow::Status Vec3D::fill_arrow_array_builder( + arrow::FixedSizeListBuilder *builder, const Vec3D *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto value_builder = static_cast(builder->value_builder()); + + static_assert(sizeof(elements[0].xyz) == sizeof(elements[0])); + ARROW_RETURN_NOT_OK( + value_builder->AppendValues(elements[0].xyz, num_elements * 3, nullptr) + ); + ARROW_RETURN_NOT_OK(builder->AppendValues(num_elements)); + + return arrow::Status::OK(); } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/vec3d.hpp b/rerun_cpp/src/datatypes/vec3d.hpp index 72fad69ea7a2..92bd757db95a 100644 --- a/rerun_cpp/src/datatypes/vec3d.hpp +++ b/rerun_cpp/src/datatypes/vec3d.hpp @@ -3,12 +3,8 @@ #pragma once +#include #include -#include - -namespace arrow { - class DataType; -} namespace rr { namespace datatypes { @@ -19,6 +15,15 @@ namespace rr { public: /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> + new_arrow_array_builder(arrow::MemoryPool* memory_pool); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::FixedSizeListBuilder* builder, const Vec3D* elements, size_t num_elements + ); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/vec4d.cpp b/rerun_cpp/src/datatypes/vec4d.cpp index 208e9089c47c..dff32d481cf0 100644 --- a/rerun_cpp/src/datatypes/vec4d.cpp +++ b/rerun_cpp/src/datatypes/vec4d.cpp @@ -8,8 +8,45 @@ namespace rr { namespace datatypes { std::shared_ptr Vec4D::to_arrow_datatype() { - return arrow::fixed_size_list(arrow::field("item", arrow::float32(), false, nullptr), - 4); + return arrow::fixed_size_list( + arrow::field("item", arrow::float32(), false, nullptr), + 4 + ); + } + + arrow::Result> Vec4D::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + memory_pool, + std::make_shared(memory_pool), + 4 + )); + } + + arrow::Status Vec4D::fill_arrow_array_builder( + arrow::FixedSizeListBuilder *builder, const Vec4D *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto value_builder = static_cast(builder->value_builder()); + + static_assert(sizeof(elements[0].xyzw) == sizeof(elements[0])); + ARROW_RETURN_NOT_OK( + value_builder->AppendValues(elements[0].xyzw, num_elements * 4, nullptr) + ); + ARROW_RETURN_NOT_OK(builder->AppendValues(num_elements)); + + return arrow::Status::OK(); } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/vec4d.hpp b/rerun_cpp/src/datatypes/vec4d.hpp index f9745ba3aaa2..7ab3e76a2283 100644 --- a/rerun_cpp/src/datatypes/vec4d.hpp +++ b/rerun_cpp/src/datatypes/vec4d.hpp @@ -3,12 +3,8 @@ #pragma once +#include #include -#include - -namespace arrow { - class DataType; -} namespace rr { namespace datatypes { @@ -19,6 +15,15 @@ namespace rr { public: /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Creates a new array builder with an array of this type. + static arrow::Result> + new_arrow_array_builder(arrow::MemoryPool* memory_pool); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::FixedSizeListBuilder* builder, const Vec4D* elements, size_t num_elements + ); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/recording_stream.cpp b/rerun_cpp/src/recording_stream.cpp index b5a2ebfcccc5..74b40c0c41d4 100644 --- a/rerun_cpp/src/recording_stream.cpp +++ b/rerun_cpp/src/recording_stream.cpp @@ -40,8 +40,10 @@ namespace rr { return s_global; } - void RecordingStream::log_data_row(const char* entity_path, uint32_t num_instances, - size_t num_data_cells, const DataCell* data_cells) { + void RecordingStream::log_data_row( + const char* entity_path, uint32_t num_instances, size_t num_data_cells, + const DataCell* data_cells + ) { // Map to C API: std::vector c_data_cells; c_data_cells.reserve(num_data_cells); diff --git a/rerun_cpp/src/recording_stream.hpp b/rerun_cpp/src/recording_stream.hpp index 991c04deb96b..30262b4f2d59 100644 --- a/rerun_cpp/src/recording_stream.hpp +++ b/rerun_cpp/src/recording_stream.hpp @@ -17,8 +17,9 @@ namespace rr { class RecordingStream { public: - RecordingStream(const char* app_id, const char* addr, - StoreKind store_kind = StoreKind::Recording); + RecordingStream( + const char* app_id, const char* addr, StoreKind store_kind = StoreKind::Recording + ); ~RecordingStream(); /// Must be called first, if at all. @@ -32,8 +33,10 @@ namespace rr { /// /// I.e. logs a number of components arrays (each with a same number of instances) to a /// single entity path. - void log_data_row(const char* entity_path, uint32_t num_instances, size_t num_data_cells, - const DataCell* data_cells); + void log_data_row( + const char* entity_path, uint32_t num_instances, size_t num_data_cells, + const DataCell* data_cells + ); private: RecordingStream() : _id{0} {} diff --git a/rerun_cpp/src/rerun.cpp b/rerun_cpp/src/rerun.cpp index bf02a7bd9108..0136925b4799 100644 --- a/rerun_cpp/src/rerun.cpp +++ b/rerun_cpp/src/rerun.cpp @@ -17,30 +17,22 @@ namespace rr { arrow::Result> points3(size_t num_points, const float* xyz) { arrow::MemoryPool* pool = arrow::default_memory_pool(); - auto x_builder = std::make_shared(pool); - auto y_builder = std::make_shared(pool); - auto z_builder = std::make_shared(pool); - auto nullable = false; - auto data_type = arrow::struct_({field("x", arrow::float32(), nullable), - field("y", arrow::float32(), nullable), - field("z", arrow::float32(), nullable)}); - auto struct_builder = - arrow::StructBuilder(data_type, pool, {x_builder, y_builder, z_builder}); - - for (size_t i = 0; i < num_points; ++i) { - ARROW_RETURN_NOT_OK(struct_builder.Append()); - ARROW_RETURN_NOT_OK(x_builder->Append(xyz[3 * i + 0])); - ARROW_RETURN_NOT_OK(y_builder->Append(xyz[3 * i + 1])); - ARROW_RETURN_NOT_OK(z_builder->Append(xyz[3 * i + 2])); - } + ARROW_ASSIGN_OR_RAISE(auto builder, rr::components::Point3D::new_arrow_array_builder(pool)); + ARROW_RETURN_NOT_OK(rr::components::Point3D::fill_arrow_array_builder( + builder.get(), + (rr::components::Point3D*)xyz, + num_points + )); std::shared_ptr array; - ARROW_RETURN_NOT_OK(struct_builder.Finish(&array)); + ARROW_RETURN_NOT_OK(builder->Finish(&array)); auto name = "points"; // Unused, but should be the name of the field in the archetype - auto schema = arrow::schema({arrow::field(name, data_type, nullable)}); + auto schema = arrow::schema( + {arrow::field(name, rr::components::Point3D::to_arrow_datatype(), nullable)} + ); return arrow::Table::Make(schema, {array}); }