From d0fb223537f8a86335ff16d960a16c8b3501e764 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Thu, 20 Jul 2023 18:22:59 +0200 Subject: [PATCH 01/20] newlines after method declarations --- crates/re_types_builder/src/codegen/cpp/method.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/re_types_builder/src/codegen/cpp/method.rs b/crates/re_types_builder/src/codegen/cpp/method.rs index b036452a85b7..917bd24aa897 100644 --- a/crates/re_types_builder/src/codegen/cpp/method.rs +++ b/crates/re_types_builder/src/codegen/cpp/method.rs @@ -125,12 +125,14 @@ impl Method { #declaration { #definition_body } + #NEWLINE_TOKEN } } else { quote! { #NEWLINE_TOKEN #docs #declaration; + #NEWLINE_TOKEN } } } From be7672798c1a1400bc37aae81c41458e6558645d Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Fri, 21 Jul 2023 13:38:15 +0200 Subject: [PATCH 02/20] wip to_arrow --- .../re_types_builder/src/codegen/cpp/mod.rs | 162 +++++++++++++++++- examples/cpp/minimal/main.cpp | 70 ++++++-- rerun_cpp/src/components/affix_fuzzer1.cpp | 18 ++ rerun_cpp/src/components/affix_fuzzer1.hpp | 9 +- rerun_cpp/src/components/affix_fuzzer10.cpp | 13 ++ rerun_cpp/src/components/affix_fuzzer10.hpp | 9 +- rerun_cpp/src/components/affix_fuzzer11.cpp | 13 ++ rerun_cpp/src/components/affix_fuzzer11.hpp | 9 +- rerun_cpp/src/components/affix_fuzzer12.cpp | 13 ++ rerun_cpp/src/components/affix_fuzzer12.hpp | 9 +- rerun_cpp/src/components/affix_fuzzer13.cpp | 13 ++ rerun_cpp/src/components/affix_fuzzer13.hpp | 9 +- rerun_cpp/src/components/affix_fuzzer14.cpp | 18 ++ rerun_cpp/src/components/affix_fuzzer14.hpp | 9 +- rerun_cpp/src/components/affix_fuzzer15.cpp | 18 ++ rerun_cpp/src/components/affix_fuzzer15.hpp | 9 +- rerun_cpp/src/components/affix_fuzzer16.cpp | 13 ++ rerun_cpp/src/components/affix_fuzzer16.hpp | 9 +- rerun_cpp/src/components/affix_fuzzer17.cpp | 13 ++ rerun_cpp/src/components/affix_fuzzer17.hpp | 9 +- rerun_cpp/src/components/affix_fuzzer18.cpp | 13 ++ rerun_cpp/src/components/affix_fuzzer18.hpp | 9 +- rerun_cpp/src/components/affix_fuzzer19.cpp | 18 ++ rerun_cpp/src/components/affix_fuzzer19.hpp | 9 +- rerun_cpp/src/components/affix_fuzzer2.cpp | 18 ++ rerun_cpp/src/components/affix_fuzzer2.hpp | 9 +- rerun_cpp/src/components/affix_fuzzer3.cpp | 18 ++ rerun_cpp/src/components/affix_fuzzer3.hpp | 9 +- rerun_cpp/src/components/affix_fuzzer4.cpp | 18 ++ rerun_cpp/src/components/affix_fuzzer4.hpp | 9 +- rerun_cpp/src/components/affix_fuzzer5.cpp | 18 ++ rerun_cpp/src/components/affix_fuzzer5.hpp | 9 +- rerun_cpp/src/components/affix_fuzzer6.cpp | 18 ++ rerun_cpp/src/components/affix_fuzzer6.hpp | 9 +- rerun_cpp/src/components/affix_fuzzer7.cpp | 13 ++ rerun_cpp/src/components/affix_fuzzer7.hpp | 9 +- rerun_cpp/src/components/affix_fuzzer8.cpp | 13 ++ rerun_cpp/src/components/affix_fuzzer8.hpp | 9 +- rerun_cpp/src/components/affix_fuzzer9.cpp | 13 ++ rerun_cpp/src/components/affix_fuzzer9.hpp | 9 +- rerun_cpp/src/components/class_id.cpp | 13 ++ rerun_cpp/src/components/class_id.hpp | 9 +- rerun_cpp/src/components/color.cpp | 13 ++ rerun_cpp/src/components/color.hpp | 9 +- rerun_cpp/src/components/draw_order.cpp | 13 ++ rerun_cpp/src/components/draw_order.hpp | 9 +- rerun_cpp/src/components/instance_key.cpp | 13 ++ rerun_cpp/src/components/instance_key.hpp | 9 +- rerun_cpp/src/components/keypoint_id.cpp | 13 ++ rerun_cpp/src/components/keypoint_id.hpp | 9 +- rerun_cpp/src/components/label.cpp | 13 ++ rerun_cpp/src/components/label.hpp | 9 +- rerun_cpp/src/components/point2d.cpp | 18 ++ rerun_cpp/src/components/point2d.hpp | 9 +- rerun_cpp/src/components/point3d.cpp | 18 ++ rerun_cpp/src/components/point3d.hpp | 9 +- rerun_cpp/src/components/radius.cpp | 13 ++ rerun_cpp/src/components/radius.hpp | 9 +- rerun_cpp/src/components/transform3d.cpp | 18 ++ rerun_cpp/src/components/transform3d.hpp | 9 +- rerun_cpp/src/datatypes/affix_fuzzer1.cpp | 17 ++ rerun_cpp/src/datatypes/affix_fuzzer1.hpp | 9 +- rerun_cpp/src/datatypes/affix_fuzzer2.cpp | 13 ++ rerun_cpp/src/datatypes/affix_fuzzer2.hpp | 9 +- rerun_cpp/src/datatypes/affix_fuzzer5.cpp | 17 ++ rerun_cpp/src/datatypes/affix_fuzzer5.hpp | 9 +- rerun_cpp/src/datatypes/flattened_scalar.cpp | 17 ++ rerun_cpp/src/datatypes/flattened_scalar.hpp | 10 +- rerun_cpp/src/datatypes/mat3x3.cpp | 13 ++ rerun_cpp/src/datatypes/mat3x3.hpp | 9 +- rerun_cpp/src/datatypes/mat4x4.cpp | 13 ++ rerun_cpp/src/datatypes/mat4x4.hpp | 9 +- rerun_cpp/src/datatypes/point2d.cpp | 17 ++ rerun_cpp/src/datatypes/point2d.hpp | 9 +- rerun_cpp/src/datatypes/point3d.cpp | 17 ++ rerun_cpp/src/datatypes/point3d.hpp | 9 +- rerun_cpp/src/datatypes/quaternion.cpp | 13 ++ rerun_cpp/src/datatypes/quaternion.hpp | 9 +- .../src/datatypes/rotation_axis_angle.cpp | 18 ++ .../src/datatypes/rotation_axis_angle.hpp | 10 +- .../src/datatypes/translation_and_mat3x3.cpp | 18 ++ .../src/datatypes/translation_and_mat3x3.hpp | 10 +- .../translation_rotation_scale3d.cpp | 18 ++ .../translation_rotation_scale3d.hpp | 10 +- rerun_cpp/src/datatypes/vec2d.cpp | 13 ++ rerun_cpp/src/datatypes/vec2d.hpp | 9 +- rerun_cpp/src/datatypes/vec3d.cpp | 13 ++ rerun_cpp/src/datatypes/vec3d.hpp | 9 +- rerun_cpp/src/datatypes/vec4d.cpp | 13 ++ rerun_cpp/src/datatypes/vec4d.hpp | 9 +- 90 files changed, 1240 insertions(+), 59 deletions(-) diff --git a/crates/re_types_builder/src/codegen/cpp/mod.rs b/crates/re_types_builder/src/codegen/cpp/mod.rs index 0f7a35387243..6c19eaf29b49 100644 --- a/crates/re_types_builder/src/codegen/cpp/mod.rs +++ b/crates/re_types_builder/src/codegen/cpp/mod.rs @@ -259,16 +259,28 @@ impl QuotedObject { } }; + let datatype = arrow_registry.get(&obj.fqname); + match obj.kind { ObjectKind::Datatype | ObjectKind::Component => { methods.push(arrow_data_type_method( - &arrow_registry.get(&obj.fqname), + &datatype, &mut hpp_includes, &mut cpp_includes, &mut hpp_declarations, )); + + methods.push(to_arrow_method( + &datatype, + &pascal_case_ident, + &mut hpp_includes, + &mut cpp_includes, + &mut hpp_declarations, + )); + } + ObjectKind::Archetype => { + // TODO(andreas): Should also be convertable to arrow? } - ObjectKind::Archetype => {} }; let hpp_method_section = if methods.is_empty() { @@ -578,8 +590,8 @@ fn arrow_data_type_method( hpp_declarations: &mut ForwardDecls, ) -> Method { hpp_declarations.insert("arrow", ForwardDecl::Class("DataType".to_owned())); - cpp_includes.system.insert("arrow/api.h".to_owned()); hpp_includes.system.insert("memory".to_owned()); // std::shared_ptr + cpp_includes.system.insert("arrow/api.h".to_owned()); let quoted_datatype = quote_arrow_data_type(datatype, cpp_includes, true); @@ -595,6 +607,145 @@ fn arrow_data_type_method( } } +fn to_arrow_method( + datatype: &DataType, + pascal_case_ident: &Ident, + hpp_includes: &mut Includes, + cpp_includes: &mut Includes, + hpp_declarations: &mut ForwardDecls, +) -> Method { + hpp_declarations.insert("arrow", ForwardDecl::Class("ArrayBuilder".to_owned())); + hpp_declarations.insert("arrow", ForwardDecl::Class("MemoryPool".to_owned())); + hpp_includes.system.insert("arrow/result.h".to_owned()); + hpp_includes.system.insert("memory".to_owned()); // std::shared_ptr + cpp_includes.system.insert("arrow/api.h".to_owned()); + + let DataType::Extension(_fqname, logical_datatype, _metadata) = datatype else { + panic!("Can only generate arrow serialization code for extension types."); + }; + + let pool = format_ident!("memory_pool"); + + /// TODO: shorter code here for trivial ones? + let builder_initialization = match logical_datatype.as_ref() { + DataType::Boolean => { + quote!(auto builder = std::make_shared(#pool);) + } + DataType::Int8 => quote!(auto builder = std::make_shared(#pool);), + DataType::Int16 => { + quote!(auto builder = std::make_shared(#pool);) + } + DataType::Int32 => { + quote!(auto builder = std::make_shared(#pool);) + } + DataType::Int64 => { + quote!(auto builder = std::make_shared(#pool);) + } + DataType::UInt8 => { + quote!(auto builder = std::make_shared(#pool);) + } + DataType::UInt16 => { + quote!(auto builder = std::make_shared(#pool);) + } + DataType::UInt32 => { + quote!(auto builder = std::make_shared(#pool);) + } + DataType::UInt64 => { + quote!(auto builder = std::make_shared(#pool);) + } + DataType::Float16 => { + quote!(auto builder = std::make_shared(#pool);) + } + DataType::Float32 => { + quote!(auto builder = std::make_shared(#pool);) + } + DataType::Float64 => { + quote!(auto builder = std::make_shared(#pool);) + } + DataType::Binary => { + quote!(auto builder = std::make_shared(#pool);) + } + DataType::LargeBinary => { + quote!(auto builder = std::make_shared(#pool);) + } + DataType::Utf8 => { + quote!(auto builder = std::make_shared(#pool);) + } + DataType::LargeUtf8 => { + quote!(auto builder = std::make_shared(#pool);) + } + DataType::List(_) => { + quote!(auto builder = std::make_shared(#pool);) + } + DataType::FixedSizeList(_, _) => { + quote!(auto builder = std::make_shared(#pool);) + } + DataType::Struct(_) => { + quote! { + auto datatype = #pascal_case_ident::to_arrow_datatype(); + let builder = std::make_shared( + datatype, + #pool, + {}, #TODO_TOKEN + ); + } + } + DataType::Union(_, _, mode) => { + let builder_type = match mode { + arrow2::datatypes::UnionMode::Dense => "DenseUnionBuilder", + arrow2::datatypes::UnionMode::Sparse => "SparseUnionBuilder", + }; + let builder_type = format_ident!("{builder_type}"); + + quote! { + auto datatype = #pascal_case_ident::to_arrow_datatype(); + let builder = std::make_shared( + datatype, + #pool, + {}, #TODO_TOKEN + ); + } + } + DataType::Extension(fqname, datatype, _metadata) => { + //if let datatype // TODO: unions not yet supported + + // Extension in extension only happens if this is a transparent type pointing to another extension type. + // TODO: remove unnecessary namespacing. + let quoted_fqname = quote_fqname_as_type_path(cpp_includes, fqname); + quote! { + static_assert(sizeof(#pascal_case_ident) == sizeof(#quoted_fqname), "Expected fully transparent type."); + auto builder = #quoted_fqname::to_arrow(#pool, reinterpret_cast(elements), num_elements); + } + } + _ => unimplemented!("Arrow serialization for type: {:?}", logical_datatype), + }; + + Method { + docs: "Fills out an arrow array builder with an array of this type.".into(), + declaration: MethodDeclaration { + is_static: true, + return_type: quote! { arrow::Result> }, + // TODO(andreas): Pass in validity map. + name_and_parameters: quote! { + to_arrow(arrow::MemoryPool* #pool, const #pascal_case_ident* elements, size_t num_elements) + }, + }, + definition_body: quote! { + if (!#pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + #NEWLINE_TOKEN + #NEWLINE_TOKEN + #builder_initialization + return builder; + }, + inline: false, + } +} + /// e.g. `static Angle radians(float radians);` -> `auto angle = Angle::radians(radians);` fn static_constructor_for_enum_type( objects: &Objects, @@ -902,8 +1053,9 @@ fn quote_arrow_data_type( // In the future we'll add the extension type here to the schema. quote_arrow_data_type(datatype, includes, false) } else { - let fqname_use = quote_fqname_as_type_path(includes, fqname); - quote! { #fqname_use::to_arrow_datatype() } + // TODO: remove unnecessary namespacing. + let quoted_fqname = quote_fqname_as_type_path(includes, fqname); + quote! { #quoted_fqname::to_arrow_datatype() } } } diff --git a/examples/cpp/minimal/main.cpp b/examples/cpp/minimal/main.cpp index dd8e246b152a..14c3d57fb145 100644 --- a/examples/cpp/minimal/main.cpp +++ b/examples/cpp/minimal/main.cpp @@ -1,6 +1,37 @@ #include #include +#include + +arrow::Result> points2(size_t num_points, const float* xy) { + arrow::MemoryPool* pool = arrow::default_memory_pool(); + + auto x_builder = std::make_shared(pool); + auto y_builder = std::make_shared(pool); + + auto nullable = false; + + auto data_type = rr::components::Point2D::to_arrow_datatype(); + auto struct_builder = + arrow::StructBuilder(data_type, pool, {x_builder, y_builder}); + + for (size_t i = 0; i < num_points; ++i) { + ARROW_RETURN_NOT_OK(struct_builder.Append()); + ARROW_RETURN_NOT_OK(x_builder->Append(xy[2 * i + 0])); + ARROW_RETURN_NOT_OK(y_builder->Append(xy[2 * i + 1])); +} + + std::shared_ptr array; + ARROW_RETURN_NOT_OK(struct_builder.Finish(&array)); + + auto name = "points"; // Unused, but should be the name of the field in the archetype + auto schema = arrow::schema({arrow::field(name, data_type, nullable)}); + + return arrow::Table::Make(schema, {array}); +} + + + int main(int argc, char** argv) { loguru::g_preamble_uptime = false; loguru::g_preamble_thread = false; @@ -10,18 +41,37 @@ int main(int argc, char** argv) { auto rr_stream = rr::RecordingStream{"c-example-app", "127.0.0.1:9876"}; - float xyz[9] = {0.0, 0.0, 0.0, 1.0, 2.0, 3.0, 5.0, 5.0, 5.0}; - auto points = rr::points3(3, xyz).ValueOrDie(); - auto buffer = rr::ipc_from_table(*points).ValueOrDie(); + // Points3D, fully custom. + { + float xyz[9] = {0.0, 0.0, 0.0, 1.0, 2.0, 3.0, 5.0, 5.0, 5.0}; + auto points = rr::points3(3, xyz).ValueOrDie(); + auto buffer = rr::ipc_from_table(*points).ValueOrDie(); + + const rr::DataCell data_cells[1] = {rr::DataCell{ + .component_name = "rerun.point3d", + .num_bytes = static_cast(buffer->size()), + .bytes = buffer->data(), + }}; + + uint32_t num_instances = 3; + rr_stream.log_data_row("3d/points", num_instances, 1, data_cells); + } + + // Points2D, via generated code. + { + float xy[6] = {0.0, 0.0, 1.0, 3.0, 5.0, 5.0}; + auto points = points2(3, xy).ValueOrDie(); + auto buffer = rr::ipc_from_table(*points).ValueOrDie(); - const rr::DataCell data_cells[1] = {rr::DataCell{ - .component_name = "rerun.point3d", - .num_bytes = static_cast(buffer->size()), - .bytes = buffer->data(), - }}; + const rr::DataCell data_cells[1] = {rr::DataCell{ + .component_name = "rerun.point2d", + .num_bytes = static_cast(buffer->size()), + .bytes = buffer->data(), + }}; - uint32_t num_instances = 3; - rr_stream.log_data_row("points", num_instances, 1, data_cells); + uint32_t num_instances = 3; + rr_stream.log_data_row("2d/points", num_instances, 1, data_cells); + } // Test some type instantiation auto tls = rr::datatypes::TranslationRotationScale3D{}; diff --git a/rerun_cpp/src/components/affix_fuzzer1.cpp b/rerun_cpp/src/components/affix_fuzzer1.cpp index 14a27aa3c059..476c33a0568b 100644 --- a/rerun_cpp/src/components/affix_fuzzer1.cpp +++ b/rerun_cpp/src/components/affix_fuzzer1.cpp @@ -12,5 +12,23 @@ namespace rr { std::shared_ptr AffixFuzzer1::to_arrow_datatype() { return rr::datatypes::AffixFuzzer1::to_arrow_datatype(); } + + arrow::Result> AffixFuzzer1::to_arrow( + arrow::MemoryPool *memory_pool, const AffixFuzzer1 *elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + static_assert(sizeof(AffixFuzzer1) == sizeof(rr::datatypes::AffixFuzzer1), + "Expected fully transparent type."); + auto builder = rr::datatypes::AffixFuzzer1::to_arrow( + memory_pool, + reinterpret_cast(elements), + num_elements); + return builder; + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer1.hpp b/rerun_cpp/src/components/affix_fuzzer1.hpp index 35e7ce40be07..9edabb00debe 100644 --- a/rerun_cpp/src/components/affix_fuzzer1.hpp +++ b/rerun_cpp/src/components/affix_fuzzer1.hpp @@ -5,13 +5,16 @@ #include "../datatypes/affix_fuzzer1.hpp" +#include #include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace components { @@ -24,6 +27,10 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer1* elements, size_t num_elements); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer10.cpp b/rerun_cpp/src/components/affix_fuzzer10.cpp index 81a99b9b8215..b0daff857fc0 100644 --- a/rerun_cpp/src/components/affix_fuzzer10.cpp +++ b/rerun_cpp/src/components/affix_fuzzer10.cpp @@ -10,5 +10,18 @@ namespace rr { std::shared_ptr AffixFuzzer10::to_arrow_datatype() { return arrow::utf8(); } + + arrow::Result> AffixFuzzer10::to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer10* elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto builder = std::make_shared(memory_pool); + return builder; + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer10.hpp b/rerun_cpp/src/components/affix_fuzzer10.hpp index 5293a168d2b9..591a3cb35d56 100644 --- a/rerun_cpp/src/components/affix_fuzzer10.hpp +++ b/rerun_cpp/src/components/affix_fuzzer10.hpp @@ -3,6 +3,7 @@ #pragma once +#include #include #include #include @@ -10,8 +11,10 @@ #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace components { @@ -24,6 +27,10 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer10* elements, size_t num_elements); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer11.cpp b/rerun_cpp/src/components/affix_fuzzer11.cpp index a71618858bf4..c276e6851d6d 100644 --- a/rerun_cpp/src/components/affix_fuzzer11.cpp +++ b/rerun_cpp/src/components/affix_fuzzer11.cpp @@ -10,5 +10,18 @@ namespace rr { std::shared_ptr AffixFuzzer11::to_arrow_datatype() { return arrow::list(arrow::field("item", arrow::float32(), true, nullptr)); } + + arrow::Result> AffixFuzzer11::to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer11* elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto builder = std::make_shared(memory_pool); + return builder; + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer11.hpp b/rerun_cpp/src/components/affix_fuzzer11.hpp index d0cd2ae0858f..87212f1ac256 100644 --- a/rerun_cpp/src/components/affix_fuzzer11.hpp +++ b/rerun_cpp/src/components/affix_fuzzer11.hpp @@ -3,6 +3,7 @@ #pragma once +#include #include #include #include @@ -10,8 +11,10 @@ #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace components { @@ -24,6 +27,10 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer11* elements, size_t num_elements); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer12.cpp b/rerun_cpp/src/components/affix_fuzzer12.cpp index 253eec0c8844..787a383ec4cd 100644 --- a/rerun_cpp/src/components/affix_fuzzer12.cpp +++ b/rerun_cpp/src/components/affix_fuzzer12.cpp @@ -10,5 +10,18 @@ namespace rr { std::shared_ptr AffixFuzzer12::to_arrow_datatype() { return arrow::list(arrow::field("item", arrow::utf8(), false, nullptr)); } + + arrow::Result> AffixFuzzer12::to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer12* elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto builder = std::make_shared(memory_pool); + return builder; + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer12.hpp b/rerun_cpp/src/components/affix_fuzzer12.hpp index d553f5109506..33f378cc1545 100644 --- a/rerun_cpp/src/components/affix_fuzzer12.hpp +++ b/rerun_cpp/src/components/affix_fuzzer12.hpp @@ -3,6 +3,7 @@ #pragma once +#include #include #include #include @@ -10,8 +11,10 @@ #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace components { @@ -24,6 +27,10 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer12* elements, size_t num_elements); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer13.cpp b/rerun_cpp/src/components/affix_fuzzer13.cpp index 4fa2852ef5a3..83f30a3071d8 100644 --- a/rerun_cpp/src/components/affix_fuzzer13.cpp +++ b/rerun_cpp/src/components/affix_fuzzer13.cpp @@ -10,5 +10,18 @@ namespace rr { std::shared_ptr AffixFuzzer13::to_arrow_datatype() { return arrow::list(arrow::field("item", arrow::utf8(), true, nullptr)); } + + arrow::Result> AffixFuzzer13::to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer13* elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto builder = std::make_shared(memory_pool); + return builder; + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer13.hpp b/rerun_cpp/src/components/affix_fuzzer13.hpp index f6afa2733905..b034598b4a07 100644 --- a/rerun_cpp/src/components/affix_fuzzer13.hpp +++ b/rerun_cpp/src/components/affix_fuzzer13.hpp @@ -3,6 +3,7 @@ #pragma once +#include #include #include #include @@ -11,8 +12,10 @@ #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace components { @@ -25,6 +28,10 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer13* elements, size_t num_elements); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer14.cpp b/rerun_cpp/src/components/affix_fuzzer14.cpp index e37954f9bc6a..382990eb5a78 100644 --- a/rerun_cpp/src/components/affix_fuzzer14.cpp +++ b/rerun_cpp/src/components/affix_fuzzer14.cpp @@ -12,5 +12,23 @@ namespace rr { std::shared_ptr AffixFuzzer14::to_arrow_datatype() { return rr::datatypes::AffixFuzzer3::to_arrow_datatype(); } + + arrow::Result> AffixFuzzer14::to_arrow( + arrow::MemoryPool *memory_pool, const AffixFuzzer14 *elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + static_assert(sizeof(AffixFuzzer14) == sizeof(rr::datatypes::AffixFuzzer3), + "Expected fully transparent type."); + auto builder = rr::datatypes::AffixFuzzer3::to_arrow( + memory_pool, + reinterpret_cast(elements), + num_elements); + return builder; + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer14.hpp b/rerun_cpp/src/components/affix_fuzzer14.hpp index a74dca7d43d9..d9ccc447bc5e 100644 --- a/rerun_cpp/src/components/affix_fuzzer14.hpp +++ b/rerun_cpp/src/components/affix_fuzzer14.hpp @@ -5,13 +5,16 @@ #include "../datatypes/affix_fuzzer3.hpp" +#include #include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace components { @@ -24,6 +27,10 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer14* elements, size_t num_elements); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer15.cpp b/rerun_cpp/src/components/affix_fuzzer15.cpp index 45d1a31b6e7a..96836f0c4600 100644 --- a/rerun_cpp/src/components/affix_fuzzer15.cpp +++ b/rerun_cpp/src/components/affix_fuzzer15.cpp @@ -12,5 +12,23 @@ namespace rr { std::shared_ptr AffixFuzzer15::to_arrow_datatype() { return rr::datatypes::AffixFuzzer3::to_arrow_datatype(); } + + arrow::Result> AffixFuzzer15::to_arrow( + arrow::MemoryPool *memory_pool, const AffixFuzzer15 *elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + static_assert(sizeof(AffixFuzzer15) == sizeof(rr::datatypes::AffixFuzzer3), + "Expected fully transparent type."); + auto builder = rr::datatypes::AffixFuzzer3::to_arrow( + memory_pool, + reinterpret_cast(elements), + num_elements); + return builder; + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer15.hpp b/rerun_cpp/src/components/affix_fuzzer15.hpp index 6c95217a2f20..4db8444c2bb1 100644 --- a/rerun_cpp/src/components/affix_fuzzer15.hpp +++ b/rerun_cpp/src/components/affix_fuzzer15.hpp @@ -5,14 +5,17 @@ #include "../datatypes/affix_fuzzer3.hpp" +#include #include #include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace components { @@ -25,6 +28,10 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer15* elements, size_t num_elements); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer16.cpp b/rerun_cpp/src/components/affix_fuzzer16.cpp index a257a5981278..5d1d3d7a01ed 100644 --- a/rerun_cpp/src/components/affix_fuzzer16.cpp +++ b/rerun_cpp/src/components/affix_fuzzer16.cpp @@ -13,5 +13,18 @@ namespace rr { return arrow::list(arrow::field( "item", rr::datatypes::AffixFuzzer3::to_arrow_datatype(), false, nullptr)); } + + arrow::Result> AffixFuzzer16::to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer16* elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto builder = std::make_shared(memory_pool); + return builder; + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer16.hpp b/rerun_cpp/src/components/affix_fuzzer16.hpp index dcb477706185..7d039147da77 100644 --- a/rerun_cpp/src/components/affix_fuzzer16.hpp +++ b/rerun_cpp/src/components/affix_fuzzer16.hpp @@ -5,14 +5,17 @@ #include "../datatypes/affix_fuzzer3.hpp" +#include #include #include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace components { @@ -25,6 +28,10 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer16* elements, size_t num_elements); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer17.cpp b/rerun_cpp/src/components/affix_fuzzer17.cpp index 2b9d7cb30041..345e45341c70 100644 --- a/rerun_cpp/src/components/affix_fuzzer17.cpp +++ b/rerun_cpp/src/components/affix_fuzzer17.cpp @@ -13,5 +13,18 @@ namespace rr { return arrow::list(arrow::field( "item", rr::datatypes::AffixFuzzer3::to_arrow_datatype(), true, nullptr)); } + + arrow::Result> AffixFuzzer17::to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer17* elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto builder = std::make_shared(memory_pool); + return builder; + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer17.hpp b/rerun_cpp/src/components/affix_fuzzer17.hpp index 89b27364f836..aab196cc3b43 100644 --- a/rerun_cpp/src/components/affix_fuzzer17.hpp +++ b/rerun_cpp/src/components/affix_fuzzer17.hpp @@ -5,6 +5,7 @@ #include "../datatypes/affix_fuzzer3.hpp" +#include #include #include #include @@ -12,8 +13,10 @@ #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace components { @@ -27,6 +30,10 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer17* elements, size_t num_elements); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer18.cpp b/rerun_cpp/src/components/affix_fuzzer18.cpp index e6c172976e91..c497d378a913 100644 --- a/rerun_cpp/src/components/affix_fuzzer18.cpp +++ b/rerun_cpp/src/components/affix_fuzzer18.cpp @@ -13,5 +13,18 @@ namespace rr { return arrow::list(arrow::field( "item", rr::datatypes::AffixFuzzer4::to_arrow_datatype(), true, nullptr)); } + + arrow::Result> AffixFuzzer18::to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer18* elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto builder = std::make_shared(memory_pool); + return builder; + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer18.hpp b/rerun_cpp/src/components/affix_fuzzer18.hpp index 599f893187d2..b8ee0f2b4788 100644 --- a/rerun_cpp/src/components/affix_fuzzer18.hpp +++ b/rerun_cpp/src/components/affix_fuzzer18.hpp @@ -5,6 +5,7 @@ #include "../datatypes/affix_fuzzer4.hpp" +#include #include #include #include @@ -12,8 +13,10 @@ #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace components { @@ -27,6 +30,10 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer18* elements, size_t num_elements); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer19.cpp b/rerun_cpp/src/components/affix_fuzzer19.cpp index 8d375e459b22..705271184bcd 100644 --- a/rerun_cpp/src/components/affix_fuzzer19.cpp +++ b/rerun_cpp/src/components/affix_fuzzer19.cpp @@ -12,5 +12,23 @@ namespace rr { std::shared_ptr AffixFuzzer19::to_arrow_datatype() { return rr::datatypes::AffixFuzzer5::to_arrow_datatype(); } + + arrow::Result> AffixFuzzer19::to_arrow( + arrow::MemoryPool *memory_pool, const AffixFuzzer19 *elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + static_assert(sizeof(AffixFuzzer19) == sizeof(rr::datatypes::AffixFuzzer5), + "Expected fully transparent type."); + auto builder = rr::datatypes::AffixFuzzer5::to_arrow( + memory_pool, + reinterpret_cast(elements), + num_elements); + return builder; + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer19.hpp b/rerun_cpp/src/components/affix_fuzzer19.hpp index 234201301457..75385a901b47 100644 --- a/rerun_cpp/src/components/affix_fuzzer19.hpp +++ b/rerun_cpp/src/components/affix_fuzzer19.hpp @@ -5,13 +5,16 @@ #include "../datatypes/affix_fuzzer5.hpp" +#include #include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace components { @@ -24,6 +27,10 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer19* elements, size_t num_elements); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer2.cpp b/rerun_cpp/src/components/affix_fuzzer2.cpp index cb49f0249c94..6104164da41d 100644 --- a/rerun_cpp/src/components/affix_fuzzer2.cpp +++ b/rerun_cpp/src/components/affix_fuzzer2.cpp @@ -12,5 +12,23 @@ namespace rr { std::shared_ptr AffixFuzzer2::to_arrow_datatype() { return rr::datatypes::AffixFuzzer1::to_arrow_datatype(); } + + arrow::Result> AffixFuzzer2::to_arrow( + arrow::MemoryPool *memory_pool, const AffixFuzzer2 *elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + static_assert(sizeof(AffixFuzzer2) == sizeof(rr::datatypes::AffixFuzzer1), + "Expected fully transparent type."); + auto builder = rr::datatypes::AffixFuzzer1::to_arrow( + memory_pool, + reinterpret_cast(elements), + num_elements); + return builder; + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer2.hpp b/rerun_cpp/src/components/affix_fuzzer2.hpp index e1e2d868cf82..e7245d956a0d 100644 --- a/rerun_cpp/src/components/affix_fuzzer2.hpp +++ b/rerun_cpp/src/components/affix_fuzzer2.hpp @@ -5,13 +5,16 @@ #include "../datatypes/affix_fuzzer1.hpp" +#include #include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace components { @@ -24,6 +27,10 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer2* elements, size_t num_elements); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer3.cpp b/rerun_cpp/src/components/affix_fuzzer3.cpp index 727da20b00c4..1b017fc6d1bc 100644 --- a/rerun_cpp/src/components/affix_fuzzer3.cpp +++ b/rerun_cpp/src/components/affix_fuzzer3.cpp @@ -12,5 +12,23 @@ namespace rr { std::shared_ptr AffixFuzzer3::to_arrow_datatype() { return rr::datatypes::AffixFuzzer1::to_arrow_datatype(); } + + arrow::Result> AffixFuzzer3::to_arrow( + arrow::MemoryPool *memory_pool, const AffixFuzzer3 *elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + static_assert(sizeof(AffixFuzzer3) == sizeof(rr::datatypes::AffixFuzzer1), + "Expected fully transparent type."); + auto builder = rr::datatypes::AffixFuzzer1::to_arrow( + memory_pool, + reinterpret_cast(elements), + num_elements); + return builder; + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer3.hpp b/rerun_cpp/src/components/affix_fuzzer3.hpp index b124479821b8..7a226c16f4a2 100644 --- a/rerun_cpp/src/components/affix_fuzzer3.hpp +++ b/rerun_cpp/src/components/affix_fuzzer3.hpp @@ -5,13 +5,16 @@ #include "../datatypes/affix_fuzzer1.hpp" +#include #include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace components { @@ -24,6 +27,10 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer3* elements, size_t num_elements); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer4.cpp b/rerun_cpp/src/components/affix_fuzzer4.cpp index 7b5c4ab0f69e..b6c7e4f75ae3 100644 --- a/rerun_cpp/src/components/affix_fuzzer4.cpp +++ b/rerun_cpp/src/components/affix_fuzzer4.cpp @@ -12,5 +12,23 @@ namespace rr { std::shared_ptr AffixFuzzer4::to_arrow_datatype() { return rr::datatypes::AffixFuzzer1::to_arrow_datatype(); } + + arrow::Result> AffixFuzzer4::to_arrow( + arrow::MemoryPool *memory_pool, const AffixFuzzer4 *elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + static_assert(sizeof(AffixFuzzer4) == sizeof(rr::datatypes::AffixFuzzer1), + "Expected fully transparent type."); + auto builder = rr::datatypes::AffixFuzzer1::to_arrow( + memory_pool, + reinterpret_cast(elements), + num_elements); + return builder; + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer4.hpp b/rerun_cpp/src/components/affix_fuzzer4.hpp index bd0f478a2bad..bb477ef2359c 100644 --- a/rerun_cpp/src/components/affix_fuzzer4.hpp +++ b/rerun_cpp/src/components/affix_fuzzer4.hpp @@ -5,14 +5,17 @@ #include "../datatypes/affix_fuzzer1.hpp" +#include #include #include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace components { @@ -25,6 +28,10 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer4* elements, size_t num_elements); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer5.cpp b/rerun_cpp/src/components/affix_fuzzer5.cpp index ba18315c31c9..5cd55fbeda2a 100644 --- a/rerun_cpp/src/components/affix_fuzzer5.cpp +++ b/rerun_cpp/src/components/affix_fuzzer5.cpp @@ -12,5 +12,23 @@ namespace rr { std::shared_ptr AffixFuzzer5::to_arrow_datatype() { return rr::datatypes::AffixFuzzer1::to_arrow_datatype(); } + + arrow::Result> AffixFuzzer5::to_arrow( + arrow::MemoryPool *memory_pool, const AffixFuzzer5 *elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + static_assert(sizeof(AffixFuzzer5) == sizeof(rr::datatypes::AffixFuzzer1), + "Expected fully transparent type."); + auto builder = rr::datatypes::AffixFuzzer1::to_arrow( + memory_pool, + reinterpret_cast(elements), + num_elements); + return builder; + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer5.hpp b/rerun_cpp/src/components/affix_fuzzer5.hpp index c376bd5bff54..fc0b4007653e 100644 --- a/rerun_cpp/src/components/affix_fuzzer5.hpp +++ b/rerun_cpp/src/components/affix_fuzzer5.hpp @@ -5,14 +5,17 @@ #include "../datatypes/affix_fuzzer1.hpp" +#include #include #include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace components { @@ -25,6 +28,10 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer5* elements, size_t num_elements); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer6.cpp b/rerun_cpp/src/components/affix_fuzzer6.cpp index 03699aeee233..2381e808af3b 100644 --- a/rerun_cpp/src/components/affix_fuzzer6.cpp +++ b/rerun_cpp/src/components/affix_fuzzer6.cpp @@ -12,5 +12,23 @@ namespace rr { std::shared_ptr AffixFuzzer6::to_arrow_datatype() { return rr::datatypes::AffixFuzzer1::to_arrow_datatype(); } + + arrow::Result> AffixFuzzer6::to_arrow( + arrow::MemoryPool *memory_pool, const AffixFuzzer6 *elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + static_assert(sizeof(AffixFuzzer6) == sizeof(rr::datatypes::AffixFuzzer1), + "Expected fully transparent type."); + auto builder = rr::datatypes::AffixFuzzer1::to_arrow( + memory_pool, + reinterpret_cast(elements), + num_elements); + return builder; + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer6.hpp b/rerun_cpp/src/components/affix_fuzzer6.hpp index 0de4f39d7170..a39106a820dd 100644 --- a/rerun_cpp/src/components/affix_fuzzer6.hpp +++ b/rerun_cpp/src/components/affix_fuzzer6.hpp @@ -5,14 +5,17 @@ #include "../datatypes/affix_fuzzer1.hpp" +#include #include #include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace components { @@ -25,6 +28,10 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer6* elements, size_t num_elements); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer7.cpp b/rerun_cpp/src/components/affix_fuzzer7.cpp index 5b820f97cc2c..b4f935b6efa4 100644 --- a/rerun_cpp/src/components/affix_fuzzer7.cpp +++ b/rerun_cpp/src/components/affix_fuzzer7.cpp @@ -13,5 +13,18 @@ namespace rr { return arrow::list(arrow::field( "item", rr::datatypes::AffixFuzzer1::to_arrow_datatype(), true, nullptr)); } + + arrow::Result> AffixFuzzer7::to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer7* elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto builder = std::make_shared(memory_pool); + return builder; + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer7.hpp b/rerun_cpp/src/components/affix_fuzzer7.hpp index 33de99180505..b4b2905575fa 100644 --- a/rerun_cpp/src/components/affix_fuzzer7.hpp +++ b/rerun_cpp/src/components/affix_fuzzer7.hpp @@ -5,6 +5,7 @@ #include "../datatypes/affix_fuzzer1.hpp" +#include #include #include #include @@ -12,8 +13,10 @@ #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace components { @@ -26,6 +29,10 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer7* elements, size_t num_elements); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer8.cpp b/rerun_cpp/src/components/affix_fuzzer8.cpp index 1be2d22a9c4c..91d9e4ac6ee1 100644 --- a/rerun_cpp/src/components/affix_fuzzer8.cpp +++ b/rerun_cpp/src/components/affix_fuzzer8.cpp @@ -10,5 +10,18 @@ namespace rr { std::shared_ptr AffixFuzzer8::to_arrow_datatype() { return arrow::float32(); } + + arrow::Result> AffixFuzzer8::to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer8* elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto builder = std::make_shared(memory_pool); + return builder; + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer8.hpp b/rerun_cpp/src/components/affix_fuzzer8.hpp index 48defb1b7a94..a9ef965c0203 100644 --- a/rerun_cpp/src/components/affix_fuzzer8.hpp +++ b/rerun_cpp/src/components/affix_fuzzer8.hpp @@ -3,14 +3,17 @@ #pragma once +#include #include #include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace components { @@ -23,6 +26,10 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer8* elements, size_t num_elements); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer9.cpp b/rerun_cpp/src/components/affix_fuzzer9.cpp index 54d23bf742fc..43482faa1f71 100644 --- a/rerun_cpp/src/components/affix_fuzzer9.cpp +++ b/rerun_cpp/src/components/affix_fuzzer9.cpp @@ -10,5 +10,18 @@ namespace rr { std::shared_ptr AffixFuzzer9::to_arrow_datatype() { return arrow::utf8(); } + + arrow::Result> AffixFuzzer9::to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer9* elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto builder = std::make_shared(memory_pool); + return builder; + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer9.hpp b/rerun_cpp/src/components/affix_fuzzer9.hpp index 8a89f340411f..3fb7fa0150c8 100644 --- a/rerun_cpp/src/components/affix_fuzzer9.hpp +++ b/rerun_cpp/src/components/affix_fuzzer9.hpp @@ -3,14 +3,17 @@ #pragma once +#include #include #include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace components { @@ -23,6 +26,10 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer9* elements, size_t num_elements); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/class_id.cpp b/rerun_cpp/src/components/class_id.cpp index 96bc2887d7cc..2a9e2ad59bff 100644 --- a/rerun_cpp/src/components/class_id.cpp +++ b/rerun_cpp/src/components/class_id.cpp @@ -10,5 +10,18 @@ namespace rr { std::shared_ptr ClassId::to_arrow_datatype() { return arrow::uint16(); } + + arrow::Result> ClassId::to_arrow( + arrow::MemoryPool* memory_pool, const ClassId* elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto builder = std::make_shared(memory_pool); + return builder; + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/class_id.hpp b/rerun_cpp/src/components/class_id.hpp index e8663e0b93a7..e16885dc6693 100644 --- a/rerun_cpp/src/components/class_id.hpp +++ b/rerun_cpp/src/components/class_id.hpp @@ -3,13 +3,16 @@ #pragma once +#include #include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace components { @@ -22,6 +25,10 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const ClassId* elements, size_t num_elements); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/color.cpp b/rerun_cpp/src/components/color.cpp index a8a219d507d4..a42c4d39d6d1 100644 --- a/rerun_cpp/src/components/color.cpp +++ b/rerun_cpp/src/components/color.cpp @@ -10,5 +10,18 @@ namespace rr { std::shared_ptr Color::to_arrow_datatype() { return arrow::uint32(); } + + arrow::Result> Color::to_arrow( + arrow::MemoryPool* memory_pool, const Color* elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto builder = std::make_shared(memory_pool); + return builder; + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/color.hpp b/rerun_cpp/src/components/color.hpp index 22c487e46286..ffe3fce5cbea 100644 --- a/rerun_cpp/src/components/color.hpp +++ b/rerun_cpp/src/components/color.hpp @@ -3,13 +3,16 @@ #pragma once +#include #include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace components { @@ -23,6 +26,10 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const Color* elements, size_t num_elements); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/draw_order.cpp b/rerun_cpp/src/components/draw_order.cpp index 25c1f3d88ef5..39efbcc9e3b4 100644 --- a/rerun_cpp/src/components/draw_order.cpp +++ b/rerun_cpp/src/components/draw_order.cpp @@ -10,5 +10,18 @@ namespace rr { std::shared_ptr DrawOrder::to_arrow_datatype() { return arrow::float32(); } + + arrow::Result> DrawOrder::to_arrow( + arrow::MemoryPool* memory_pool, const DrawOrder* elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto builder = std::make_shared(memory_pool); + return builder; + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/draw_order.hpp b/rerun_cpp/src/components/draw_order.hpp index 8da0c3efface..a3e9fc586364 100644 --- a/rerun_cpp/src/components/draw_order.hpp +++ b/rerun_cpp/src/components/draw_order.hpp @@ -3,13 +3,16 @@ #pragma once +#include #include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace components { @@ -28,6 +31,10 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const DrawOrder* elements, size_t num_elements); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/instance_key.cpp b/rerun_cpp/src/components/instance_key.cpp index 3ebf77f59f00..3de7de491f2f 100644 --- a/rerun_cpp/src/components/instance_key.cpp +++ b/rerun_cpp/src/components/instance_key.cpp @@ -10,5 +10,18 @@ namespace rr { std::shared_ptr InstanceKey::to_arrow_datatype() { return arrow::uint64(); } + + arrow::Result> InstanceKey::to_arrow( + arrow::MemoryPool* memory_pool, const InstanceKey* elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto builder = std::make_shared(memory_pool); + return builder; + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/instance_key.hpp b/rerun_cpp/src/components/instance_key.hpp index 561b9d753acc..f1ad7ea5245a 100644 --- a/rerun_cpp/src/components/instance_key.hpp +++ b/rerun_cpp/src/components/instance_key.hpp @@ -3,13 +3,16 @@ #pragma once +#include #include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace components { @@ -22,6 +25,10 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const InstanceKey* elements, size_t num_elements); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/keypoint_id.cpp b/rerun_cpp/src/components/keypoint_id.cpp index c8bcd72a6b63..89642da4b68f 100644 --- a/rerun_cpp/src/components/keypoint_id.cpp +++ b/rerun_cpp/src/components/keypoint_id.cpp @@ -10,5 +10,18 @@ namespace rr { std::shared_ptr KeypointId::to_arrow_datatype() { return arrow::uint16(); } + + arrow::Result> KeypointId::to_arrow( + arrow::MemoryPool* memory_pool, const KeypointId* elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto builder = std::make_shared(memory_pool); + return builder; + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/keypoint_id.hpp b/rerun_cpp/src/components/keypoint_id.hpp index b2c8a985aeb6..9d9839155a72 100644 --- a/rerun_cpp/src/components/keypoint_id.hpp +++ b/rerun_cpp/src/components/keypoint_id.hpp @@ -3,13 +3,16 @@ #pragma once +#include #include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace components { @@ -22,6 +25,10 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const KeypointId* elements, size_t num_elements); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/label.cpp b/rerun_cpp/src/components/label.cpp index af2d9bffdac8..2ae584f899ea 100644 --- a/rerun_cpp/src/components/label.cpp +++ b/rerun_cpp/src/components/label.cpp @@ -10,5 +10,18 @@ namespace rr { std::shared_ptr Label::to_arrow_datatype() { return arrow::utf8(); } + + arrow::Result> Label::to_arrow( + arrow::MemoryPool* memory_pool, const Label* elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto builder = std::make_shared(memory_pool); + return builder; + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/label.hpp b/rerun_cpp/src/components/label.hpp index f2b51c7efb30..bb8b4efe74b9 100644 --- a/rerun_cpp/src/components/label.hpp +++ b/rerun_cpp/src/components/label.hpp @@ -3,14 +3,17 @@ #pragma once +#include #include #include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace components { @@ -23,6 +26,10 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const Label* elements, size_t num_elements); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/point2d.cpp b/rerun_cpp/src/components/point2d.cpp index 0e53d5d09b09..e951cfeda2c2 100644 --- a/rerun_cpp/src/components/point2d.cpp +++ b/rerun_cpp/src/components/point2d.cpp @@ -12,5 +12,23 @@ namespace rr { std::shared_ptr Point2D::to_arrow_datatype() { return rr::datatypes::Point2D::to_arrow_datatype(); } + + arrow::Result> Point2D::to_arrow( + arrow::MemoryPool *memory_pool, const Point2D *elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + static_assert(sizeof(Point2D) == sizeof(rr::datatypes::Point2D), + "Expected fully transparent type."); + auto builder = rr::datatypes::Point2D::to_arrow( + memory_pool, + reinterpret_cast(elements), + num_elements); + return builder; + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/point2d.hpp b/rerun_cpp/src/components/point2d.hpp index af611cf8e760..fabdc2a54a5f 100644 --- a/rerun_cpp/src/components/point2d.hpp +++ b/rerun_cpp/src/components/point2d.hpp @@ -5,13 +5,16 @@ #include "../datatypes/point2d.hpp" +#include #include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace components { @@ -24,6 +27,10 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const Point2D* elements, size_t num_elements); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/point3d.cpp b/rerun_cpp/src/components/point3d.cpp index ff4b363341bb..9bf9e0f5a244 100644 --- a/rerun_cpp/src/components/point3d.cpp +++ b/rerun_cpp/src/components/point3d.cpp @@ -12,5 +12,23 @@ namespace rr { std::shared_ptr Point3D::to_arrow_datatype() { return rr::datatypes::Point3D::to_arrow_datatype(); } + + arrow::Result> Point3D::to_arrow( + arrow::MemoryPool *memory_pool, const Point3D *elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + static_assert(sizeof(Point3D) == sizeof(rr::datatypes::Point3D), + "Expected fully transparent type."); + auto builder = rr::datatypes::Point3D::to_arrow( + memory_pool, + reinterpret_cast(elements), + num_elements); + return builder; + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/point3d.hpp b/rerun_cpp/src/components/point3d.hpp index f3f28c0f0891..166f64dbad75 100644 --- a/rerun_cpp/src/components/point3d.hpp +++ b/rerun_cpp/src/components/point3d.hpp @@ -5,13 +5,16 @@ #include "../datatypes/point3d.hpp" +#include #include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace components { @@ -24,6 +27,10 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const Point3D* elements, size_t num_elements); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/radius.cpp b/rerun_cpp/src/components/radius.cpp index 0ebee94ca034..f73f437f6708 100644 --- a/rerun_cpp/src/components/radius.cpp +++ b/rerun_cpp/src/components/radius.cpp @@ -10,5 +10,18 @@ namespace rr { std::shared_ptr Radius::to_arrow_datatype() { return arrow::float32(); } + + arrow::Result> Radius::to_arrow( + arrow::MemoryPool* memory_pool, const Radius* elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto builder = std::make_shared(memory_pool); + return builder; + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/radius.hpp b/rerun_cpp/src/components/radius.hpp index 9fc1aafde961..6ab8800695dc 100644 --- a/rerun_cpp/src/components/radius.hpp +++ b/rerun_cpp/src/components/radius.hpp @@ -3,13 +3,16 @@ #pragma once +#include #include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace components { @@ -22,6 +25,10 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const Radius* elements, size_t num_elements); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/transform3d.cpp b/rerun_cpp/src/components/transform3d.cpp index 022fa1c2668e..085796a837d7 100644 --- a/rerun_cpp/src/components/transform3d.cpp +++ b/rerun_cpp/src/components/transform3d.cpp @@ -12,5 +12,23 @@ namespace rr { std::shared_ptr Transform3D::to_arrow_datatype() { return rr::datatypes::Transform3D::to_arrow_datatype(); } + + arrow::Result> Transform3D::to_arrow( + arrow::MemoryPool *memory_pool, const Transform3D *elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + static_assert(sizeof(Transform3D) == sizeof(rr::datatypes::Transform3D), + "Expected fully transparent type."); + auto builder = rr::datatypes::Transform3D::to_arrow( + memory_pool, + reinterpret_cast(elements), + num_elements); + return builder; + } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/transform3d.hpp b/rerun_cpp/src/components/transform3d.hpp index 9d049d24358e..e8e3e3060d08 100644 --- a/rerun_cpp/src/components/transform3d.hpp +++ b/rerun_cpp/src/components/transform3d.hpp @@ -5,13 +5,16 @@ #include "../datatypes/transform3d.hpp" +#include #include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace components { @@ -25,6 +28,10 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const Transform3D* elements, size_t num_elements); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/datatypes/affix_fuzzer1.cpp b/rerun_cpp/src/datatypes/affix_fuzzer1.cpp index ddcc8c816f3c..619e67ddf23a 100644 --- a/rerun_cpp/src/datatypes/affix_fuzzer1.cpp +++ b/rerun_cpp/src/datatypes/affix_fuzzer1.cpp @@ -34,5 +34,22 @@ namespace rr { arrow::field("from_parent", arrow::boolean(), true, nullptr), }); } + + arrow::Result> AffixFuzzer1::to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer1* elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto datatype = AffixFuzzer1::to_arrow_datatype(); + let builder = + std::make_shared(datatype, memory_pool, {}, + // TODO(#2647): code-gen for C++ + ); + return builder; + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/affix_fuzzer1.hpp b/rerun_cpp/src/datatypes/affix_fuzzer1.hpp index 922ff3042a81..ed8c62a4cdcf 100644 --- a/rerun_cpp/src/datatypes/affix_fuzzer1.hpp +++ b/rerun_cpp/src/datatypes/affix_fuzzer1.hpp @@ -5,6 +5,7 @@ #include "../datatypes/flattened_scalar.hpp" +#include #include #include #include @@ -12,8 +13,10 @@ #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace datatypes { @@ -39,6 +42,10 @@ namespace rr { public: /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer1* elements, size_t num_elements); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/affix_fuzzer2.cpp b/rerun_cpp/src/datatypes/affix_fuzzer2.cpp index 7b83c7618ccd..35ee82382f52 100644 --- a/rerun_cpp/src/datatypes/affix_fuzzer2.cpp +++ b/rerun_cpp/src/datatypes/affix_fuzzer2.cpp @@ -10,5 +10,18 @@ namespace rr { std::shared_ptr AffixFuzzer2::to_arrow_datatype() { return arrow::float32(); } + + arrow::Result> AffixFuzzer2::to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer2* elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto builder = std::make_shared(memory_pool); + return builder; + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/affix_fuzzer2.hpp b/rerun_cpp/src/datatypes/affix_fuzzer2.hpp index 3db3f9b54b7b..a79eaa88cd1e 100644 --- a/rerun_cpp/src/datatypes/affix_fuzzer2.hpp +++ b/rerun_cpp/src/datatypes/affix_fuzzer2.hpp @@ -3,14 +3,17 @@ #pragma once +#include #include #include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace datatypes { @@ -23,6 +26,10 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer2* elements, size_t num_elements); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/affix_fuzzer5.cpp b/rerun_cpp/src/datatypes/affix_fuzzer5.cpp index 7211ae386353..e592df705313 100644 --- a/rerun_cpp/src/datatypes/affix_fuzzer5.cpp +++ b/rerun_cpp/src/datatypes/affix_fuzzer5.cpp @@ -17,5 +17,22 @@ namespace rr { nullptr), }); } + + arrow::Result> AffixFuzzer5::to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer5* elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto datatype = AffixFuzzer5::to_arrow_datatype(); + let builder = + std::make_shared(datatype, memory_pool, {}, + // TODO(#2647): code-gen for C++ + ); + return builder; + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/affix_fuzzer5.hpp b/rerun_cpp/src/datatypes/affix_fuzzer5.hpp index bff009a92ce3..f829aa48381c 100644 --- a/rerun_cpp/src/datatypes/affix_fuzzer5.hpp +++ b/rerun_cpp/src/datatypes/affix_fuzzer5.hpp @@ -5,14 +5,17 @@ #include "../datatypes/affix_fuzzer4.hpp" +#include #include #include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace datatypes { @@ -25,6 +28,10 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const AffixFuzzer5* elements, size_t num_elements); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/flattened_scalar.cpp b/rerun_cpp/src/datatypes/flattened_scalar.cpp index f54aff725814..1eba191539f2 100644 --- a/rerun_cpp/src/datatypes/flattened_scalar.cpp +++ b/rerun_cpp/src/datatypes/flattened_scalar.cpp @@ -12,5 +12,22 @@ namespace rr { arrow::field("value", arrow::float32(), false, nullptr), }); } + + arrow::Result> FlattenedScalar::to_arrow( + arrow::MemoryPool* memory_pool, const FlattenedScalar* elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto datatype = FlattenedScalar::to_arrow_datatype(); + let builder = + std::make_shared(datatype, memory_pool, {}, + // TODO(#2647): code-gen for C++ + ); + return builder; + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/flattened_scalar.hpp b/rerun_cpp/src/datatypes/flattened_scalar.hpp index 0e5ce80b7690..b7e4b52016e0 100644 --- a/rerun_cpp/src/datatypes/flattened_scalar.hpp +++ b/rerun_cpp/src/datatypes/flattened_scalar.hpp @@ -3,13 +3,16 @@ #pragma once +#include #include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace datatypes { @@ -21,6 +24,11 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const FlattenedScalar* elements, + size_t num_elements); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/mat3x3.cpp b/rerun_cpp/src/datatypes/mat3x3.cpp index 02c917d274a3..61d9ab50d838 100644 --- a/rerun_cpp/src/datatypes/mat3x3.cpp +++ b/rerun_cpp/src/datatypes/mat3x3.cpp @@ -11,5 +11,18 @@ namespace rr { return arrow::fixed_size_list(arrow::field("item", arrow::float32(), false, nullptr), 9); } + + arrow::Result> Mat3x3::to_arrow( + arrow::MemoryPool* memory_pool, const Mat3x3* elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto builder = std::make_shared(memory_pool); + return builder; + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/mat3x3.hpp b/rerun_cpp/src/datatypes/mat3x3.hpp index 9e605c364223..e4839179b111 100644 --- a/rerun_cpp/src/datatypes/mat3x3.hpp +++ b/rerun_cpp/src/datatypes/mat3x3.hpp @@ -3,12 +3,15 @@ #pragma once +#include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace datatypes { @@ -19,6 +22,10 @@ namespace rr { public: /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const Mat3x3* elements, size_t num_elements); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/mat4x4.cpp b/rerun_cpp/src/datatypes/mat4x4.cpp index 212fbfe5f87c..7240cedaed30 100644 --- a/rerun_cpp/src/datatypes/mat4x4.cpp +++ b/rerun_cpp/src/datatypes/mat4x4.cpp @@ -11,5 +11,18 @@ namespace rr { return arrow::fixed_size_list(arrow::field("item", arrow::float32(), false, nullptr), 16); } + + arrow::Result> Mat4x4::to_arrow( + arrow::MemoryPool* memory_pool, const Mat4x4* elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto builder = std::make_shared(memory_pool); + return builder; + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/mat4x4.hpp b/rerun_cpp/src/datatypes/mat4x4.hpp index 2793f21f4be7..d90db72311a7 100644 --- a/rerun_cpp/src/datatypes/mat4x4.hpp +++ b/rerun_cpp/src/datatypes/mat4x4.hpp @@ -3,12 +3,15 @@ #pragma once +#include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace datatypes { @@ -19,6 +22,10 @@ namespace rr { public: /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const Mat4x4* elements, size_t num_elements); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/point2d.cpp b/rerun_cpp/src/datatypes/point2d.cpp index da68dbd6b781..6b11a3230c9f 100644 --- a/rerun_cpp/src/datatypes/point2d.cpp +++ b/rerun_cpp/src/datatypes/point2d.cpp @@ -13,5 +13,22 @@ namespace rr { arrow::field("y", arrow::float32(), false, nullptr), }); } + + arrow::Result> Point2D::to_arrow( + arrow::MemoryPool* memory_pool, const Point2D* elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto datatype = Point2D::to_arrow_datatype(); + let builder = + std::make_shared(datatype, memory_pool, {}, + // TODO(#2647): code-gen for C++ + ); + return builder; + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/point2d.hpp b/rerun_cpp/src/datatypes/point2d.hpp index 9d1cb74e9d7b..84d9fb562143 100644 --- a/rerun_cpp/src/datatypes/point2d.hpp +++ b/rerun_cpp/src/datatypes/point2d.hpp @@ -3,12 +3,15 @@ #pragma once +#include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace datatypes { @@ -21,6 +24,10 @@ namespace rr { public: /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const Point2D* elements, size_t num_elements); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/point3d.cpp b/rerun_cpp/src/datatypes/point3d.cpp index f6bae5a0411b..b55b32fc8fe8 100644 --- a/rerun_cpp/src/datatypes/point3d.cpp +++ b/rerun_cpp/src/datatypes/point3d.cpp @@ -14,5 +14,22 @@ namespace rr { arrow::field("z", arrow::float32(), false, nullptr), }); } + + arrow::Result> Point3D::to_arrow( + arrow::MemoryPool* memory_pool, const Point3D* elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto datatype = Point3D::to_arrow_datatype(); + let builder = + std::make_shared(datatype, memory_pool, {}, + // TODO(#2647): code-gen for C++ + ); + return builder; + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/point3d.hpp b/rerun_cpp/src/datatypes/point3d.hpp index 78b6e9b04e0b..534289e63bc4 100644 --- a/rerun_cpp/src/datatypes/point3d.hpp +++ b/rerun_cpp/src/datatypes/point3d.hpp @@ -3,12 +3,15 @@ #pragma once +#include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace datatypes { @@ -23,6 +26,10 @@ namespace rr { public: /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const Point3D* elements, size_t num_elements); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/quaternion.cpp b/rerun_cpp/src/datatypes/quaternion.cpp index f206392e3c29..f16f3689ce12 100644 --- a/rerun_cpp/src/datatypes/quaternion.cpp +++ b/rerun_cpp/src/datatypes/quaternion.cpp @@ -11,5 +11,18 @@ namespace rr { return arrow::fixed_size_list(arrow::field("item", arrow::float32(), false, nullptr), 4); } + + arrow::Result> Quaternion::to_arrow( + arrow::MemoryPool* memory_pool, const Quaternion* elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto builder = std::make_shared(memory_pool); + return builder; + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/quaternion.hpp b/rerun_cpp/src/datatypes/quaternion.hpp index 11abeed97f9a..23007d2387e2 100644 --- a/rerun_cpp/src/datatypes/quaternion.hpp +++ b/rerun_cpp/src/datatypes/quaternion.hpp @@ -3,12 +3,15 @@ #pragma once +#include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace datatypes { @@ -19,6 +22,10 @@ namespace rr { public: /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const Quaternion* elements, size_t num_elements); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/rotation_axis_angle.cpp b/rerun_cpp/src/datatypes/rotation_axis_angle.cpp index 98c460ff92db..f5ffa9926b06 100644 --- a/rerun_cpp/src/datatypes/rotation_axis_angle.cpp +++ b/rerun_cpp/src/datatypes/rotation_axis_angle.cpp @@ -16,5 +16,23 @@ namespace rr { arrow::field("angle", rr::datatypes::Angle::to_arrow_datatype(), false, nullptr), }); } + + arrow::Result> RotationAxisAngle::to_arrow( + arrow::MemoryPool* memory_pool, const RotationAxisAngle* elements, + size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto datatype = RotationAxisAngle::to_arrow_datatype(); + let builder = + std::make_shared(datatype, memory_pool, {}, + // TODO(#2647): code-gen for C++ + ); + return builder; + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/rotation_axis_angle.hpp b/rerun_cpp/src/datatypes/rotation_axis_angle.hpp index 894140bc5667..d689df4d10b7 100644 --- a/rerun_cpp/src/datatypes/rotation_axis_angle.hpp +++ b/rerun_cpp/src/datatypes/rotation_axis_angle.hpp @@ -6,12 +6,15 @@ #include "../datatypes/angle.hpp" #include "../datatypes/vec3d.hpp" +#include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace datatypes { @@ -30,6 +33,11 @@ namespace rr { public: /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const RotationAxisAngle* elements, + size_t num_elements); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/translation_and_mat3x3.cpp b/rerun_cpp/src/datatypes/translation_and_mat3x3.cpp index c94faa6e9bad..bfe83f477d49 100644 --- a/rerun_cpp/src/datatypes/translation_and_mat3x3.cpp +++ b/rerun_cpp/src/datatypes/translation_and_mat3x3.cpp @@ -18,5 +18,23 @@ namespace rr { arrow::field("from_parent", arrow::boolean(), false, nullptr), }); } + + arrow::Result> TranslationAndMat3x3::to_arrow( + arrow::MemoryPool* memory_pool, const TranslationAndMat3x3* elements, + size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto datatype = TranslationAndMat3x3::to_arrow_datatype(); + let builder = + std::make_shared(datatype, memory_pool, {}, + // TODO(#2647): code-gen for C++ + ); + return builder; + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/translation_and_mat3x3.hpp b/rerun_cpp/src/datatypes/translation_and_mat3x3.hpp index b56858e24443..157bec963561 100644 --- a/rerun_cpp/src/datatypes/translation_and_mat3x3.hpp +++ b/rerun_cpp/src/datatypes/translation_and_mat3x3.hpp @@ -6,13 +6,16 @@ #include "../datatypes/mat3x3.hpp" #include "../datatypes/vec3d.hpp" +#include #include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace datatypes { @@ -33,6 +36,11 @@ namespace rr { public: /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const TranslationAndMat3x3* elements, + size_t num_elements); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/translation_rotation_scale3d.cpp b/rerun_cpp/src/datatypes/translation_rotation_scale3d.cpp index 9b09dd315da9..b56f45ccd3de 100644 --- a/rerun_cpp/src/datatypes/translation_rotation_scale3d.cpp +++ b/rerun_cpp/src/datatypes/translation_rotation_scale3d.cpp @@ -21,5 +21,23 @@ namespace rr { arrow::field("from_parent", arrow::boolean(), false, nullptr), }); } + + arrow::Result> TranslationRotationScale3D::to_arrow( + arrow::MemoryPool* memory_pool, const TranslationRotationScale3D* elements, + size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto datatype = TranslationRotationScale3D::to_arrow_datatype(); + let builder = + std::make_shared(datatype, memory_pool, {}, + // TODO(#2647): code-gen for C++ + ); + return builder; + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/translation_rotation_scale3d.hpp b/rerun_cpp/src/datatypes/translation_rotation_scale3d.hpp index dfca120794c2..8b2d621da71f 100644 --- a/rerun_cpp/src/datatypes/translation_rotation_scale3d.hpp +++ b/rerun_cpp/src/datatypes/translation_rotation_scale3d.hpp @@ -7,13 +7,16 @@ #include "../datatypes/scale3d.hpp" #include "../datatypes/vec3d.hpp" +#include #include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace datatypes { @@ -35,6 +38,11 @@ namespace rr { public: /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const TranslationRotationScale3D* elements, + size_t num_elements); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/vec2d.cpp b/rerun_cpp/src/datatypes/vec2d.cpp index d07bbc291717..d2178481aa3e 100644 --- a/rerun_cpp/src/datatypes/vec2d.cpp +++ b/rerun_cpp/src/datatypes/vec2d.cpp @@ -11,5 +11,18 @@ namespace rr { return arrow::fixed_size_list(arrow::field("item", arrow::float32(), false, nullptr), 2); } + + arrow::Result> Vec2D::to_arrow( + arrow::MemoryPool* memory_pool, const Vec2D* elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto builder = std::make_shared(memory_pool); + return builder; + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/vec2d.hpp b/rerun_cpp/src/datatypes/vec2d.hpp index 50bea24f4e0a..0051c5a44f06 100644 --- a/rerun_cpp/src/datatypes/vec2d.hpp +++ b/rerun_cpp/src/datatypes/vec2d.hpp @@ -3,12 +3,15 @@ #pragma once +#include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace datatypes { @@ -19,6 +22,10 @@ namespace rr { public: /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const Vec2D* elements, size_t num_elements); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/vec3d.cpp b/rerun_cpp/src/datatypes/vec3d.cpp index e7baacc0f235..7b136afdae37 100644 --- a/rerun_cpp/src/datatypes/vec3d.cpp +++ b/rerun_cpp/src/datatypes/vec3d.cpp @@ -11,5 +11,18 @@ namespace rr { return arrow::fixed_size_list(arrow::field("item", arrow::float32(), false, nullptr), 3); } + + arrow::Result> Vec3D::to_arrow( + arrow::MemoryPool* memory_pool, const Vec3D* elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto builder = std::make_shared(memory_pool); + return builder; + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/vec3d.hpp b/rerun_cpp/src/datatypes/vec3d.hpp index 72fad69ea7a2..c9ba8bf3b240 100644 --- a/rerun_cpp/src/datatypes/vec3d.hpp +++ b/rerun_cpp/src/datatypes/vec3d.hpp @@ -3,12 +3,15 @@ #pragma once +#include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace datatypes { @@ -19,6 +22,10 @@ namespace rr { public: /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const Vec3D* elements, size_t num_elements); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/vec4d.cpp b/rerun_cpp/src/datatypes/vec4d.cpp index 208e9089c47c..058e1ea267cb 100644 --- a/rerun_cpp/src/datatypes/vec4d.cpp +++ b/rerun_cpp/src/datatypes/vec4d.cpp @@ -11,5 +11,18 @@ namespace rr { return arrow::fixed_size_list(arrow::field("item", arrow::float32(), false, nullptr), 4); } + + arrow::Result> Vec4D::to_arrow( + arrow::MemoryPool* memory_pool, const Vec4D* elements, size_t num_elements) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + auto builder = std::make_shared(memory_pool); + return builder; + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/vec4d.hpp b/rerun_cpp/src/datatypes/vec4d.hpp index f9745ba3aaa2..0086942146fe 100644 --- a/rerun_cpp/src/datatypes/vec4d.hpp +++ b/rerun_cpp/src/datatypes/vec4d.hpp @@ -3,12 +3,15 @@ #pragma once +#include #include #include namespace arrow { + class ArrayBuilder; class DataType; -} + class MemoryPool; +} // namespace arrow namespace rr { namespace datatypes { @@ -19,6 +22,10 @@ namespace rr { public: /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + + /// Fills out an arrow array builder with an array of this type. + static arrow::Result> to_arrow( + arrow::MemoryPool* memory_pool, const Vec4D* elements, size_t num_elements); }; } // namespace datatypes } // namespace rr From 168a1f4be443d9cf1467442ce8957a171bccea18 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Fri, 21 Jul 2023 18:59:06 +0200 Subject: [PATCH 03/20] build hierarchy of array builder in to_arrow method --- .../re_types_builder/src/codegen/cpp/mod.rs | 167 ++++++++++++------ 1 file changed, 113 insertions(+), 54 deletions(-) diff --git a/crates/re_types_builder/src/codegen/cpp/mod.rs b/crates/re_types_builder/src/codegen/cpp/mod.rs index 6c19eaf29b49..d651eed5a108 100644 --- a/crates/re_types_builder/src/codegen/cpp/mod.rs +++ b/crates/re_types_builder/src/codegen/cpp/mod.rs @@ -4,6 +4,7 @@ mod method; use std::collections::BTreeSet; +use anyhow::Context; use arrow2::datatypes::DataType; use camino::{Utf8Path, Utf8PathBuf}; use itertools::Itertools; @@ -503,6 +504,8 @@ impl QuotedObject { let hpp = quote! { #hpp_includes + #hpp_declarations + namespace rr { namespace #namespace_ident { namespace detail { @@ -607,86 +610,102 @@ fn arrow_data_type_method( } } -fn to_arrow_method( +fn quote_fill_arrow_builder( datatype: &DataType, - pascal_case_ident: &Ident, - hpp_includes: &mut Includes, + extension_type_ident: Option<&Ident>, + builder_name: &Ident, + pool: &Ident, cpp_includes: &mut Includes, - hpp_declarations: &mut ForwardDecls, -) -> Method { - hpp_declarations.insert("arrow", ForwardDecl::Class("ArrayBuilder".to_owned())); - hpp_declarations.insert("arrow", ForwardDecl::Class("MemoryPool".to_owned())); - hpp_includes.system.insert("arrow/result.h".to_owned()); - hpp_includes.system.insert("memory".to_owned()); // std::shared_ptr - cpp_includes.system.insert("arrow/api.h".to_owned()); - - let DataType::Extension(_fqname, logical_datatype, _metadata) = datatype else { - panic!("Can only generate arrow serialization code for extension types."); - }; - - let pool = format_ident!("memory_pool"); - - /// TODO: shorter code here for trivial ones? - let builder_initialization = match logical_datatype.as_ref() { +) -> TokenStream { + // TODO: shorter code here for trivial ones? + match datatype { DataType::Boolean => { - quote!(auto builder = std::make_shared(#pool);) + quote!(auto #builder_name = std::make_shared(#pool);) } - DataType::Int8 => quote!(auto builder = std::make_shared(#pool);), + DataType::Int8 => quote!(auto #builder_name = std::make_shared(#pool);), DataType::Int16 => { - quote!(auto builder = std::make_shared(#pool);) + quote!(auto #builder_name = std::make_shared(#pool);) } DataType::Int32 => { - quote!(auto builder = std::make_shared(#pool);) + quote!(auto #builder_name = std::make_shared(#pool);) } DataType::Int64 => { - quote!(auto builder = std::make_shared(#pool);) + quote!(auto #builder_name = std::make_shared(#pool);) } DataType::UInt8 => { - quote!(auto builder = std::make_shared(#pool);) + quote!(auto #builder_name = std::make_shared(#pool);) } DataType::UInt16 => { - quote!(auto builder = std::make_shared(#pool);) + quote!(auto #builder_name = std::make_shared(#pool);) } DataType::UInt32 => { - quote!(auto builder = std::make_shared(#pool);) + quote!(auto #builder_name = std::make_shared(#pool);) } DataType::UInt64 => { - quote!(auto builder = std::make_shared(#pool);) + quote!(auto #builder_name = std::make_shared(#pool);) } DataType::Float16 => { - quote!(auto builder = std::make_shared(#pool);) + quote!(auto #builder_name = std::make_shared(#pool);) } DataType::Float32 => { - quote!(auto builder = std::make_shared(#pool);) + quote!(auto #builder_name = std::make_shared(#pool);) } DataType::Float64 => { - quote!(auto builder = std::make_shared(#pool);) + quote!(auto #builder_name = std::make_shared(#pool);) } DataType::Binary => { - quote!(auto builder = std::make_shared(#pool);) + quote!(auto #builder_name = std::make_shared(#pool);) } DataType::LargeBinary => { - quote!(auto builder = std::make_shared(#pool);) + quote!(auto #builder_name = std::make_shared(#pool);) } DataType::Utf8 => { - quote!(auto builder = std::make_shared(#pool);) + quote!(auto #builder_name = std::make_shared(#pool);) } DataType::LargeUtf8 => { - quote!(auto builder = std::make_shared(#pool);) + quote!(auto #builder_name = std::make_shared(#pool);) } - DataType::List(_) => { - quote!(auto builder = std::make_shared(#pool);) + DataType::List(field) => { + // TODO(andreas): Nesting list of lists might require adjusting the name. + let child_builder_name = format_ident!("child_builder"); + let child_builder_impl = quote_fill_arrow_builder( + &field.as_ref().data_type, + None, + &child_builder_name, + pool, + cpp_includes, + ); + quote! { + #child_builder_impl + auto #builder_name = std::make_shared(#pool, #child_builder_name); + } } - DataType::FixedSizeList(_, _) => { - quote!(auto builder = std::make_shared(#pool);) + DataType::FixedSizeList(field, length) => { + // TODO(andreas): Nesting list of lists might require adjusting the name. + let child_builder_name = format_ident!("child_builder"); + let child_builder_impl = quote_fill_arrow_builder( + &field.as_ref().data_type, + None, + &child_builder_name, + pool, + cpp_includes, + ); + let quoted_length = quote_integer(length); + quote! { + #child_builder_impl + auto #builder_name = std::make_shared(#pool, #child_builder_name, #quoted_length); + } } - DataType::Struct(_) => { + DataType::Struct(fields) => { + let datatype_ident = extension_type_ident + .context("Structs that aren't wrapped in extension types aren't supported. + This should only happen when using a struct that wasn't generated by the codegenerator itself.") + .unwrap(); quote! { - auto datatype = #pascal_case_ident::to_arrow_datatype(); - let builder = std::make_shared( + auto datatype = #datatype_ident::to_arrow_datatype(); + auto builder = std::make_shared( datatype, - #pool, - {}, #TODO_TOKEN + #pool ); } } @@ -697,29 +716,68 @@ fn to_arrow_method( }; let builder_type = format_ident!("{builder_type}"); + let datatype_ident = extension_type_ident + .context("Structs that aren't wrapped in extension types aren't supported. + This should only happen when using a struct that wasn't generated by the codegenerator itself.") + .unwrap(); quote! { - auto datatype = #pascal_case_ident::to_arrow_datatype(); - let builder = std::make_shared( + datatype = #datatype_ident::to_arrow_datatype(); + auto builder = std::make_shared( datatype, #pool, - {}, #TODO_TOKEN + {} #TODO_TOKEN ); } } DataType::Extension(fqname, datatype, _metadata) => { - //if let datatype // TODO: unions not yet supported + if let DataType::Union(..) = datatype.as_ref() { + // TODO: not yet implemented + return quote! { + #TODO_TOKEN + auto #builder_name = std::make_shared(); + }; + } - // Extension in extension only happens if this is a transparent type pointing to another extension type. - // TODO: remove unnecessary namespacing. let quoted_fqname = quote_fqname_as_type_path(cpp_includes, fqname); quote! { - static_assert(sizeof(#pascal_case_ident) == sizeof(#quoted_fqname), "Expected fully transparent type."); - auto builder = #quoted_fqname::to_arrow(#pool, reinterpret_cast(elements), num_elements); + ARROW_ASSIGN_OR_RAISE( + auto #builder_name, + #quoted_fqname::to_arrow(#pool, reinterpret_cast(elements), num_elements) + ); } } - _ => unimplemented!("Arrow serialization for type: {:?}", logical_datatype), + _ => unimplemented!("Arrow serialization for type: {:?}", datatype), + } +} + +fn to_arrow_method( + datatype: &DataType, + pascal_case_ident: &Ident, + hpp_includes: &mut Includes, + cpp_includes: &mut Includes, + hpp_declarations: &mut ForwardDecls, +) -> Method { + hpp_declarations.insert("arrow", ForwardDecl::Class("ArrayBuilder".to_owned())); + hpp_declarations.insert("arrow", ForwardDecl::Class("MemoryPool".to_owned())); + hpp_includes.system.insert("arrow/result.h".to_owned()); + hpp_includes.system.insert("memory".to_owned()); // std::shared_ptr + cpp_includes.system.insert("arrow/api.h".to_owned()); + + let DataType::Extension(_fqname, logical_datatype, _metadata) = datatype else { + panic!("Can only generate arrow serialization code for extension types."); }; + let pool = format_ident!("memory_pool"); + let builder = format_ident!("builder"); + + let builder_initialization = quote_fill_arrow_builder( + logical_datatype, + Some(pascal_case_ident), + &builder, + &pool, + cpp_includes, + ); + Method { docs: "Fills out an arrow array builder with an array of this type.".into(), declaration: MethodDeclaration { @@ -740,7 +798,8 @@ fn to_arrow_method( #NEWLINE_TOKEN #NEWLINE_TOKEN #builder_initialization - return builder; + #NEWLINE_TOKEN + return #builder; }, inline: false, } From 71f2a8c4ead956a9b1d254d191a0b9e1c788e788 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Fri, 21 Jul 2023 19:00:17 +0200 Subject: [PATCH 04/20] formatting options for less deep nesting --- .clang-format | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.clang-format b/.clang-format index 5ae127526eed..8c45504e06a9 100644 --- a/.clang-format +++ b/.clang-format @@ -3,6 +3,8 @@ BasedOnStyle: Google # Make it slightly more similar to Rust. # Based loosely on https://gist.github.com/YodaEmbedding/c2c77dc693d11f3734d78489f9a6eea4 AccessModifierOffset: -2 +AlignAfterOpenBracket: AlwaysBreak +AllowAllArgumentsOnNextLine: false AllowShortBlocksOnASingleLine: false AllowShortCaseLabelsOnASingleLine: false AllowShortFunctionsOnASingleLine: Empty @@ -19,9 +21,9 @@ InsertTrailingCommas: Wrapped MaxEmptyLinesToKeep: 1 NamespaceIndentation: All PointerAlignment: Left +ReflowComments: true SeparateDefinitionBlocks: Always SpacesBeforeTrailingComments: 1 -ReflowComments: true # Don't change include blocks, we want to control this manually. # Sorting headers however is allowed as all our headers should be standalone. From 5fbfa1fac967fb35f180a752be434480cff3867e Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Mon, 24 Jul 2023 13:12:23 +0200 Subject: [PATCH 05/20] generate new_arrow_array_builder and start fill_arrow_array_builder --- .../src/codegen/cpp/forward_decl.rs | 1 + .../src/codegen/cpp/method.rs | 4 +- .../re_types_builder/src/codegen/cpp/mod.rs | 481 +++++++++++------- 3 files changed, 287 insertions(+), 199 deletions(-) diff --git a/crates/re_types_builder/src/codegen/cpp/forward_decl.rs b/crates/re_types_builder/src/codegen/cpp/forward_decl.rs index 8498e27c3e1f..6757828c7b14 100644 --- a/crates/re_types_builder/src/codegen/cpp/forward_decl.rs +++ b/crates/re_types_builder/src/codegen/cpp/forward_decl.rs @@ -37,6 +37,7 @@ pub struct ForwardDecls { } impl ForwardDecls { + #[allow(dead_code)] pub fn insert(&mut self, namespace: impl Into, decl: ForwardDecl) { self.declarations_per_namespace .entry(namespace.into()) diff --git a/crates/re_types_builder/src/codegen/cpp/method.rs b/crates/re_types_builder/src/codegen/cpp/method.rs index 917bd24aa897..1f34eaf3526b 100644 --- a/crates/re_types_builder/src/codegen/cpp/method.rs +++ b/crates/re_types_builder/src/codegen/cpp/method.rs @@ -3,7 +3,7 @@ use quote::quote; use crate::Docs; -use super::{doc_comment, quote_docstrings, NEWLINE_TOKEN}; +use super::{quote_doc_comment, quote_docstrings, NEWLINE_TOKEN}; #[derive(Default)] pub struct MethodDeclaration { @@ -82,7 +82,7 @@ impl quote::ToTokens for MethodDocumentation { match self { Self::None => {} Self::String(s) => { - tokens.extend(doc_comment(s)); + tokens.extend(quote_doc_comment(s)); } Self::Docs(docs) => tokens.extend(quote_docstrings(docs)), } diff --git a/crates/re_types_builder/src/codegen/cpp/mod.rs b/crates/re_types_builder/src/codegen/cpp/mod.rs index d651eed5a108..fb7689546467 100644 --- a/crates/re_types_builder/src/codegen/cpp/mod.rs +++ b/crates/re_types_builder/src/codegen/cpp/mod.rs @@ -13,12 +13,13 @@ use quote::{format_ident, quote}; use rayon::prelude::*; use crate::codegen::common::write_file; +use crate::Object; use crate::{ codegen::AUTOGEN_WARNING, ArrowRegistry, Docs, ElementType, ObjectField, ObjectKind, Objects, Type, }; -use self::forward_decl::{ForwardDecl, ForwardDecls}; +use self::forward_decl::ForwardDecls; use self::includes::Includes; use self::method::{Method, MethodDeclaration}; @@ -35,11 +36,11 @@ const SYS_INCLUDE_PATH_PREFIX_TOKEN: &str = "SYS_INCLUDE_PATH_PREFIX_TOKEN"; const SYS_INCLUDE_PATH_SUFFIX_TOKEN: &str = "SYS_INCLUDE_PATH_SUFFIX_TOKEN"; const TODO_TOKEN: &str = "TODO_TOKEN"; -fn comment(text: &str) -> TokenStream { +fn quote_comment(text: &str) -> TokenStream { quote! { #NORMAL_COMMENT_PREFIX_TOKEN #text #NORMAL_COMMENT_SUFFIX_TOKEN } } -fn doc_comment(text: &str) -> TokenStream { +fn quote_doc_comment(text: &str) -> TokenStream { quote! { #DOC_COMMENT_PREFIX_TOKEN #text #DOC_COMMENT_SUFFIX_TOKEN } } @@ -165,7 +166,7 @@ impl crate::CodeGenerator for CppCodeGenerator { fn generate_hpp_cpp( objects: &Objects, arrow_registry: &ArrowRegistry, - obj: &crate::Object, + obj: &Object, ) -> (TokenStream, TokenStream) { let QuotedObject { hpp, cpp } = QuotedObject::new(arrow_registry, objects, obj); let snake_case_name = obj.snake_case_name(); @@ -198,7 +199,7 @@ struct QuotedObject { } impl QuotedObject { - pub fn new(arrow_registry: &ArrowRegistry, objects: &Objects, obj: &crate::Object) -> Self { + pub fn new(arrow_registry: &ArrowRegistry, objects: &Objects, obj: &Object) -> Self { match obj.specifics { crate::ObjectSpecifics::Struct => Self::from_struct(arrow_registry, objects, obj), crate::ObjectSpecifics::Union { .. } => Self::from_union(arrow_registry, objects, obj), @@ -208,7 +209,7 @@ impl QuotedObject { fn from_struct( arrow_registry: &ArrowRegistry, _objects: &Objects, - obj: &crate::Object, + obj: &Object, ) -> QuotedObject { let namespace_ident = format_ident!("{}", obj.kind.plural_snake_case()); // `datatypes`, `components`, or `archetypes` let pascal_case_name = &obj.name; @@ -217,8 +218,9 @@ impl QuotedObject { let mut hpp_includes = Includes::default(); hpp_includes.system.insert("cstdint".to_owned()); // we use `uint32_t` etc everywhere. + hpp_includes.system.insert("arrow/type_fwd.h".to_owned()); // Doing our own forward declarations doesn't get us super far since some arrow types like `FloatBuilder` are type aliases. let mut cpp_includes = Includes::default(); - let mut hpp_declarations = ForwardDecls::default(); + //let mut hpp_declarations = ForwardDecls::default(); let field_declarations = obj .fields @@ -264,19 +266,13 @@ impl QuotedObject { match obj.kind { ObjectKind::Datatype | ObjectKind::Component => { - methods.push(arrow_data_type_method( - &datatype, - &mut hpp_includes, - &mut cpp_includes, - &mut hpp_declarations, - )); - - methods.push(to_arrow_method( + methods.push(arrow_data_type_method(&datatype, &mut cpp_includes)); + methods.push(new_arrow_array_builder_method(&datatype, &mut cpp_includes)); + methods.push(fill_arrow_array_builder_method( &datatype, + obj, &pascal_case_ident, - &mut hpp_includes, &mut cpp_includes, - &mut hpp_declarations, )); } ObjectKind::Archetype => { @@ -295,7 +291,6 @@ impl QuotedObject { }; let hpp = quote! { #hpp_includes - #hpp_declarations namespace rr { namespace #namespace_ident { @@ -322,11 +317,7 @@ impl QuotedObject { Self { hpp, cpp } } - fn from_union( - arrow_registry: &ArrowRegistry, - objects: &Objects, - obj: &crate::Object, - ) -> QuotedObject { + fn from_union(arrow_registry: &ArrowRegistry, objects: &Objects, obj: &Object) -> QuotedObject { // We implement sum-types as tagged unions; // Putting non-POD types in a union requires C++11. // @@ -355,7 +346,7 @@ impl QuotedObject { let data_typename = format_ident!("{pascal_case_name}Data"); let tag_fields = std::iter::once({ - let comment = doc_comment( + let comment = quote_doc_comment( "Having a special empty state makes it possible to implement move-semantics. \ We need to be able to leave the object in a state which we can run the destructor on."); let tag_name = format_ident!("NONE"); @@ -377,7 +368,9 @@ impl QuotedObject { hpp_includes.system.insert("cstdint".to_owned()); // we use `uint32_t` etc everywhere. hpp_includes.system.insert("utility".to_owned()); // std::move hpp_includes.system.insert("cstring".to_owned()); // std::memcpy + hpp_includes.system.insert("arrow/type_fwd.h".to_owned()); // Doing our own forward declarations doesn't get us super far since some arrow types like `FloatBuilder` are type aliases. let mut cpp_includes = Includes::default(); + #[allow(unused)] let mut hpp_declarations = ForwardDecls::default(); let enum_data_declarations = obj @@ -428,19 +421,16 @@ impl QuotedObject { // `enum Angle { Radians(f32), Degrees(f32) };` }; - methods.push(arrow_data_type_method( - &arrow_registry.get(&obj.fqname), - &mut hpp_includes, - &mut cpp_includes, - &mut hpp_declarations, - )); + let datatype = arrow_registry.get(&obj.fqname); + methods.push(arrow_data_type_method(&datatype, &mut cpp_includes)); + methods.push(new_arrow_array_builder_method(&datatype, &mut cpp_includes)); let destructor = if obj.has_default_destructor(objects) { // No destructor needed quote! {} } else { let destructor_match_arms = std::iter::once({ - let comment = comment("Nothing to destroy"); + let comment = quote_comment("Nothing to destroy"); quote! { case detail::#tag_typename::NONE: { break; #comment @@ -452,7 +442,7 @@ impl QuotedObject { let field_ident = format_ident!("{}", crate::to_snake_case(&obj_field.name)); if obj_field.typ.has_default_destructor(objects) { - let comment = comment("has a trivial destructor"); + let comment = quote_comment("has a trivial destructor"); quote! { case detail::#tag_typename::#tag_ident: { break; #comment @@ -498,7 +488,7 @@ impl QuotedObject { } }; - let swap_comment = comment("This bitwise swap would fail for self-referential types, but we don't have any of those."); + let swap_comment = quote_comment("This bitwise swap would fail for self-referential types, but we don't have any of those."); let hpp_methods = methods.iter().map(|m| m.to_hpp_tokens()); let hpp = quote! { @@ -586,14 +576,7 @@ impl QuotedObject { } } -fn arrow_data_type_method( - datatype: &DataType, - hpp_includes: &mut Includes, - cpp_includes: &mut Includes, - hpp_declarations: &mut ForwardDecls, -) -> Method { - hpp_declarations.insert("arrow", ForwardDecl::Class("DataType".to_owned())); - hpp_includes.system.insert("memory".to_owned()); // std::shared_ptr +fn arrow_data_type_method(datatype: &DataType, cpp_includes: &mut Includes) -> Method { cpp_includes.system.insert("arrow/api.h".to_owned()); let quoted_datatype = quote_arrow_data_type(datatype, cpp_includes, true); @@ -610,186 +593,63 @@ fn arrow_data_type_method( } } -fn quote_fill_arrow_builder( - datatype: &DataType, - extension_type_ident: Option<&Ident>, - builder_name: &Ident, - pool: &Ident, - cpp_includes: &mut Includes, -) -> TokenStream { - // TODO: shorter code here for trivial ones? - match datatype { - DataType::Boolean => { - quote!(auto #builder_name = std::make_shared(#pool);) - } - DataType::Int8 => quote!(auto #builder_name = std::make_shared(#pool);), - DataType::Int16 => { - quote!(auto #builder_name = std::make_shared(#pool);) - } - DataType::Int32 => { - quote!(auto #builder_name = std::make_shared(#pool);) - } - DataType::Int64 => { - quote!(auto #builder_name = std::make_shared(#pool);) - } - DataType::UInt8 => { - quote!(auto #builder_name = std::make_shared(#pool);) - } - DataType::UInt16 => { - quote!(auto #builder_name = std::make_shared(#pool);) - } - DataType::UInt32 => { - quote!(auto #builder_name = std::make_shared(#pool);) - } - DataType::UInt64 => { - quote!(auto #builder_name = std::make_shared(#pool);) - } - DataType::Float16 => { - quote!(auto #builder_name = std::make_shared(#pool);) - } - DataType::Float32 => { - quote!(auto #builder_name = std::make_shared(#pool);) - } - DataType::Float64 => { - quote!(auto #builder_name = std::make_shared(#pool);) - } - DataType::Binary => { - quote!(auto #builder_name = std::make_shared(#pool);) - } - DataType::LargeBinary => { - quote!(auto #builder_name = std::make_shared(#pool);) - } - DataType::Utf8 => { - quote!(auto #builder_name = std::make_shared(#pool);) - } - DataType::LargeUtf8 => { - quote!(auto #builder_name = std::make_shared(#pool);) - } - DataType::List(field) => { - // TODO(andreas): Nesting list of lists might require adjusting the name. - let child_builder_name = format_ident!("child_builder"); - let child_builder_impl = quote_fill_arrow_builder( - &field.as_ref().data_type, - None, - &child_builder_name, - pool, - cpp_includes, - ); - quote! { - #child_builder_impl - auto #builder_name = std::make_shared(#pool, #child_builder_name); - } - } - DataType::FixedSizeList(field, length) => { - // TODO(andreas): Nesting list of lists might require adjusting the name. - let child_builder_name = format_ident!("child_builder"); - let child_builder_impl = quote_fill_arrow_builder( - &field.as_ref().data_type, - None, - &child_builder_name, - pool, - cpp_includes, - ); - let quoted_length = quote_integer(length); - quote! { - #child_builder_impl - auto #builder_name = std::make_shared(#pool, #child_builder_name, #quoted_length); - } - } - DataType::Struct(fields) => { - let datatype_ident = extension_type_ident - .context("Structs that aren't wrapped in extension types aren't supported. - This should only happen when using a struct that wasn't generated by the codegenerator itself.") - .unwrap(); - quote! { - auto datatype = #datatype_ident::to_arrow_datatype(); - auto builder = std::make_shared( - datatype, - #pool - ); - } - } - DataType::Union(_, _, mode) => { - let builder_type = match mode { - arrow2::datatypes::UnionMode::Dense => "DenseUnionBuilder", - arrow2::datatypes::UnionMode::Sparse => "SparseUnionBuilder", - }; - let builder_type = format_ident!("{builder_type}"); +fn new_arrow_array_builder_method(datatype: &DataType, cpp_includes: &mut Includes) -> Method { + let arrow_builder_type = arrow_array_builder_type(datatype); - let datatype_ident = extension_type_ident - .context("Structs that aren't wrapped in extension types aren't supported. - This should only happen when using a struct that wasn't generated by the codegenerator itself.") - .unwrap(); - quote! { - datatype = #datatype_ident::to_arrow_datatype(); - auto builder = std::make_shared( - datatype, - #pool, - {} #TODO_TOKEN - ); - } - } - DataType::Extension(fqname, datatype, _metadata) => { - if let DataType::Union(..) = datatype.as_ref() { - // TODO: not yet implemented - return quote! { - #TODO_TOKEN - auto #builder_name = std::make_shared(); - }; - } + cpp_includes.system.insert("arrow/api.h".to_owned()); - let quoted_fqname = quote_fqname_as_type_path(cpp_includes, fqname); - quote! { - ARROW_ASSIGN_OR_RAISE( - auto #builder_name, - #quoted_fqname::to_arrow(#pool, reinterpret_cast(elements), num_elements) - ); + let arrow_builder_type = format_ident!("{arrow_builder_type}"); + let builder_instantiation = + quote_arrow_array_builder_type_instantiation(datatype, cpp_includes, true); + + Method { + docs: "Creates a new array builder with an array of this type.".into(), + declaration: MethodDeclaration { + is_static: true, + return_type: quote! { arrow::Result> }, + name_and_parameters: quote!(new_arrow_array_builder(arrow::MemoryPool * memory_pool)), + }, + definition_body: quote! { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); } - } - _ => unimplemented!("Arrow serialization for type: {:?}", datatype), + #NEWLINE_TOKEN + #NEWLINE_TOKEN + return arrow::Result(#builder_instantiation); + }, + inline: false, } } -fn to_arrow_method( +fn fill_arrow_array_builder_method( datatype: &DataType, + obj: &Object, pascal_case_ident: &Ident, - hpp_includes: &mut Includes, cpp_includes: &mut Includes, - hpp_declarations: &mut ForwardDecls, ) -> Method { - hpp_declarations.insert("arrow", ForwardDecl::Class("ArrayBuilder".to_owned())); - hpp_declarations.insert("arrow", ForwardDecl::Class("MemoryPool".to_owned())); - hpp_includes.system.insert("arrow/result.h".to_owned()); - hpp_includes.system.insert("memory".to_owned()); // std::shared_ptr - cpp_includes.system.insert("arrow/api.h".to_owned()); - let DataType::Extension(_fqname, logical_datatype, _metadata) = datatype else { panic!("Can only generate arrow serialization code for extension types."); }; - let pool = format_ident!("memory_pool"); let builder = format_ident!("builder"); - let builder_initialization = quote_fill_arrow_builder( - logical_datatype, - Some(pascal_case_ident), - &builder, - &pool, - cpp_includes, - ); + let fill_builder = + quote_fill_arrow_array_builder(logical_datatype, &obj.fields, &builder, cpp_includes) + .context(format!("Generating serialization for {}", obj.fqname)) + .unwrap(); Method { - docs: "Fills out an arrow array builder with an array of this type.".into(), + docs: "Fills an arrow array builder with an array of this type.".into(), declaration: MethodDeclaration { is_static: true, return_type: quote! { arrow::Result> }, // TODO(andreas): Pass in validity map. name_and_parameters: quote! { - to_arrow(arrow::MemoryPool* #pool, const #pascal_case_ident* elements, size_t num_elements) + fill_arrow_array_builder(arrow::MemoryPool* memory_pool, const #pascal_case_ident* elements, size_t num_elements) }, }, definition_body: quote! { - if (!#pool) { + if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } if (!elements) { @@ -797,7 +657,9 @@ fn to_arrow_method( } #NEWLINE_TOKEN #NEWLINE_TOKEN - #builder_initialization + ARROW_ASSIGN_OR_RAISE(auto #builder, new_arrow_array_builder(memory_pool)); + ARROW_RETURN_NOT_OK(#builder->Reserve(num_elements)); + #fill_builder #NEWLINE_TOKEN return #builder; }, @@ -805,6 +667,231 @@ fn to_arrow_method( } } +fn quote_fill_arrow_array_builder( + datatype: &DataType, + fields: &[ObjectField], + builder: &Ident, + _cpp_includes: &mut Includes, +) -> anyhow::Result { + let loop_todo = + quote_comment("TODO(andreas): Optimize loops to use batch appends when possible."); + + let tokens = match datatype.to_logical_type() { + DataType::Boolean + | DataType::Int8 + | DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::UInt8 + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 + | DataType::Float16 + | DataType::Float32 + | DataType::Float64 + | DataType::Binary + | DataType::LargeBinary + | DataType::Utf8 + | DataType::LargeUtf8 => { + anyhow::ensure!( + fields.len() == 1, + "Expected exactly one field for primitive type {:?}", + datatype + ); + let field: &ObjectField = &fields[0]; + let field_name = format_ident!("{}", field.name); + + let append_single = if field.is_nullable { + quote! { + if (elements[i].#field_name.has_value()) { + ARROW_RETURN_NOT_OK(#builder->Append(elements[i].#field_name.value())); + } else { + ARROW_RETURN_NOT_OK(#builder->AppendNull()); + } + } + } else { + quote! { + ARROW_RETURN_NOT_OK(#builder->Append(elements[i].#field_name)); + } + }; + + quote! { + #NEWLINE_TOKEN + #loop_todo + for (size_t i = 0; i < num_elements; i += 1) { + #append_single + } + } + } + DataType::FixedSizeList(_field, ..) | DataType::List(_field) => { + let todo = quote_comment("TODO: fill value builder"); + quote! { + auto value_builder = #builder->value_builder(); + #NEWLINE_TOKEN + #loop_todo + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + #todo + ARROW_RETURN_NOT_OK(builder->Append()); + } + } + } + DataType::Struct(_fields) => { + let todo = quote_comment("TODO: fill field builders"); + quote! { + #NEWLINE_TOKEN + #loop_todo + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + #todo + ARROW_RETURN_NOT_OK(builder->Append()); + } + } + } + DataType::Union(_, _, _) => { + let todo = quote_comment("TODO: fill field builders"); + quote! { + #NEWLINE_TOKEN + #loop_todo + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + #todo + } + } + } + DataType::Extension(..) => { + unreachable!("Logical type can't be an extension type.") + } + _ => anyhow::bail!( + "Arrow serialization for type {:?} not implemented", + datatype + ), + }; + + Ok(tokens) +} + +fn arrow_array_builder_type(datatype: &DataType) -> &'static str { + match datatype.to_logical_type() { + DataType::Boolean => "BooleanBuilder", + DataType::Int8 => "Int8Builder", + DataType::Int16 => "Int16Builder", + DataType::Int32 => "Int32Builder", + DataType::Int64 => "Int64Builder", + DataType::UInt8 => "UInt8Builder", + DataType::UInt16 => "UInt16Builder", + DataType::UInt32 => "UInt32Builder", + DataType::UInt64 => "UInt64Builder", + DataType::Float16 => "HalfFloatBuilder", + DataType::Float32 => "FloatBuilder", + DataType::Float64 => "DoubleBuilder", + DataType::Binary => "BinaryBuilder", + DataType::LargeBinary => "LargeBinaryBuilder", + DataType::Utf8 => "StringBuilder", + DataType::LargeUtf8 => "LargeStringBuilder", + DataType::FixedSizeList(..) => "FixedSizeListBuilder", + DataType::List(..) => "ListBuilder", + DataType::Struct(..) => "StructBuilder", + DataType::Null => "NullBuilder", + DataType::Union(_, _, mode) => match mode { + arrow2::datatypes::UnionMode::Dense => "DenseUnionBuilder", + arrow2::datatypes::UnionMode::Sparse => "SparseUnionBuilder", + }, + DataType::Extension(_, _, _metadata) => { + unreachable!("Logical type can't be an extension type.") + } + _ => unimplemented!( + "Arrow serialization for type {:?} not implemented", + datatype + ), + } +} + +fn quote_arrow_array_builder_type_instantiation( + datatype: &DataType, + cpp_includes: &mut Includes, + is_top_level_type: bool, +) -> TokenStream { + let builder_type = arrow_array_builder_type(datatype); + let builder_type = format_ident!("{builder_type}"); + + match datatype { + DataType::Boolean + | DataType::Int8 + | DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::UInt8 + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 + | DataType::Float16 + | DataType::Float32 + | DataType::Float64 + | DataType::Binary + | DataType::LargeBinary + | DataType::Utf8 + | DataType::LargeUtf8 + | DataType::Null => { + quote!(std::make_shared(memory_pool)) + } + DataType::List(field) => { + let element_builder = quote_arrow_array_builder_type_instantiation( + field.data_type(), + cpp_includes, + false, + ); + quote!(std::make_shared(memory_pool, #element_builder)) + } + DataType::FixedSizeList(field, length) => { + let quoted_length = quote_integer(length); + let element_builder = quote_arrow_array_builder_type_instantiation( + field.data_type(), + cpp_includes, + false, + ); + quote!(std::make_shared(memory_pool, #element_builder, #quoted_length)) + } + DataType::Struct(fields) => { + let field_builders = fields.iter().map(|field| { + quote_arrow_array_builder_type_instantiation(field.data_type(), cpp_includes, false) + }); + quote! { + std::make_shared( + to_arrow_datatype(), + memory_pool, + std::vector>({ #(#field_builders,)* }) + ) + } + } + DataType::Union(fields, _, _) => { + let field_builders = fields.iter().map(|field| { + quote_arrow_array_builder_type_instantiation(field.data_type(), cpp_includes, false) + }); + quote! { + std::make_shared( + memory_pool, + std::vector>({ #(#field_builders,)* }), + to_arrow_datatype() + ) + } + } + DataType::Extension(fqname, datatype, _metadata) => { + if is_top_level_type { + quote_arrow_array_builder_type_instantiation(datatype.as_ref(), cpp_includes, false) + } else { + // Propagating error here is hard since we're in a nested context. + // But also not that important since we *know* that this only fails for null pools and we already checked that now. + let quoted_fqname = quote_fqname_as_type_path(cpp_includes, fqname); + quote! { + #quoted_fqname::new_arrow_array_builder(memory_pool).ValueOrDie() + } + } + } + _ => unimplemented!( + "Arrow serialization for type {:?} not implemented", + datatype + ), + } +} + /// e.g. `static Angle radians(float radians);` -> `auto angle = Angle::radians(radians);` fn static_constructor_for_enum_type( objects: &Objects, @@ -1033,7 +1120,7 @@ fn quote_fqname_as_type_path(includes: &mut Includes, fqname: &str) -> TokenStre fn quote_docstrings(docs: &Docs) -> TokenStream { let lines = crate::codegen::get_documentation(docs, &["cpp", "c++"]); - let quoted_lines = lines.iter().map(|docstring| doc_comment(docstring)); + let quoted_lines = lines.iter().map(|docstring| quote_doc_comment(docstring)); quote! { #NEWLINE_TOKEN #(#quoted_lines)* From 7a3f06fc95633eec78f13fc7ebb33a391875fce2 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Mon, 24 Jul 2023 13:16:42 +0200 Subject: [PATCH 06/20] nicer formatting for large function calls - put closing bracket in block alignment --- .clang-format | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.clang-format b/.clang-format index 8c45504e06a9..23ca7b00fcd8 100644 --- a/.clang-format +++ b/.clang-format @@ -3,7 +3,7 @@ BasedOnStyle: Google # Make it slightly more similar to Rust. # Based loosely on https://gist.github.com/YodaEmbedding/c2c77dc693d11f3734d78489f9a6eea4 AccessModifierOffset: -2 -AlignAfterOpenBracket: AlwaysBreak +AlignAfterOpenBracket: BlockIndent AllowAllArgumentsOnNextLine: false AllowShortBlocksOnASingleLine: false AllowShortCaseLabelsOnASingleLine: false From 588576b1e6401a0ea19ce31e3154b34f68cf668c Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Mon, 24 Jul 2023 17:20:39 +0200 Subject: [PATCH 07/20] fill_arrow_array_builder has now very basic struct handling --- .../re_types_builder/src/codegen/cpp/mod.rs | 140 +++++++++++++----- crates/rerun_c/src/rerun.h | 5 +- 2 files changed, 105 insertions(+), 40 deletions(-) diff --git a/crates/re_types_builder/src/codegen/cpp/mod.rs b/crates/re_types_builder/src/codegen/cpp/mod.rs index fb7689546467..b17c243c2021 100644 --- a/crates/re_types_builder/src/codegen/cpp/mod.rs +++ b/crates/re_types_builder/src/codegen/cpp/mod.rs @@ -595,9 +595,6 @@ fn arrow_data_type_method(datatype: &DataType, cpp_includes: &mut Includes) -> M fn new_arrow_array_builder_method(datatype: &DataType, cpp_includes: &mut Includes) -> Method { let arrow_builder_type = arrow_array_builder_type(datatype); - - cpp_includes.system.insert("arrow/api.h".to_owned()); - let arrow_builder_type = format_ident!("{arrow_builder_type}"); let builder_instantiation = quote_arrow_array_builder_type_instantiation(datatype, cpp_includes, true); @@ -632,51 +629,58 @@ fn fill_arrow_array_builder_method( }; let builder = format_ident!("builder"); + let arrow_builder_type = arrow_array_builder_type(datatype); + let arrow_builder_type = format_ident!("{arrow_builder_type}"); - let fill_builder = - quote_fill_arrow_array_builder(logical_datatype, &obj.fields, &builder, cpp_includes) - .context(format!("Generating serialization for {}", obj.fqname)) - .unwrap(); + let fill_builder = quote_fill_arrow_array_builder( + pascal_case_ident, + logical_datatype, + &obj.fields, + &builder, + cpp_includes, + ) + .context(format!("Generating serialization for {}", obj.fqname)) + .unwrap(); Method { docs: "Fills an arrow array builder with an array of this type.".into(), declaration: MethodDeclaration { is_static: true, - return_type: quote! { arrow::Result> }, + return_type: quote! { arrow::Status }, // TODO(andreas): Pass in validity map. name_and_parameters: quote! { - fill_arrow_array_builder(arrow::MemoryPool* memory_pool, const #pascal_case_ident* elements, size_t num_elements) + fill_arrow_array_builder(arrow::#arrow_builder_type* #builder, const #pascal_case_ident* elements, size_t num_elements) }, }, definition_body: quote! { - if (!memory_pool) { - return arrow::Status::Invalid("Memory pool is null."); + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } #NEWLINE_TOKEN #NEWLINE_TOKEN - ARROW_ASSIGN_OR_RAISE(auto #builder, new_arrow_array_builder(memory_pool)); - ARROW_RETURN_NOT_OK(#builder->Reserve(num_elements)); #fill_builder #NEWLINE_TOKEN - return #builder; + #NEWLINE_TOKEN + return arrow::Status::OK(); }, inline: false, } } fn quote_fill_arrow_array_builder( + pascal_case_ident: &Ident, datatype: &DataType, fields: &[ObjectField], builder: &Ident, - _cpp_includes: &mut Includes, + cpp_includes: &mut Includes, ) -> anyhow::Result { let loop_todo = quote_comment("TODO(andreas): Optimize loops to use batch appends when possible."); - let tokens = match datatype.to_logical_type() { + let tokens = match datatype { DataType::Boolean | DataType::Int8 | DataType::Int16 @@ -698,27 +702,14 @@ fn quote_fill_arrow_array_builder( "Expected exactly one field for primitive type {:?}", datatype ); - let field: &ObjectField = &fields[0]; - let field_name = format_ident!("{}", field.name); - - let append_single = if field.is_nullable { - quote! { - if (elements[i].#field_name.has_value()) { - ARROW_RETURN_NOT_OK(#builder->Append(elements[i].#field_name.value())); - } else { - ARROW_RETURN_NOT_OK(#builder->AppendNull()); - } - } - } else { - quote! { - ARROW_RETURN_NOT_OK(#builder->Append(elements[i].#field_name)); - } - }; + let append_single = quote_append_field_to_builder(&fields[0], builder); quote! { #NEWLINE_TOKEN #loop_todo - for (size_t i = 0; i < num_elements; i += 1) { + ARROW_RETURN_NOT_OK(#builder->Reserve(num_elements)); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto& element = elements[elem_idx]; #append_single } } @@ -729,19 +720,58 @@ fn quote_fill_arrow_array_builder( auto value_builder = #builder->value_builder(); #NEWLINE_TOKEN #loop_todo + ARROW_RETURN_NOT_OK(value_builder->Reserve(num_elements)); for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto& element = elements[elem_idx]; #todo ARROW_RETURN_NOT_OK(builder->Append()); } } } - DataType::Struct(_fields) => { - let todo = quote_comment("TODO: fill field builders"); + DataType::Struct(field_datatypes) => { + let fill_fields = fields.iter().zip(field_datatypes).enumerate().map( + |(i, (field, arrow_field))| { + let builder_index = quote_integer(i); + match arrow_field.data_type() { + DataType::FixedSizeList(..) | DataType::List(..) => { + quote!( + return arrow::Status::NotImplemented( + "TODO(andreas): lists in structs are not yet supported" + ); + ) + } + DataType::Extension(_fqname, _, _) => { + quote!( + return arrow::Status::NotImplemented( + "TODO(andreas): extensions in structs are not yet supported" + ); + ) + } + _ => { + let element_builder = format_ident!("element_builder"); + let element_builder_type = arrow_array_builder_type(arrow_field.data_type()); + let element_builder_type = format_ident!("{element_builder_type}"); + let field_append = quote_append_field_to_builder(field, &element_builder); + quote! { + { + auto #element_builder = static_cast(builder->field_builder(#builder_index)); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto& element = elements[elem_idx]; + #field_append + } + #NEWLINE_TOKEN + } + } + } + } + }, + ); + quote! { + #(#fill_fields)* #NEWLINE_TOKEN #loop_todo for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { - #todo ARROW_RETURN_NOT_OK(builder->Append()); } } @@ -752,12 +782,28 @@ fn quote_fill_arrow_array_builder( #NEWLINE_TOKEN #loop_todo for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto& element = elements[elem_idx]; #todo } } } - DataType::Extension(..) => { - unreachable!("Logical type can't be an extension type.") + DataType::Extension(fqname, datatype, _) => { + assert_eq!(fields.len(), 1); + if let DataType::Union(..) = datatype.as_ref() { + quote!(return arrow::Status::NotImplemented("TODO(andreas): Implement fill_arrow_array_builder for unions");) + } else if fields[0].is_nullable { + // Idea: pass in a tagged union for both optional and non-optional arrays to all fill_arrow_array_builder methods? + quote!(return arrow::Status::NotImplemented(("TODO(andreas) Handle nullable extensions"));) + } else { + let quoted_fqname = quote_fqname_as_type_path(cpp_includes, fqname); + // TODO: add a static assertion on size. + quote! { + static_assert(sizeof(#quoted_fqname) == sizeof(#pascal_case_ident)); + ARROW_RETURN_NOT_OK(#quoted_fqname::fill_arrow_array_builder( + builder, reinterpret_cast(elements), num_elements + )); + } + } } _ => anyhow::bail!( "Arrow serialization for type {:?} not implemented", @@ -768,6 +814,24 @@ fn quote_fill_arrow_array_builder( Ok(tokens) } +fn quote_append_field_to_builder(field: &ObjectField, builder: &Ident) -> TokenStream { + // TODO: Deal with lists, structs, unions, etc.? + let field_name = format_ident!("{}", field.name); + if field.is_nullable { + quote! { + if (element.#field_name.has_value()) { + ARROW_RETURN_NOT_OK(#builder->Append(element.#field_name.value())); + } else { + ARROW_RETURN_NOT_OK(#builder->AppendNull()); + } + } + } else { + quote! { + ARROW_RETURN_NOT_OK(#builder->Append(element.#field_name)); + } + } +} + fn arrow_array_builder_type(datatype: &DataType) -> &'static str { match datatype.to_logical_type() { DataType::Boolean => "BooleanBuilder", diff --git a/crates/rerun_c/src/rerun.h b/crates/rerun_c/src/rerun.h index 6ee2220e6394..c7efa3040265 100644 --- a/crates/rerun_c/src/rerun.h +++ b/crates/rerun_c/src/rerun.h @@ -82,8 +82,9 @@ extern const char* rr_version_string(void); /// Usually you only have one recording stream, so you can call /// `rr_recording_stream_new` once, ignore its return value, and use /// `RERUN_REC_STREAM_DEFAULT` everywhere in your code. -extern rr_recording_stream rr_recording_stream_new(const struct rr_store_info* store_info, - const char* tcp_addr); +extern rr_recording_stream rr_recording_stream_new( + const struct rr_store_info* store_info, const char* tcp_addr +); /// Free the given recording stream. The handle will be invalid after this. extern void rr_recording_stream_free(rr_recording_stream stream); From 98fc0e799138fbea2c4ea5acea6e8cc96b7d421c Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Mon, 24 Jul 2023 17:28:58 +0200 Subject: [PATCH 08/20] use generated serialization in points2 & points3 example methods --- examples/cpp/minimal/main.cpp | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/examples/cpp/minimal/main.cpp b/examples/cpp/minimal/main.cpp index 14c3d57fb145..d742c9474108 100644 --- a/examples/cpp/minimal/main.cpp +++ b/examples/cpp/minimal/main.cpp @@ -6,32 +6,19 @@ arrow::Result> points2(size_t num_points, const float* xy) { arrow::MemoryPool* pool = arrow::default_memory_pool(); - auto x_builder = std::make_shared(pool); - auto y_builder = std::make_shared(pool); - auto nullable = false; - - auto data_type = rr::components::Point2D::to_arrow_datatype(); - auto struct_builder = - arrow::StructBuilder(data_type, pool, {x_builder, y_builder}); - - for (size_t i = 0; i < num_points; ++i) { - ARROW_RETURN_NOT_OK(struct_builder.Append()); - ARROW_RETURN_NOT_OK(x_builder->Append(xy[2 * i + 0])); - ARROW_RETURN_NOT_OK(y_builder->Append(xy[2 * i + 1])); -} + ARROW_ASSIGN_OR_RAISE(auto builder, rr::components::Point2D::new_arrow_array_builder(pool)); + rr::components::Point2D::fill_arrow_array_builder(builder.get(), (rr::components::Point2D*)xy, num_points); std::shared_ptr array; - ARROW_RETURN_NOT_OK(struct_builder.Finish(&array)); + ARROW_RETURN_NOT_OK(builder->Finish(&array)); auto name = "points"; // Unused, but should be the name of the field in the archetype - auto schema = arrow::schema({arrow::field(name, data_type, nullable)}); + auto schema = arrow::schema({arrow::field(name, rr::components::Point2D::to_arrow_datatype(), false)}); return arrow::Table::Make(schema, {array}); } - - int main(int argc, char** argv) { loguru::g_preamble_uptime = false; loguru::g_preamble_thread = false; From 08fe5c0e2c6b00d60e2a38242ae88a9ee880a582 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Mon, 24 Jul 2023 17:35:30 +0200 Subject: [PATCH 09/20] stub union implementation --- .../re_types_builder/src/codegen/cpp/mod.rs | 20 ++++++++----- examples/cpp/minimal/main.cpp | 10 +++++-- rerun_cpp/src/rerun.cpp | 28 +++++++------------ 3 files changed, 30 insertions(+), 28 deletions(-) diff --git a/crates/re_types_builder/src/codegen/cpp/mod.rs b/crates/re_types_builder/src/codegen/cpp/mod.rs index b17c243c2021..168579ff5492 100644 --- a/crates/re_types_builder/src/codegen/cpp/mod.rs +++ b/crates/re_types_builder/src/codegen/cpp/mod.rs @@ -424,6 +424,12 @@ impl QuotedObject { let datatype = arrow_registry.get(&obj.fqname); methods.push(arrow_data_type_method(&datatype, &mut cpp_includes)); methods.push(new_arrow_array_builder_method(&datatype, &mut cpp_includes)); + methods.push(fill_arrow_array_builder_method( + &datatype, + obj, + &pascal_case_ident, + &mut cpp_includes, + )); let destructor = if obj.has_default_destructor(objects) { // No destructor needed @@ -715,7 +721,6 @@ fn quote_fill_arrow_array_builder( } } DataType::FixedSizeList(_field, ..) | DataType::List(_field) => { - let todo = quote_comment("TODO: fill value builder"); quote! { auto value_builder = #builder->value_builder(); #NEWLINE_TOKEN @@ -723,7 +728,9 @@ fn quote_fill_arrow_array_builder( ARROW_RETURN_NOT_OK(value_builder->Reserve(num_elements)); for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { const auto& element = elements[elem_idx]; - #todo + return arrow::Status::NotImplemented( + "TODO(andreas): fill value builder" + ); ARROW_RETURN_NOT_OK(builder->Append()); } } @@ -777,21 +784,20 @@ fn quote_fill_arrow_array_builder( } } DataType::Union(_, _, _) => { - let todo = quote_comment("TODO: fill field builders"); quote! { #NEWLINE_TOKEN #loop_todo for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { const auto& element = elements[elem_idx]; - #todo } + return arrow::Status::NotImplemented( + "TODO(andreas): unions are not yet implemented" + ); } } DataType::Extension(fqname, datatype, _) => { assert_eq!(fields.len(), 1); - if let DataType::Union(..) = datatype.as_ref() { - quote!(return arrow::Status::NotImplemented("TODO(andreas): Implement fill_arrow_array_builder for unions");) - } else if fields[0].is_nullable { + if fields[0].is_nullable { // Idea: pass in a tagged union for both optional and non-optional arrays to all fill_arrow_array_builder methods? quote!(return arrow::Status::NotImplemented(("TODO(andreas) Handle nullable extensions"));) } else { diff --git a/examples/cpp/minimal/main.cpp b/examples/cpp/minimal/main.cpp index d742c9474108..8a09e31d4ce9 100644 --- a/examples/cpp/minimal/main.cpp +++ b/examples/cpp/minimal/main.cpp @@ -6,15 +6,19 @@ arrow::Result> points2(size_t num_points, const float* xy) { arrow::MemoryPool* pool = arrow::default_memory_pool(); - ARROW_ASSIGN_OR_RAISE(auto builder, rr::components::Point2D::new_arrow_array_builder(pool)); - rr::components::Point2D::fill_arrow_array_builder(builder.get(), (rr::components::Point2D*)xy, num_points); + ARROW_RETURN_NOT_OK(rr::components::Point2D::fill_arrow_array_builder( + builder.get(), + (rr::components::Point2D*)xy, + num_points + )); std::shared_ptr array; ARROW_RETURN_NOT_OK(builder->Finish(&array)); auto name = "points"; // Unused, but should be the name of the field in the archetype - auto schema = arrow::schema({arrow::field(name, rr::components::Point2D::to_arrow_datatype(), false)}); + auto schema = + arrow::schema({arrow::field(name, rr::components::Point2D::to_arrow_datatype(), false)}); return arrow::Table::Make(schema, {array}); } diff --git a/rerun_cpp/src/rerun.cpp b/rerun_cpp/src/rerun.cpp index bf02a7bd9108..0136925b4799 100644 --- a/rerun_cpp/src/rerun.cpp +++ b/rerun_cpp/src/rerun.cpp @@ -17,30 +17,22 @@ namespace rr { arrow::Result> points3(size_t num_points, const float* xyz) { arrow::MemoryPool* pool = arrow::default_memory_pool(); - auto x_builder = std::make_shared(pool); - auto y_builder = std::make_shared(pool); - auto z_builder = std::make_shared(pool); - auto nullable = false; - auto data_type = arrow::struct_({field("x", arrow::float32(), nullable), - field("y", arrow::float32(), nullable), - field("z", arrow::float32(), nullable)}); - auto struct_builder = - arrow::StructBuilder(data_type, pool, {x_builder, y_builder, z_builder}); - - for (size_t i = 0; i < num_points; ++i) { - ARROW_RETURN_NOT_OK(struct_builder.Append()); - ARROW_RETURN_NOT_OK(x_builder->Append(xyz[3 * i + 0])); - ARROW_RETURN_NOT_OK(y_builder->Append(xyz[3 * i + 1])); - ARROW_RETURN_NOT_OK(z_builder->Append(xyz[3 * i + 2])); - } + ARROW_ASSIGN_OR_RAISE(auto builder, rr::components::Point3D::new_arrow_array_builder(pool)); + ARROW_RETURN_NOT_OK(rr::components::Point3D::fill_arrow_array_builder( + builder.get(), + (rr::components::Point3D*)xyz, + num_points + )); std::shared_ptr array; - ARROW_RETURN_NOT_OK(struct_builder.Finish(&array)); + ARROW_RETURN_NOT_OK(builder->Finish(&array)); auto name = "points"; // Unused, but should be the name of the field in the archetype - auto schema = arrow::schema({arrow::field(name, data_type, nullable)}); + auto schema = arrow::schema( + {arrow::field(name, rr::components::Point3D::to_arrow_datatype(), nullable)} + ); return arrow::Table::Make(schema, {array}); } From a138716710a1efbcc952f032ab85f04f6153cb08 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Mon, 24 Jul 2023 17:53:42 +0200 Subject: [PATCH 10/20] comment on arrow/type_fwd being problematic --- crates/re_types_builder/src/codegen/cpp/mod.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/crates/re_types_builder/src/codegen/cpp/mod.rs b/crates/re_types_builder/src/codegen/cpp/mod.rs index 168579ff5492..c8fa616d5477 100644 --- a/crates/re_types_builder/src/codegen/cpp/mod.rs +++ b/crates/re_types_builder/src/codegen/cpp/mod.rs @@ -218,9 +218,13 @@ impl QuotedObject { let mut hpp_includes = Includes::default(); hpp_includes.system.insert("cstdint".to_owned()); // we use `uint32_t` etc everywhere. - hpp_includes.system.insert("arrow/type_fwd.h".to_owned()); // Doing our own forward declarations doesn't get us super far since some arrow types like `FloatBuilder` are type aliases. + + // Doing our own forward declarations doesn't get us super far since some arrow types like `FloatBuilder` are type aliases. + // TODO(andreas): This drags in arrow headers into the public api though. We probably should try harder with forward declarations. + hpp_includes.system.insert("arrow/type_fwd.h".to_owned()); let mut cpp_includes = Includes::default(); - //let mut hpp_declarations = ForwardDecls::default(); + #[allow(unused)] + let mut hpp_declarations = ForwardDecls::default(); let field_declarations = obj .fields @@ -292,6 +296,8 @@ impl QuotedObject { let hpp = quote! { #hpp_includes + #hpp_declarations + namespace rr { namespace #namespace_ident { #quoted_docs @@ -368,7 +374,11 @@ impl QuotedObject { hpp_includes.system.insert("cstdint".to_owned()); // we use `uint32_t` etc everywhere. hpp_includes.system.insert("utility".to_owned()); // std::move hpp_includes.system.insert("cstring".to_owned()); // std::memcpy - hpp_includes.system.insert("arrow/type_fwd.h".to_owned()); // Doing our own forward declarations doesn't get us super far since some arrow types like `FloatBuilder` are type aliases. + + // Doing our own forward declarations doesn't get us super far since some arrow types like `FloatBuilder` are type aliases. + // TODO(andreas): This drags in arrow headers into the public api though. We probably should try harder with f + hpp_includes.system.insert("arrow/type_fwd.h".to_owned()); + let mut cpp_includes = Includes::default(); #[allow(unused)] let mut hpp_declarations = ForwardDecls::default(); From b0f6163c95d98a264cb216fe9a100458637a4231 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Tue, 25 Jul 2023 16:32:56 +0200 Subject: [PATCH 11/20] code formatting --- rerun_cpp/src/recording_stream.cpp | 6 ++++-- rerun_cpp/src/recording_stream.hpp | 11 +++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/rerun_cpp/src/recording_stream.cpp b/rerun_cpp/src/recording_stream.cpp index b5a2ebfcccc5..74b40c0c41d4 100644 --- a/rerun_cpp/src/recording_stream.cpp +++ b/rerun_cpp/src/recording_stream.cpp @@ -40,8 +40,10 @@ namespace rr { return s_global; } - void RecordingStream::log_data_row(const char* entity_path, uint32_t num_instances, - size_t num_data_cells, const DataCell* data_cells) { + void RecordingStream::log_data_row( + const char* entity_path, uint32_t num_instances, size_t num_data_cells, + const DataCell* data_cells + ) { // Map to C API: std::vector c_data_cells; c_data_cells.reserve(num_data_cells); diff --git a/rerun_cpp/src/recording_stream.hpp b/rerun_cpp/src/recording_stream.hpp index 991c04deb96b..30262b4f2d59 100644 --- a/rerun_cpp/src/recording_stream.hpp +++ b/rerun_cpp/src/recording_stream.hpp @@ -17,8 +17,9 @@ namespace rr { class RecordingStream { public: - RecordingStream(const char* app_id, const char* addr, - StoreKind store_kind = StoreKind::Recording); + RecordingStream( + const char* app_id, const char* addr, StoreKind store_kind = StoreKind::Recording + ); ~RecordingStream(); /// Must be called first, if at all. @@ -32,8 +33,10 @@ namespace rr { /// /// I.e. logs a number of components arrays (each with a same number of instances) to a /// single entity path. - void log_data_row(const char* entity_path, uint32_t num_instances, size_t num_data_cells, - const DataCell* data_cells); + void log_data_row( + const char* entity_path, uint32_t num_instances, size_t num_data_cells, + const DataCell* data_cells + ); private: RecordingStream() : _id{0} {} From d98b2245e0cd1dafade7dba203b5e7539ed5101d Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Tue, 25 Jul 2023 16:59:39 +0200 Subject: [PATCH 12/20] trivial batch-passing optimization for fully transparent primitive types --- .../re_types_builder/src/codegen/cpp/mod.rs | 68 ++++++++++--------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/crates/re_types_builder/src/codegen/cpp/mod.rs b/crates/re_types_builder/src/codegen/cpp/mod.rs index c8fa616d5477..4d6988117d11 100644 --- a/crates/re_types_builder/src/codegen/cpp/mod.rs +++ b/crates/re_types_builder/src/codegen/cpp/mod.rs @@ -693,9 +693,6 @@ fn quote_fill_arrow_array_builder( builder: &Ident, cpp_includes: &mut Includes, ) -> anyhow::Result { - let loop_todo = - quote_comment("TODO(andreas): Optimize loops to use batch appends when possible."); - let tokens = match datatype { DataType::Boolean | DataType::Int8 @@ -718,23 +715,13 @@ fn quote_fill_arrow_array_builder( "Expected exactly one field for primitive type {:?}", datatype ); - let append_single = quote_append_field_to_builder(&fields[0], builder); - - quote! { - #NEWLINE_TOKEN - #loop_todo - ARROW_RETURN_NOT_OK(#builder->Reserve(num_elements)); - for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { - const auto& element = elements[elem_idx]; - #append_single - } - } + let field = &fields[0]; + quote_append_elements_to_builder(field, builder, true) } DataType::FixedSizeList(_field, ..) | DataType::List(_field) => { quote! { auto value_builder = #builder->value_builder(); #NEWLINE_TOKEN - #loop_todo ARROW_RETURN_NOT_OK(value_builder->Reserve(num_elements)); for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { const auto& element = elements[elem_idx]; @@ -768,15 +755,11 @@ fn quote_fill_arrow_array_builder( let element_builder = format_ident!("element_builder"); let element_builder_type = arrow_array_builder_type(arrow_field.data_type()); let element_builder_type = format_ident!("{element_builder_type}"); - let field_append = quote_append_field_to_builder(field, &element_builder); + let field_append = quote_append_elements_to_builder(field, &element_builder, false); quote! { { auto #element_builder = static_cast(builder->field_builder(#builder_index)); - for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { - const auto& element = elements[elem_idx]; - #field_append - } - #NEWLINE_TOKEN + #field_append } } } @@ -787,7 +770,6 @@ fn quote_fill_arrow_array_builder( quote! { #(#fill_fields)* #NEWLINE_TOKEN - #loop_todo for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { ARROW_RETURN_NOT_OK(builder->Append()); } @@ -796,7 +778,6 @@ fn quote_fill_arrow_array_builder( DataType::Union(_, _, _) => { quote! { #NEWLINE_TOKEN - #loop_todo for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { const auto& element = elements[elem_idx]; } @@ -805,14 +786,13 @@ fn quote_fill_arrow_array_builder( ); } } - DataType::Extension(fqname, datatype, _) => { + DataType::Extension(fqname, _datatype, _) => { assert_eq!(fields.len(), 1); if fields[0].is_nullable { // Idea: pass in a tagged union for both optional and non-optional arrays to all fill_arrow_array_builder methods? quote!(return arrow::Status::NotImplemented(("TODO(andreas) Handle nullable extensions"));) } else { let quoted_fqname = quote_fqname_as_type_path(cpp_includes, fqname); - // TODO: add a static assertion on size. quote! { static_assert(sizeof(#quoted_fqname) == sizeof(#pascal_case_ident)); ARROW_RETURN_NOT_OK(#quoted_fqname::fill_arrow_array_builder( @@ -830,20 +810,42 @@ fn quote_fill_arrow_array_builder( Ok(tokens) } -fn quote_append_field_to_builder(field: &ObjectField, builder: &Ident) -> TokenStream { - // TODO: Deal with lists, structs, unions, etc.? +fn quote_append_elements_to_builder( + field: &ObjectField, + builder: &Ident, + is_transparent: bool, +) -> TokenStream { let field_name = format_ident!("{}", field.name); if field.is_nullable { quote! { - if (element.#field_name.has_value()) { - ARROW_RETURN_NOT_OK(#builder->Append(element.#field_name.value())); - } else { - ARROW_RETURN_NOT_OK(#builder->AppendNull()); + ARROW_RETURN_NOT_OK(#builder->Reserve(num_elements)); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto& element = elements[elem_idx]; + if (element.#field_name.has_value()) { + ARROW_RETURN_NOT_OK(#builder->Append(element.#field_name.value())); + } else { + ARROW_RETURN_NOT_OK(#builder->AppendNull()); + } } } + } else if is_transparent + && !matches!( + &field.typ, + Type::String | Type::Vector { .. } | Type::Object(..) | Type::Array { .. } + ) + { + // Trivial optimization: If this is the only field of this type and it's a trivial field (not array/string/blob), + // we can just pass the whole array as-is! + quote! { + static_assert(sizeof(*elements) == sizeof(elements->#field_name)); + ARROW_RETURN_NOT_OK(#builder->AppendValues(&elements->#field_name, num_elements)); + } } else { quote! { - ARROW_RETURN_NOT_OK(#builder->Append(element.#field_name)); + ARROW_RETURN_NOT_OK(#builder->Reserve(num_elements)); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + ARROW_RETURN_NOT_OK(#builder->Append(elements[elem_idx].#field_name)); + } } } } @@ -1279,7 +1281,7 @@ fn quote_arrow_data_type( // In the future we'll add the extension type here to the schema. quote_arrow_data_type(datatype, includes, false) } else { - // TODO: remove unnecessary namespacing. + // TODO(andreas): remove unnecessary namespacing. let quoted_fqname = quote_fqname_as_type_path(includes, fqname); quote! { #quoted_fqname::to_arrow_datatype() } } From c4b3b9411981838575f1b65d419eb901f8f41467 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Tue, 25 Jul 2023 17:04:16 +0200 Subject: [PATCH 13/20] use fast path for struct append --- crates/re_types_builder/src/codegen/cpp/mod.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/re_types_builder/src/codegen/cpp/mod.rs b/crates/re_types_builder/src/codegen/cpp/mod.rs index 4d6988117d11..d5accb1ba128 100644 --- a/crates/re_types_builder/src/codegen/cpp/mod.rs +++ b/crates/re_types_builder/src/codegen/cpp/mod.rs @@ -770,9 +770,7 @@ fn quote_fill_arrow_array_builder( quote! { #(#fill_fields)* #NEWLINE_TOKEN - for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { - ARROW_RETURN_NOT_OK(builder->Append()); - } + ARROW_RETURN_NOT_OK(builder->AppendValues(num_elements, nullptr)); } } DataType::Union(_, _, _) => { From baed7c655e786f45d0c273dad729bf31761fed58 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Tue, 25 Jul 2023 17:49:41 +0200 Subject: [PATCH 14/20] very basic fixed size list support --- .../re_types_builder/src/codegen/cpp/mod.rs | 71 +++++++++++++++---- 1 file changed, 59 insertions(+), 12 deletions(-) diff --git a/crates/re_types_builder/src/codegen/cpp/mod.rs b/crates/re_types_builder/src/codegen/cpp/mod.rs index d5accb1ba128..fa8a0b8945dc 100644 --- a/crates/re_types_builder/src/codegen/cpp/mod.rs +++ b/crates/re_types_builder/src/codegen/cpp/mod.rs @@ -718,20 +718,67 @@ fn quote_fill_arrow_array_builder( let field = &fields[0]; quote_append_elements_to_builder(field, builder, true) } - DataType::FixedSizeList(_field, ..) | DataType::List(_field) => { - quote! { - auto value_builder = #builder->value_builder(); - #NEWLINE_TOKEN - ARROW_RETURN_NOT_OK(value_builder->Reserve(num_elements)); - for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { - const auto& element = elements[elem_idx]; - return arrow::Status::NotImplemented( - "TODO(andreas): fill value builder" - ); - ARROW_RETURN_NOT_OK(builder->Append()); + DataType::FixedSizeList(field, length) => { + anyhow::ensure!( + fields.len() == 1, + "Expected exactly one field for list type {:?}", + datatype + ); + match field.data_type() { + DataType::Boolean + | DataType::Int8 + | DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::UInt8 + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 + | DataType::Float16 + | DataType::Float32 + | DataType::Float64 => {} + _ => anyhow::bail!("Unimplemented field type for fixed size list: {:?}", field), + }; + + let value_builder_type = arrow_array_builder_type(field.data_type()); + let value_builder_type = format_ident!("{value_builder_type}"); + let field_name = format_ident!("{}", &fields[0].name); + let length = quote_integer(length); + + if field.is_nullable { + quote! { + auto value_builder = static_cast(#builder->value_builder()); + ARROW_RETURN_NOT_OK(#builder->Reserve(num_elements * #length)); + ARROW_RETURN_NOT_OK(value_builder->Reserve(num_elements * #length)); + #NEWLINE_TOKEN #NEWLINE_TOKEN + for (auto elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto& element = elements[elem_idx]; + if (element.#field_name.has_value()) { + value_builder->AppendValues(element.#field_name.value(), #length, nullptr); + ARROW_RETURN_NOT_OK(#builder->Append()); + } else { + ARROW_RETURN_NOT_OK(#builder->AppendNull()); + } + } + } + } else { + // TODO(andreas): Optimize for fully transparent types to do a single AppendValues call on value_builder. + quote! { + auto value_builder = static_cast(#builder->value_builder()); + #NEWLINE_TOKEN #NEWLINE_TOKEN + static_assert(sizeof(elements[0].#field_name) == sizeof(elements[0])); + ARROW_RETURN_NOT_OK(value_builder->AppendValues(elements[0].#field_name, num_elements * #length, nullptr)); + ARROW_RETURN_NOT_OK(#builder->AppendValues(num_elements)); } } } + DataType::List(_field) => { + quote!( + return arrow::Status::NotImplemented( + "TODO(andreas): dynamically sized lists are not yet supported" + ); + ) + } DataType::Struct(field_datatypes) => { let fill_fields = fields.iter().zip(field_datatypes).enumerate().map( |(i, (field, arrow_field))| { @@ -817,7 +864,7 @@ fn quote_append_elements_to_builder( if field.is_nullable { quote! { ARROW_RETURN_NOT_OK(#builder->Reserve(num_elements)); - for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + for (auto elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { const auto& element = elements[elem_idx]; if (element.#field_name.has_value()) { ARROW_RETURN_NOT_OK(#builder->Append(element.#field_name.value())); From 731aef43db5070c97ef70fe8ba69bbdbadefdc84 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Tue, 25 Jul 2023 18:58:39 +0200 Subject: [PATCH 15/20] list support and better fixed sized list support --- .../re_types_builder/src/codegen/cpp/mod.rs | 126 ++++++++++++------ 1 file changed, 84 insertions(+), 42 deletions(-) diff --git a/crates/re_types_builder/src/codegen/cpp/mod.rs b/crates/re_types_builder/src/codegen/cpp/mod.rs index fa8a0b8945dc..49ff7d7a4ad8 100644 --- a/crates/re_types_builder/src/codegen/cpp/mod.rs +++ b/crates/re_types_builder/src/codegen/cpp/mod.rs @@ -715,70 +715,97 @@ fn quote_fill_arrow_array_builder( "Expected exactly one field for primitive type {:?}", datatype ); - let field = &fields[0]; - quote_append_elements_to_builder(field, builder, true) + let object_field = &fields[0]; + quote_append_elements_to_builder(object_field, datatype, builder, true) } - DataType::FixedSizeList(field, length) => { + DataType::List(field) | DataType::FixedSizeList(field, _) => { anyhow::ensure!( fields.len() == 1, "Expected exactly one field for list type {:?}", datatype ); - match field.data_type() { - DataType::Boolean - | DataType::Int8 - | DataType::Int16 - | DataType::Int32 - | DataType::Int64 - | DataType::UInt8 - | DataType::UInt16 - | DataType::UInt32 - | DataType::UInt64 - | DataType::Float16 - | DataType::Float32 - | DataType::Float64 => {} - _ => anyhow::bail!("Unimplemented field type for fixed size list: {:?}", field), - }; + let object_field = &fields[0]; + + if matches!(field.data_type(), DataType::Extension { .. }) { + return Ok(quote!(return arrow::Status::NotImplemented( + "TODO(andreas): custom data types in lists/fixedsizelist are not yet implemented" + );)); + } let value_builder_type = arrow_array_builder_type(field.data_type()); let value_builder_type = format_ident!("{value_builder_type}"); - let field_name = format_ident!("{}", &fields[0].name); - let length = quote_integer(length); + let field_name = format_ident!("{}", &object_field.name); + let (num_items_per_element, reserve_factor) = match datatype { + DataType::List(..) => { + if field.is_nullable { + (quote!(element.#field_name.value().size()), quote_integer(1)) + } else { + (quote!(element.#field_name.size()), quote_integer(2)) + } + } + DataType::FixedSizeList(_, length) => { + let length = quote_integer(length); + (length.clone(), length) + } + _ => unreachable!(), + }; + + let setup = quote! { + auto value_builder = static_cast(#builder->value_builder()); + ARROW_RETURN_NOT_OK(#builder->Reserve(num_elements)); + ARROW_RETURN_NOT_OK(value_builder->Reserve(num_elements * #reserve_factor)); + #NEWLINE_TOKEN #NEWLINE_TOKEN + }; if field.is_nullable { + let item_append = if trivial_batch_append(field.data_type()) { + // `&expression[0]` is not pretty but works on both arrays and vectors! + quote! { + ARROW_RETURN_NOT_OK(value_builder->AppendValues(&element.#field_name.value()[0], #num_items_per_element, nullptr)); + } + } else { + quote! { + for (auto item_idx = 0; item_idx < #num_items_per_element; item_idx += 1) { + value_builder->Append(element.#field_name.value()[item_idx]); + } + } + }; quote! { - auto value_builder = static_cast(#builder->value_builder()); - ARROW_RETURN_NOT_OK(#builder->Reserve(num_elements * #length)); - ARROW_RETURN_NOT_OK(value_builder->Reserve(num_elements * #length)); - #NEWLINE_TOKEN #NEWLINE_TOKEN + #setup for (auto elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { const auto& element = elements[elem_idx]; if (element.#field_name.has_value()) { - value_builder->AppendValues(element.#field_name.value(), #length, nullptr); + #item_append ARROW_RETURN_NOT_OK(#builder->Append()); } else { ARROW_RETURN_NOT_OK(#builder->AppendNull()); } } } - } else { - // TODO(andreas): Optimize for fully transparent types to do a single AppendValues call on value_builder. + } else if matches!(datatype, DataType::FixedSizeList(..)) + && trivial_batch_append(field.data_type()) + { + // Optimize common case: Trivial batch of transparent fixed size elements. quote! { auto value_builder = static_cast(#builder->value_builder()); #NEWLINE_TOKEN #NEWLINE_TOKEN static_assert(sizeof(elements[0].#field_name) == sizeof(elements[0])); - ARROW_RETURN_NOT_OK(value_builder->AppendValues(elements[0].#field_name, num_elements * #length, nullptr)); + ARROW_RETURN_NOT_OK(value_builder->AppendValues(elements[0].#field_name, num_elements * #num_items_per_element, nullptr)); ARROW_RETURN_NOT_OK(#builder->AppendValues(num_elements)); } + } else { + quote! { + #setup + for (auto elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto& element = elements[elem_idx]; + for (auto item_idx = 0; item_idx < #num_items_per_element; item_idx += 1) { + value_builder->Append(element.#field_name[item_idx]); + } + ARROW_RETURN_NOT_OK(#builder->Append()); + } + } } } - DataType::List(_field) => { - quote!( - return arrow::Status::NotImplemented( - "TODO(andreas): dynamically sized lists are not yet supported" - ); - ) - } DataType::Struct(field_datatypes) => { let fill_fields = fields.iter().zip(field_datatypes).enumerate().map( |(i, (field, arrow_field))| { @@ -802,7 +829,7 @@ fn quote_fill_arrow_array_builder( let element_builder = format_ident!("element_builder"); let element_builder_type = arrow_array_builder_type(arrow_field.data_type()); let element_builder_type = format_ident!("{element_builder_type}"); - let field_append = quote_append_elements_to_builder(field, &element_builder, false); + let field_append = quote_append_elements_to_builder(field, datatype, &element_builder, false); quote! { { auto #element_builder = static_cast(builder->field_builder(#builder_index)); @@ -855,8 +882,28 @@ fn quote_fill_arrow_array_builder( Ok(tokens) } +fn trivial_batch_append(datatype: &DataType) -> bool { + match datatype { + DataType::Null + | DataType::Boolean + | DataType::Int8 + | DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::UInt8 + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 + | DataType::Float16 + | DataType::Float32 + | DataType::Float64 => true, + _ => false, + } +} + fn quote_append_elements_to_builder( field: &ObjectField, + datatype: &DataType, builder: &Ident, is_transparent: bool, ) -> TokenStream { @@ -873,12 +920,7 @@ fn quote_append_elements_to_builder( } } } - } else if is_transparent - && !matches!( - &field.typ, - Type::String | Type::Vector { .. } | Type::Object(..) | Type::Array { .. } - ) - { + } else if is_transparent && trivial_batch_append(datatype) { // Trivial optimization: If this is the only field of this type and it's a trivial field (not array/string/blob), // we can just pass the whole array as-is! quote! { From 7598c1c2e1ab3cc21714cdaef2ee68332b5e592c Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Tue, 25 Jul 2023 19:28:00 +0200 Subject: [PATCH 16/20] comment fixes --- crates/re_types_builder/src/codegen/cpp/mod.rs | 2 +- examples/cpp/minimal/main.cpp | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/re_types_builder/src/codegen/cpp/mod.rs b/crates/re_types_builder/src/codegen/cpp/mod.rs index 49ff7d7a4ad8..3f7a2bed048a 100644 --- a/crates/re_types_builder/src/codegen/cpp/mod.rs +++ b/crates/re_types_builder/src/codegen/cpp/mod.rs @@ -376,7 +376,7 @@ impl QuotedObject { hpp_includes.system.insert("cstring".to_owned()); // std::memcpy // Doing our own forward declarations doesn't get us super far since some arrow types like `FloatBuilder` are type aliases. - // TODO(andreas): This drags in arrow headers into the public api though. We probably should try harder with f + // TODO(andreas): This drags in arrow headers into the public api though. We probably should try harder with forward declarations. hpp_includes.system.insert("arrow/type_fwd.h".to_owned()); let mut cpp_includes = Includes::default(); diff --git a/examples/cpp/minimal/main.cpp b/examples/cpp/minimal/main.cpp index 8a09e31d4ce9..43ec5d30e4ee 100644 --- a/examples/cpp/minimal/main.cpp +++ b/examples/cpp/minimal/main.cpp @@ -9,7 +9,7 @@ arrow::Result> points2(size_t num_points, const fl ARROW_ASSIGN_OR_RAISE(auto builder, rr::components::Point2D::new_arrow_array_builder(pool)); ARROW_RETURN_NOT_OK(rr::components::Point2D::fill_arrow_array_builder( builder.get(), - (rr::components::Point2D*)xy, + (const rr::components::Point2D*)xy, // TODO(andreas): Hack to get Points2D C-style array in an easy fashion num_points )); @@ -32,10 +32,10 @@ int main(int argc, char** argv) { auto rr_stream = rr::RecordingStream{"c-example-app", "127.0.0.1:9876"}; - // Points3D, fully custom. + // Points3D. { float xyz[9] = {0.0, 0.0, 0.0, 1.0, 2.0, 3.0, 5.0, 5.0, 5.0}; - auto points = rr::points3(3, xyz).ValueOrDie(); + auto points = rr::points3(3, xyz).ValueOrDie(); // TODO(andreas): phase this out. auto buffer = rr::ipc_from_table(*points).ValueOrDie(); const rr::DataCell data_cells[1] = {rr::DataCell{ @@ -48,7 +48,7 @@ int main(int argc, char** argv) { rr_stream.log_data_row("3d/points", num_instances, 1, data_cells); } - // Points2D, via generated code. + // Points2D. { float xy[6] = {0.0, 0.0, 1.0, 3.0, 5.0, 5.0}; auto points = points2(3, xy).ValueOrDie(); From 99cdb009bee903ce190fb43d6f7859bcf103ffed Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Wed, 26 Jul 2023 11:50:40 +0200 Subject: [PATCH 17/20] spelling --- crates/re_types_builder/src/codegen/cpp/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/re_types_builder/src/codegen/cpp/mod.rs b/crates/re_types_builder/src/codegen/cpp/mod.rs index 3f7a2bed048a..12dc170b9e62 100644 --- a/crates/re_types_builder/src/codegen/cpp/mod.rs +++ b/crates/re_types_builder/src/codegen/cpp/mod.rs @@ -280,7 +280,7 @@ impl QuotedObject { )); } ObjectKind::Archetype => { - // TODO(andreas): Should also be convertable to arrow? + // TODO(andreas): Should also be convertible to arrow? } }; From dd7ebf396f728b34161f712c41db7922a3ba91e9 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Wed, 26 Jul 2023 11:59:40 +0200 Subject: [PATCH 18/20] better panic context, fix clippy warning --- .../re_types_builder/src/codegen/cpp/mod.rs | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/crates/re_types_builder/src/codegen/cpp/mod.rs b/crates/re_types_builder/src/codegen/cpp/mod.rs index 12dc170b9e62..27f922ac3f93 100644 --- a/crates/re_types_builder/src/codegen/cpp/mod.rs +++ b/crates/re_types_builder/src/codegen/cpp/mod.rs @@ -641,7 +641,7 @@ fn fill_arrow_array_builder_method( cpp_includes: &mut Includes, ) -> Method { let DataType::Extension(_fqname, logical_datatype, _metadata) = datatype else { - panic!("Can only generate arrow serialization code for extension types."); + panic!("Can only generate arrow serialization code for extension types. {}", obj.fqname); }; let builder = format_ident!("builder"); @@ -883,22 +883,22 @@ fn quote_fill_arrow_array_builder( } fn trivial_batch_append(datatype: &DataType) -> bool { - match datatype { + matches!( + datatype, DataType::Null - | DataType::Boolean - | DataType::Int8 - | DataType::Int16 - | DataType::Int32 - | DataType::Int64 - | DataType::UInt8 - | DataType::UInt16 - | DataType::UInt32 - | DataType::UInt64 - | DataType::Float16 - | DataType::Float32 - | DataType::Float64 => true, - _ => false, - } + | DataType::Boolean + | DataType::Int8 + | DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::UInt8 + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 + | DataType::Float16 + | DataType::Float32 + | DataType::Float64 + ) } fn quote_append_elements_to_builder( From cd8ee5c4d9859f6bb80f50e20afc9aa76bd9745d Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Wed, 26 Jul 2023 11:59:49 +0200 Subject: [PATCH 19/20] codegen --- crates/re_types/source_hash.txt | 2 +- rerun_cpp/src/archetypes/affix_fuzzer1.hpp | 1 + rerun_cpp/src/archetypes/points2d.hpp | 1 + rerun_cpp/src/archetypes/points3d.hpp | 1 + rerun_cpp/src/archetypes/transform3d.hpp | 1 + rerun_cpp/src/components/affix_fuzzer1.cpp | 30 +++- rerun_cpp/src/components/affix_fuzzer1.hpp | 21 ++- rerun_cpp/src/components/affix_fuzzer10.cpp | 28 ++- rerun_cpp/src/components/affix_fuzzer10.hpp | 21 ++- rerun_cpp/src/components/affix_fuzzer11.cpp | 39 ++++- rerun_cpp/src/components/affix_fuzzer11.hpp | 21 ++- rerun_cpp/src/components/affix_fuzzer12.cpp | 34 +++- rerun_cpp/src/components/affix_fuzzer12.hpp | 21 ++- rerun_cpp/src/components/affix_fuzzer13.cpp | 38 +++- rerun_cpp/src/components/affix_fuzzer13.hpp | 21 ++- rerun_cpp/src/components/affix_fuzzer14.cpp | 29 +++- rerun_cpp/src/components/affix_fuzzer14.hpp | 22 +-- rerun_cpp/src/components/affix_fuzzer15.cpp | 26 ++- rerun_cpp/src/components/affix_fuzzer15.hpp | 22 +-- rerun_cpp/src/components/affix_fuzzer16.cpp | 31 +++- rerun_cpp/src/components/affix_fuzzer16.hpp | 21 ++- rerun_cpp/src/components/affix_fuzzer17.cpp | 31 +++- rerun_cpp/src/components/affix_fuzzer17.hpp | 24 +-- rerun_cpp/src/components/affix_fuzzer18.cpp | 31 +++- rerun_cpp/src/components/affix_fuzzer18.hpp | 24 +-- rerun_cpp/src/components/affix_fuzzer19.cpp | 30 +++- rerun_cpp/src/components/affix_fuzzer19.hpp | 21 ++- rerun_cpp/src/components/affix_fuzzer2.cpp | 30 +++- rerun_cpp/src/components/affix_fuzzer2.hpp | 21 ++- rerun_cpp/src/components/affix_fuzzer3.cpp | 30 +++- rerun_cpp/src/components/affix_fuzzer3.hpp | 21 ++- rerun_cpp/src/components/affix_fuzzer4.cpp | 27 ++- rerun_cpp/src/components/affix_fuzzer4.hpp | 21 ++- rerun_cpp/src/components/affix_fuzzer5.cpp | 27 ++- rerun_cpp/src/components/affix_fuzzer5.hpp | 21 ++- rerun_cpp/src/components/affix_fuzzer6.cpp | 27 ++- rerun_cpp/src/components/affix_fuzzer6.hpp | 21 ++- rerun_cpp/src/components/affix_fuzzer7.cpp | 31 +++- rerun_cpp/src/components/affix_fuzzer7.hpp | 21 ++- rerun_cpp/src/components/affix_fuzzer8.cpp | 28 ++- rerun_cpp/src/components/affix_fuzzer8.hpp | 21 ++- rerun_cpp/src/components/affix_fuzzer9.cpp | 23 ++- rerun_cpp/src/components/affix_fuzzer9.hpp | 21 ++- rerun_cpp/src/components/class_id.cpp | 21 ++- rerun_cpp/src/components/class_id.hpp | 21 ++- rerun_cpp/src/components/color.cpp | 21 ++- rerun_cpp/src/components/color.hpp | 21 ++- rerun_cpp/src/components/draw_order.cpp | 21 ++- rerun_cpp/src/components/draw_order.hpp | 21 ++- rerun_cpp/src/components/instance_key.cpp | 21 ++- rerun_cpp/src/components/instance_key.hpp | 21 ++- rerun_cpp/src/components/keypoint_id.cpp | 21 ++- rerun_cpp/src/components/keypoint_id.hpp | 21 ++- rerun_cpp/src/components/label.cpp | 23 ++- rerun_cpp/src/components/label.hpp | 21 ++- rerun_cpp/src/components/point2d.cpp | 30 +++- rerun_cpp/src/components/point2d.hpp | 21 ++- rerun_cpp/src/components/point3d.cpp | 30 +++- rerun_cpp/src/components/point3d.hpp | 21 ++- rerun_cpp/src/components/radius.cpp | 21 ++- rerun_cpp/src/components/radius.hpp | 21 ++- rerun_cpp/src/components/transform3d.cpp | 29 +++- rerun_cpp/src/components/transform3d.hpp | 21 ++- rerun_cpp/src/datatypes/affix_fuzzer1.cpp | 162 +++++++++++++++--- rerun_cpp/src/datatypes/affix_fuzzer1.hpp | 21 ++- rerun_cpp/src/datatypes/affix_fuzzer2.cpp | 28 ++- rerun_cpp/src/datatypes/affix_fuzzer2.hpp | 21 ++- rerun_cpp/src/datatypes/affix_fuzzer3.cpp | 68 +++++++- rerun_cpp/src/datatypes/affix_fuzzer3.hpp | 12 +- rerun_cpp/src/datatypes/affix_fuzzer4.cpp | 72 +++++++- rerun_cpp/src/datatypes/affix_fuzzer4.hpp | 19 +- rerun_cpp/src/datatypes/affix_fuzzer5.cpp | 43 +++-- rerun_cpp/src/datatypes/affix_fuzzer5.hpp | 21 ++- rerun_cpp/src/datatypes/angle.cpp | 36 ++++ rerun_cpp/src/datatypes/angle.hpp | 12 +- rerun_cpp/src/datatypes/flattened_scalar.cpp | 37 +++- rerun_cpp/src/datatypes/flattened_scalar.hpp | 22 ++- rerun_cpp/src/datatypes/mat3x3.cpp | 36 +++- rerun_cpp/src/datatypes/mat3x3.hpp | 20 +-- rerun_cpp/src/datatypes/mat4x4.cpp | 36 +++- rerun_cpp/src/datatypes/mat4x4.hpp | 20 +-- rerun_cpp/src/datatypes/point2d.cpp | 47 ++++- rerun_cpp/src/datatypes/point2d.hpp | 21 ++- rerun_cpp/src/datatypes/point3d.cpp | 56 +++++- rerun_cpp/src/datatypes/point3d.hpp | 21 ++- rerun_cpp/src/datatypes/quaternion.cpp | 35 +++- rerun_cpp/src/datatypes/quaternion.hpp | 21 ++- rerun_cpp/src/datatypes/rotation3d.cpp | 52 +++++- rerun_cpp/src/datatypes/rotation3d.hpp | 12 +- .../src/datatypes/rotation_axis_angle.cpp | 37 +++- .../src/datatypes/rotation_axis_angle.hpp | 23 ++- rerun_cpp/src/datatypes/scale3d.cpp | 36 ++++ rerun_cpp/src/datatypes/scale3d.hpp | 12 +- rerun_cpp/src/datatypes/transform3d.cpp | 57 +++++- rerun_cpp/src/datatypes/transform3d.hpp | 18 +- .../src/datatypes/translation_and_mat3x3.cpp | 52 ++++-- .../src/datatypes/translation_and_mat3x3.hpp | 23 ++- .../translation_rotation_scale3d.cpp | 63 +++++-- .../translation_rotation_scale3d.hpp | 23 ++- rerun_cpp/src/datatypes/vec2d.cpp | 36 +++- rerun_cpp/src/datatypes/vec2d.hpp | 20 +-- rerun_cpp/src/datatypes/vec3d.cpp | 36 +++- rerun_cpp/src/datatypes/vec3d.hpp | 20 +-- rerun_cpp/src/datatypes/vec4d.cpp | 36 +++- rerun_cpp/src/datatypes/vec4d.hpp | 20 +-- 105 files changed, 2062 insertions(+), 832 deletions(-) diff --git a/crates/re_types/source_hash.txt b/crates/re_types/source_hash.txt index e7a5b43275d8..90386b12beae 100644 --- a/crates/re_types/source_hash.txt +++ b/crates/re_types/source_hash.txt @@ -1,4 +1,4 @@ # This is a sha256 hash for all direct and indirect dependencies of this crate's build script. # It can be safely removed at anytime to force the build script to run again. # Check out build.rs to see how it's computed. -b60f0b918bc9d49fcebd79c19baeda65279d8c205bbc1367d270b92fd3395848 +360fb26536a5ab8641566344660cc6df17d791db89059100018fd7f995da9cdc diff --git a/rerun_cpp/src/archetypes/affix_fuzzer1.hpp b/rerun_cpp/src/archetypes/affix_fuzzer1.hpp index 027fb9a2a2d7..a6396d0d2d0a 100644 --- a/rerun_cpp/src/archetypes/affix_fuzzer1.hpp +++ b/rerun_cpp/src/archetypes/affix_fuzzer1.hpp @@ -23,6 +23,7 @@ #include "../components/affix_fuzzer8.hpp" #include "../components/affix_fuzzer9.hpp" +#include #include #include #include diff --git a/rerun_cpp/src/archetypes/points2d.hpp b/rerun_cpp/src/archetypes/points2d.hpp index 66fb2798a798..39fbbbd547c1 100644 --- a/rerun_cpp/src/archetypes/points2d.hpp +++ b/rerun_cpp/src/archetypes/points2d.hpp @@ -12,6 +12,7 @@ #include "../components/point2d.hpp" #include "../components/radius.hpp" +#include #include #include #include diff --git a/rerun_cpp/src/archetypes/points3d.hpp b/rerun_cpp/src/archetypes/points3d.hpp index 841b6a0d709b..6fe3b44b3fd0 100644 --- a/rerun_cpp/src/archetypes/points3d.hpp +++ b/rerun_cpp/src/archetypes/points3d.hpp @@ -12,6 +12,7 @@ #include "../components/point3d.hpp" #include "../components/radius.hpp" +#include #include #include #include diff --git a/rerun_cpp/src/archetypes/transform3d.hpp b/rerun_cpp/src/archetypes/transform3d.hpp index cb678d92f1d7..82dda3cc1ed0 100644 --- a/rerun_cpp/src/archetypes/transform3d.hpp +++ b/rerun_cpp/src/archetypes/transform3d.hpp @@ -5,6 +5,7 @@ #include "../components/transform3d.hpp" +#include #include #include diff --git a/rerun_cpp/src/components/affix_fuzzer1.cpp b/rerun_cpp/src/components/affix_fuzzer1.cpp index 476c33a0568b..5518fea03a42 100644 --- a/rerun_cpp/src/components/affix_fuzzer1.cpp +++ b/rerun_cpp/src/components/affix_fuzzer1.cpp @@ -13,22 +13,36 @@ namespace rr { return rr::datatypes::AffixFuzzer1::to_arrow_datatype(); } - arrow::Result> AffixFuzzer1::to_arrow( - arrow::MemoryPool *memory_pool, const AffixFuzzer1 *elements, size_t num_elements) { + arrow::Result> AffixFuzzer1::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result( + rr::datatypes::AffixFuzzer1::new_arrow_array_builder(memory_pool).ValueOrDie() + ); + } + + arrow::Status AffixFuzzer1::fill_arrow_array_builder( + arrow::StructBuilder *builder, const AffixFuzzer1 *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - static_assert(sizeof(AffixFuzzer1) == sizeof(rr::datatypes::AffixFuzzer1), - "Expected fully transparent type."); - auto builder = rr::datatypes::AffixFuzzer1::to_arrow( - memory_pool, + static_assert(sizeof(rr::datatypes::AffixFuzzer1) == sizeof(AffixFuzzer1)); + ARROW_RETURN_NOT_OK(rr::datatypes::AffixFuzzer1::fill_arrow_array_builder( + builder, reinterpret_cast(elements), - num_elements); - return builder; + num_elements + )); + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer1.hpp b/rerun_cpp/src/components/affix_fuzzer1.hpp index 9edabb00debe..a749a1ea4bb5 100644 --- a/rerun_cpp/src/components/affix_fuzzer1.hpp +++ b/rerun_cpp/src/components/affix_fuzzer1.hpp @@ -5,17 +5,10 @@ #include "../datatypes/affix_fuzzer1.hpp" -#include +#include #include -#include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace components { struct AffixFuzzer1 { @@ -28,9 +21,15 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer1* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const AffixFuzzer1* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer10.cpp b/rerun_cpp/src/components/affix_fuzzer10.cpp index b0daff857fc0..0c3a13d23e85 100644 --- a/rerun_cpp/src/components/affix_fuzzer10.cpp +++ b/rerun_cpp/src/components/affix_fuzzer10.cpp @@ -11,17 +11,37 @@ namespace rr { return arrow::utf8(); } - arrow::Result> AffixFuzzer10::to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer10* elements, size_t num_elements) { + arrow::Result> AffixFuzzer10::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result(std::make_shared(memory_pool)); + } + + arrow::Status AffixFuzzer10::fill_arrow_array_builder( + arrow::StringBuilder* builder, const AffixFuzzer10* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - auto builder = std::make_shared(memory_pool); - return builder; + ARROW_RETURN_NOT_OK(builder->Reserve(num_elements)); + for (auto elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto& element = elements[elem_idx]; + if (element.single_string_optional.has_value()) { + ARROW_RETURN_NOT_OK(builder->Append(element.single_string_optional.value())); + } else { + ARROW_RETURN_NOT_OK(builder->AppendNull()); + } + } + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer10.hpp b/rerun_cpp/src/components/affix_fuzzer10.hpp index 591a3cb35d56..1c3c951d9711 100644 --- a/rerun_cpp/src/components/affix_fuzzer10.hpp +++ b/rerun_cpp/src/components/affix_fuzzer10.hpp @@ -3,19 +3,12 @@ #pragma once -#include +#include #include -#include #include #include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace components { struct AffixFuzzer10 { @@ -28,9 +21,15 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer10* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StringBuilder* builder, const AffixFuzzer10* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer11.cpp b/rerun_cpp/src/components/affix_fuzzer11.cpp index c276e6851d6d..4e82609e9bca 100644 --- a/rerun_cpp/src/components/affix_fuzzer11.cpp +++ b/rerun_cpp/src/components/affix_fuzzer11.cpp @@ -11,17 +11,48 @@ namespace rr { return arrow::list(arrow::field("item", arrow::float32(), true, nullptr)); } - arrow::Result> AffixFuzzer11::to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer11* elements, size_t num_elements) { + arrow::Result> AffixFuzzer11::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result(std::make_shared( + memory_pool, + std::make_shared(memory_pool) + )); + } + + arrow::Status AffixFuzzer11::fill_arrow_array_builder( + arrow::ListBuilder *builder, const AffixFuzzer11 *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - auto builder = std::make_shared(memory_pool); - return builder; + auto value_builder = static_cast(builder->value_builder()); + ARROW_RETURN_NOT_OK(builder->Reserve(num_elements)); + ARROW_RETURN_NOT_OK(value_builder->Reserve(num_elements * 1)); + + for (auto elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto &element = elements[elem_idx]; + if (element.many_floats_optional.has_value()) { + ARROW_RETURN_NOT_OK(value_builder->AppendValues( + &element.many_floats_optional.value()[0], + element.many_floats_optional.value().size(), + nullptr + )); + ARROW_RETURN_NOT_OK(builder->Append()); + } else { + ARROW_RETURN_NOT_OK(builder->AppendNull()); + } + } + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer11.hpp b/rerun_cpp/src/components/affix_fuzzer11.hpp index 87212f1ac256..ad40ffa4036b 100644 --- a/rerun_cpp/src/components/affix_fuzzer11.hpp +++ b/rerun_cpp/src/components/affix_fuzzer11.hpp @@ -3,19 +3,12 @@ #pragma once -#include +#include #include -#include #include #include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace components { struct AffixFuzzer11 { @@ -28,9 +21,15 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer11* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::ListBuilder* builder, const AffixFuzzer11* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer12.cpp b/rerun_cpp/src/components/affix_fuzzer12.cpp index 787a383ec4cd..a37316693711 100644 --- a/rerun_cpp/src/components/affix_fuzzer12.cpp +++ b/rerun_cpp/src/components/affix_fuzzer12.cpp @@ -11,17 +11,43 @@ namespace rr { return arrow::list(arrow::field("item", arrow::utf8(), false, nullptr)); } - arrow::Result> AffixFuzzer12::to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer12* elements, size_t num_elements) { + arrow::Result> AffixFuzzer12::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result(std::make_shared( + memory_pool, + std::make_shared(memory_pool) + )); + } + + arrow::Status AffixFuzzer12::fill_arrow_array_builder( + arrow::ListBuilder *builder, const AffixFuzzer12 *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - auto builder = std::make_shared(memory_pool); - return builder; + auto value_builder = static_cast(builder->value_builder()); + ARROW_RETURN_NOT_OK(builder->Reserve(num_elements)); + ARROW_RETURN_NOT_OK(value_builder->Reserve(num_elements * 2)); + + for (auto elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto &element = elements[elem_idx]; + for (auto item_idx = 0; item_idx < element.many_strings_required.size(); + item_idx += 1) { + value_builder->Append(element.many_strings_required[item_idx]); + } + ARROW_RETURN_NOT_OK(builder->Append()); + } + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer12.hpp b/rerun_cpp/src/components/affix_fuzzer12.hpp index 33f378cc1545..d6e2761dc424 100644 --- a/rerun_cpp/src/components/affix_fuzzer12.hpp +++ b/rerun_cpp/src/components/affix_fuzzer12.hpp @@ -3,19 +3,12 @@ #pragma once -#include +#include #include -#include #include #include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace components { struct AffixFuzzer12 { @@ -28,9 +21,15 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer12* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::ListBuilder* builder, const AffixFuzzer12* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer13.cpp b/rerun_cpp/src/components/affix_fuzzer13.cpp index 83f30a3071d8..c2bdb9baa824 100644 --- a/rerun_cpp/src/components/affix_fuzzer13.cpp +++ b/rerun_cpp/src/components/affix_fuzzer13.cpp @@ -11,17 +11,47 @@ namespace rr { return arrow::list(arrow::field("item", arrow::utf8(), true, nullptr)); } - arrow::Result> AffixFuzzer13::to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer13* elements, size_t num_elements) { + arrow::Result> AffixFuzzer13::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result(std::make_shared( + memory_pool, + std::make_shared(memory_pool) + )); + } + + arrow::Status AffixFuzzer13::fill_arrow_array_builder( + arrow::ListBuilder *builder, const AffixFuzzer13 *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - auto builder = std::make_shared(memory_pool); - return builder; + auto value_builder = static_cast(builder->value_builder()); + ARROW_RETURN_NOT_OK(builder->Reserve(num_elements)); + ARROW_RETURN_NOT_OK(value_builder->Reserve(num_elements * 1)); + + for (auto elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto &element = elements[elem_idx]; + if (element.many_strings_optional.has_value()) { + for (auto item_idx = 0; item_idx < element.many_strings_optional.value().size(); + item_idx += 1) { + value_builder->Append(element.many_strings_optional.value()[item_idx]); + } + ARROW_RETURN_NOT_OK(builder->Append()); + } else { + ARROW_RETURN_NOT_OK(builder->AppendNull()); + } + } + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer13.hpp b/rerun_cpp/src/components/affix_fuzzer13.hpp index b034598b4a07..5ddd66ff404c 100644 --- a/rerun_cpp/src/components/affix_fuzzer13.hpp +++ b/rerun_cpp/src/components/affix_fuzzer13.hpp @@ -3,20 +3,13 @@ #pragma once -#include +#include #include -#include #include #include #include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace components { struct AffixFuzzer13 { @@ -29,9 +22,15 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer13* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::ListBuilder* builder, const AffixFuzzer13* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer14.cpp b/rerun_cpp/src/components/affix_fuzzer14.cpp index 382990eb5a78..3a97ebed9e83 100644 --- a/rerun_cpp/src/components/affix_fuzzer14.cpp +++ b/rerun_cpp/src/components/affix_fuzzer14.cpp @@ -13,22 +13,35 @@ namespace rr { return rr::datatypes::AffixFuzzer3::to_arrow_datatype(); } - arrow::Result> AffixFuzzer14::to_arrow( - arrow::MemoryPool *memory_pool, const AffixFuzzer14 *elements, size_t num_elements) { + arrow::Result> + AffixFuzzer14::new_arrow_array_builder(arrow::MemoryPool *memory_pool) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result( + rr::datatypes::AffixFuzzer3::new_arrow_array_builder(memory_pool).ValueOrDie() + ); + } + + arrow::Status AffixFuzzer14::fill_arrow_array_builder( + arrow::DenseUnionBuilder *builder, const AffixFuzzer14 *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - static_assert(sizeof(AffixFuzzer14) == sizeof(rr::datatypes::AffixFuzzer3), - "Expected fully transparent type."); - auto builder = rr::datatypes::AffixFuzzer3::to_arrow( - memory_pool, + static_assert(sizeof(rr::datatypes::AffixFuzzer3) == sizeof(AffixFuzzer14)); + ARROW_RETURN_NOT_OK(rr::datatypes::AffixFuzzer3::fill_arrow_array_builder( + builder, reinterpret_cast(elements), - num_elements); - return builder; + num_elements + )); + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer14.hpp b/rerun_cpp/src/components/affix_fuzzer14.hpp index d9ccc447bc5e..c7d4e8d345d6 100644 --- a/rerun_cpp/src/components/affix_fuzzer14.hpp +++ b/rerun_cpp/src/components/affix_fuzzer14.hpp @@ -5,17 +5,10 @@ #include "../datatypes/affix_fuzzer3.hpp" -#include +#include #include -#include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace components { struct AffixFuzzer14 { @@ -28,9 +21,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer14* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::DenseUnionBuilder* builder, const AffixFuzzer14* elements, + size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer15.cpp b/rerun_cpp/src/components/affix_fuzzer15.cpp index 96836f0c4600..c80eb365c0a6 100644 --- a/rerun_cpp/src/components/affix_fuzzer15.cpp +++ b/rerun_cpp/src/components/affix_fuzzer15.cpp @@ -13,22 +13,30 @@ namespace rr { return rr::datatypes::AffixFuzzer3::to_arrow_datatype(); } - arrow::Result> AffixFuzzer15::to_arrow( - arrow::MemoryPool *memory_pool, const AffixFuzzer15 *elements, size_t num_elements) { + arrow::Result> + AffixFuzzer15::new_arrow_array_builder(arrow::MemoryPool* memory_pool) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result( + rr::datatypes::AffixFuzzer3::new_arrow_array_builder(memory_pool).ValueOrDie() + ); + } + + arrow::Status AffixFuzzer15::fill_arrow_array_builder( + arrow::DenseUnionBuilder* builder, const AffixFuzzer15* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - static_assert(sizeof(AffixFuzzer15) == sizeof(rr::datatypes::AffixFuzzer3), - "Expected fully transparent type."); - auto builder = rr::datatypes::AffixFuzzer3::to_arrow( - memory_pool, - reinterpret_cast(elements), - num_elements); - return builder; + return arrow::Status::NotImplemented(("TODO(andreas) Handle nullable extensions")); + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer15.hpp b/rerun_cpp/src/components/affix_fuzzer15.hpp index 4db8444c2bb1..3fdb1a83796e 100644 --- a/rerun_cpp/src/components/affix_fuzzer15.hpp +++ b/rerun_cpp/src/components/affix_fuzzer15.hpp @@ -5,18 +5,11 @@ #include "../datatypes/affix_fuzzer3.hpp" -#include +#include #include -#include #include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace components { struct AffixFuzzer15 { @@ -29,9 +22,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer15* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::DenseUnionBuilder* builder, const AffixFuzzer15* elements, + size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer16.cpp b/rerun_cpp/src/components/affix_fuzzer16.cpp index 5d1d3d7a01ed..10d40df328e6 100644 --- a/rerun_cpp/src/components/affix_fuzzer16.cpp +++ b/rerun_cpp/src/components/affix_fuzzer16.cpp @@ -11,20 +11,41 @@ namespace rr { namespace components { std::shared_ptr AffixFuzzer16::to_arrow_datatype() { return arrow::list(arrow::field( - "item", rr::datatypes::AffixFuzzer3::to_arrow_datatype(), false, nullptr)); + "item", + rr::datatypes::AffixFuzzer3::to_arrow_datatype(), + false, + nullptr + )); } - arrow::Result> AffixFuzzer16::to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer16* elements, size_t num_elements) { + arrow::Result> AffixFuzzer16::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result(std::make_shared( + memory_pool, + rr::datatypes::AffixFuzzer3::new_arrow_array_builder(memory_pool).ValueOrDie() + )); + } + + arrow::Status AffixFuzzer16::fill_arrow_array_builder( + arrow::ListBuilder* builder, const AffixFuzzer16* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - auto builder = std::make_shared(memory_pool); - return builder; + return arrow::Status::NotImplemented( + "TODO(andreas): custom data types in lists/fixedsizelist are not yet implemented" + ); + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer16.hpp b/rerun_cpp/src/components/affix_fuzzer16.hpp index 7d039147da77..973b3d502cb8 100644 --- a/rerun_cpp/src/components/affix_fuzzer16.hpp +++ b/rerun_cpp/src/components/affix_fuzzer16.hpp @@ -5,18 +5,11 @@ #include "../datatypes/affix_fuzzer3.hpp" -#include +#include #include -#include #include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace components { struct AffixFuzzer16 { @@ -29,9 +22,15 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer16* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::ListBuilder* builder, const AffixFuzzer16* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer17.cpp b/rerun_cpp/src/components/affix_fuzzer17.cpp index 345e45341c70..8e478c4e812d 100644 --- a/rerun_cpp/src/components/affix_fuzzer17.cpp +++ b/rerun_cpp/src/components/affix_fuzzer17.cpp @@ -11,20 +11,41 @@ namespace rr { namespace components { std::shared_ptr AffixFuzzer17::to_arrow_datatype() { return arrow::list(arrow::field( - "item", rr::datatypes::AffixFuzzer3::to_arrow_datatype(), true, nullptr)); + "item", + rr::datatypes::AffixFuzzer3::to_arrow_datatype(), + true, + nullptr + )); } - arrow::Result> AffixFuzzer17::to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer17* elements, size_t num_elements) { + arrow::Result> AffixFuzzer17::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result(std::make_shared( + memory_pool, + rr::datatypes::AffixFuzzer3::new_arrow_array_builder(memory_pool).ValueOrDie() + )); + } + + arrow::Status AffixFuzzer17::fill_arrow_array_builder( + arrow::ListBuilder* builder, const AffixFuzzer17* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - auto builder = std::make_shared(memory_pool); - return builder; + return arrow::Status::NotImplemented( + "TODO(andreas): custom data types in lists/fixedsizelist are not yet implemented" + ); + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer17.hpp b/rerun_cpp/src/components/affix_fuzzer17.hpp index aab196cc3b43..849774b2cf96 100644 --- a/rerun_cpp/src/components/affix_fuzzer17.hpp +++ b/rerun_cpp/src/components/affix_fuzzer17.hpp @@ -5,19 +5,12 @@ #include "../datatypes/affix_fuzzer3.hpp" -#include +#include #include -#include #include #include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace components { struct AffixFuzzer17 { @@ -25,15 +18,22 @@ namespace rr { public: AffixFuzzer17( - std::optional> many_optional_unions) + std::optional> many_optional_unions + ) : many_optional_unions(std::move(many_optional_unions)) {} /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer17* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::ListBuilder* builder, const AffixFuzzer17* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer18.cpp b/rerun_cpp/src/components/affix_fuzzer18.cpp index c497d378a913..088de4597696 100644 --- a/rerun_cpp/src/components/affix_fuzzer18.cpp +++ b/rerun_cpp/src/components/affix_fuzzer18.cpp @@ -11,20 +11,41 @@ namespace rr { namespace components { std::shared_ptr AffixFuzzer18::to_arrow_datatype() { return arrow::list(arrow::field( - "item", rr::datatypes::AffixFuzzer4::to_arrow_datatype(), true, nullptr)); + "item", + rr::datatypes::AffixFuzzer4::to_arrow_datatype(), + true, + nullptr + )); } - arrow::Result> AffixFuzzer18::to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer18* elements, size_t num_elements) { + arrow::Result> AffixFuzzer18::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result(std::make_shared( + memory_pool, + rr::datatypes::AffixFuzzer4::new_arrow_array_builder(memory_pool).ValueOrDie() + )); + } + + arrow::Status AffixFuzzer18::fill_arrow_array_builder( + arrow::ListBuilder* builder, const AffixFuzzer18* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - auto builder = std::make_shared(memory_pool); - return builder; + return arrow::Status::NotImplemented( + "TODO(andreas): custom data types in lists/fixedsizelist are not yet implemented" + ); + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer18.hpp b/rerun_cpp/src/components/affix_fuzzer18.hpp index b8ee0f2b4788..825e29082b9e 100644 --- a/rerun_cpp/src/components/affix_fuzzer18.hpp +++ b/rerun_cpp/src/components/affix_fuzzer18.hpp @@ -5,19 +5,12 @@ #include "../datatypes/affix_fuzzer4.hpp" -#include +#include #include -#include #include #include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace components { struct AffixFuzzer18 { @@ -25,15 +18,22 @@ namespace rr { public: AffixFuzzer18( - std::optional> many_optional_unions) + std::optional> many_optional_unions + ) : many_optional_unions(std::move(many_optional_unions)) {} /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer18* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::ListBuilder* builder, const AffixFuzzer18* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer19.cpp b/rerun_cpp/src/components/affix_fuzzer19.cpp index 705271184bcd..b1630b37e408 100644 --- a/rerun_cpp/src/components/affix_fuzzer19.cpp +++ b/rerun_cpp/src/components/affix_fuzzer19.cpp @@ -13,22 +13,36 @@ namespace rr { return rr::datatypes::AffixFuzzer5::to_arrow_datatype(); } - arrow::Result> AffixFuzzer19::to_arrow( - arrow::MemoryPool *memory_pool, const AffixFuzzer19 *elements, size_t num_elements) { + arrow::Result> AffixFuzzer19::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result( + rr::datatypes::AffixFuzzer5::new_arrow_array_builder(memory_pool).ValueOrDie() + ); + } + + arrow::Status AffixFuzzer19::fill_arrow_array_builder( + arrow::StructBuilder *builder, const AffixFuzzer19 *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - static_assert(sizeof(AffixFuzzer19) == sizeof(rr::datatypes::AffixFuzzer5), - "Expected fully transparent type."); - auto builder = rr::datatypes::AffixFuzzer5::to_arrow( - memory_pool, + static_assert(sizeof(rr::datatypes::AffixFuzzer5) == sizeof(AffixFuzzer19)); + ARROW_RETURN_NOT_OK(rr::datatypes::AffixFuzzer5::fill_arrow_array_builder( + builder, reinterpret_cast(elements), - num_elements); - return builder; + num_elements + )); + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer19.hpp b/rerun_cpp/src/components/affix_fuzzer19.hpp index 75385a901b47..089daef4598f 100644 --- a/rerun_cpp/src/components/affix_fuzzer19.hpp +++ b/rerun_cpp/src/components/affix_fuzzer19.hpp @@ -5,17 +5,10 @@ #include "../datatypes/affix_fuzzer5.hpp" -#include +#include #include -#include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace components { struct AffixFuzzer19 { @@ -28,9 +21,15 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer19* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const AffixFuzzer19* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer2.cpp b/rerun_cpp/src/components/affix_fuzzer2.cpp index 6104164da41d..083d72529def 100644 --- a/rerun_cpp/src/components/affix_fuzzer2.cpp +++ b/rerun_cpp/src/components/affix_fuzzer2.cpp @@ -13,22 +13,36 @@ namespace rr { return rr::datatypes::AffixFuzzer1::to_arrow_datatype(); } - arrow::Result> AffixFuzzer2::to_arrow( - arrow::MemoryPool *memory_pool, const AffixFuzzer2 *elements, size_t num_elements) { + arrow::Result> AffixFuzzer2::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result( + rr::datatypes::AffixFuzzer1::new_arrow_array_builder(memory_pool).ValueOrDie() + ); + } + + arrow::Status AffixFuzzer2::fill_arrow_array_builder( + arrow::StructBuilder *builder, const AffixFuzzer2 *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - static_assert(sizeof(AffixFuzzer2) == sizeof(rr::datatypes::AffixFuzzer1), - "Expected fully transparent type."); - auto builder = rr::datatypes::AffixFuzzer1::to_arrow( - memory_pool, + static_assert(sizeof(rr::datatypes::AffixFuzzer1) == sizeof(AffixFuzzer2)); + ARROW_RETURN_NOT_OK(rr::datatypes::AffixFuzzer1::fill_arrow_array_builder( + builder, reinterpret_cast(elements), - num_elements); - return builder; + num_elements + )); + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer2.hpp b/rerun_cpp/src/components/affix_fuzzer2.hpp index e7245d956a0d..4db325c8df87 100644 --- a/rerun_cpp/src/components/affix_fuzzer2.hpp +++ b/rerun_cpp/src/components/affix_fuzzer2.hpp @@ -5,17 +5,10 @@ #include "../datatypes/affix_fuzzer1.hpp" -#include +#include #include -#include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace components { struct AffixFuzzer2 { @@ -28,9 +21,15 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer2* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const AffixFuzzer2* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer3.cpp b/rerun_cpp/src/components/affix_fuzzer3.cpp index 1b017fc6d1bc..9a9e36eb4e58 100644 --- a/rerun_cpp/src/components/affix_fuzzer3.cpp +++ b/rerun_cpp/src/components/affix_fuzzer3.cpp @@ -13,22 +13,36 @@ namespace rr { return rr::datatypes::AffixFuzzer1::to_arrow_datatype(); } - arrow::Result> AffixFuzzer3::to_arrow( - arrow::MemoryPool *memory_pool, const AffixFuzzer3 *elements, size_t num_elements) { + arrow::Result> AffixFuzzer3::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result( + rr::datatypes::AffixFuzzer1::new_arrow_array_builder(memory_pool).ValueOrDie() + ); + } + + arrow::Status AffixFuzzer3::fill_arrow_array_builder( + arrow::StructBuilder *builder, const AffixFuzzer3 *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - static_assert(sizeof(AffixFuzzer3) == sizeof(rr::datatypes::AffixFuzzer1), - "Expected fully transparent type."); - auto builder = rr::datatypes::AffixFuzzer1::to_arrow( - memory_pool, + static_assert(sizeof(rr::datatypes::AffixFuzzer1) == sizeof(AffixFuzzer3)); + ARROW_RETURN_NOT_OK(rr::datatypes::AffixFuzzer1::fill_arrow_array_builder( + builder, reinterpret_cast(elements), - num_elements); - return builder; + num_elements + )); + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer3.hpp b/rerun_cpp/src/components/affix_fuzzer3.hpp index 7a226c16f4a2..d0443ea19ad1 100644 --- a/rerun_cpp/src/components/affix_fuzzer3.hpp +++ b/rerun_cpp/src/components/affix_fuzzer3.hpp @@ -5,17 +5,10 @@ #include "../datatypes/affix_fuzzer1.hpp" -#include +#include #include -#include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace components { struct AffixFuzzer3 { @@ -28,9 +21,15 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer3* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const AffixFuzzer3* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer4.cpp b/rerun_cpp/src/components/affix_fuzzer4.cpp index b6c7e4f75ae3..cb4c8d7a4438 100644 --- a/rerun_cpp/src/components/affix_fuzzer4.cpp +++ b/rerun_cpp/src/components/affix_fuzzer4.cpp @@ -13,22 +13,31 @@ namespace rr { return rr::datatypes::AffixFuzzer1::to_arrow_datatype(); } - arrow::Result> AffixFuzzer4::to_arrow( - arrow::MemoryPool *memory_pool, const AffixFuzzer4 *elements, size_t num_elements) { + arrow::Result> AffixFuzzer4::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result( + rr::datatypes::AffixFuzzer1::new_arrow_array_builder(memory_pool).ValueOrDie() + ); + } + + arrow::Status AffixFuzzer4::fill_arrow_array_builder( + arrow::StructBuilder* builder, const AffixFuzzer4* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - static_assert(sizeof(AffixFuzzer4) == sizeof(rr::datatypes::AffixFuzzer1), - "Expected fully transparent type."); - auto builder = rr::datatypes::AffixFuzzer1::to_arrow( - memory_pool, - reinterpret_cast(elements), - num_elements); - return builder; + return arrow::Status::NotImplemented(("TODO(andreas) Handle nullable extensions")); + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer4.hpp b/rerun_cpp/src/components/affix_fuzzer4.hpp index bb477ef2359c..5ab17b0629be 100644 --- a/rerun_cpp/src/components/affix_fuzzer4.hpp +++ b/rerun_cpp/src/components/affix_fuzzer4.hpp @@ -5,18 +5,11 @@ #include "../datatypes/affix_fuzzer1.hpp" -#include +#include #include -#include #include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace components { struct AffixFuzzer4 { @@ -29,9 +22,15 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer4* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const AffixFuzzer4* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer5.cpp b/rerun_cpp/src/components/affix_fuzzer5.cpp index 5cd55fbeda2a..f6080fafc63d 100644 --- a/rerun_cpp/src/components/affix_fuzzer5.cpp +++ b/rerun_cpp/src/components/affix_fuzzer5.cpp @@ -13,22 +13,31 @@ namespace rr { return rr::datatypes::AffixFuzzer1::to_arrow_datatype(); } - arrow::Result> AffixFuzzer5::to_arrow( - arrow::MemoryPool *memory_pool, const AffixFuzzer5 *elements, size_t num_elements) { + arrow::Result> AffixFuzzer5::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result( + rr::datatypes::AffixFuzzer1::new_arrow_array_builder(memory_pool).ValueOrDie() + ); + } + + arrow::Status AffixFuzzer5::fill_arrow_array_builder( + arrow::StructBuilder* builder, const AffixFuzzer5* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - static_assert(sizeof(AffixFuzzer5) == sizeof(rr::datatypes::AffixFuzzer1), - "Expected fully transparent type."); - auto builder = rr::datatypes::AffixFuzzer1::to_arrow( - memory_pool, - reinterpret_cast(elements), - num_elements); - return builder; + return arrow::Status::NotImplemented(("TODO(andreas) Handle nullable extensions")); + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer5.hpp b/rerun_cpp/src/components/affix_fuzzer5.hpp index fc0b4007653e..f6ed7ebb2bbd 100644 --- a/rerun_cpp/src/components/affix_fuzzer5.hpp +++ b/rerun_cpp/src/components/affix_fuzzer5.hpp @@ -5,18 +5,11 @@ #include "../datatypes/affix_fuzzer1.hpp" -#include +#include #include -#include #include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace components { struct AffixFuzzer5 { @@ -29,9 +22,15 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer5* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const AffixFuzzer5* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer6.cpp b/rerun_cpp/src/components/affix_fuzzer6.cpp index 2381e808af3b..f6eb36be291f 100644 --- a/rerun_cpp/src/components/affix_fuzzer6.cpp +++ b/rerun_cpp/src/components/affix_fuzzer6.cpp @@ -13,22 +13,31 @@ namespace rr { return rr::datatypes::AffixFuzzer1::to_arrow_datatype(); } - arrow::Result> AffixFuzzer6::to_arrow( - arrow::MemoryPool *memory_pool, const AffixFuzzer6 *elements, size_t num_elements) { + arrow::Result> AffixFuzzer6::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result( + rr::datatypes::AffixFuzzer1::new_arrow_array_builder(memory_pool).ValueOrDie() + ); + } + + arrow::Status AffixFuzzer6::fill_arrow_array_builder( + arrow::StructBuilder* builder, const AffixFuzzer6* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - static_assert(sizeof(AffixFuzzer6) == sizeof(rr::datatypes::AffixFuzzer1), - "Expected fully transparent type."); - auto builder = rr::datatypes::AffixFuzzer1::to_arrow( - memory_pool, - reinterpret_cast(elements), - num_elements); - return builder; + return arrow::Status::NotImplemented(("TODO(andreas) Handle nullable extensions")); + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer6.hpp b/rerun_cpp/src/components/affix_fuzzer6.hpp index a39106a820dd..78f27128c35f 100644 --- a/rerun_cpp/src/components/affix_fuzzer6.hpp +++ b/rerun_cpp/src/components/affix_fuzzer6.hpp @@ -5,18 +5,11 @@ #include "../datatypes/affix_fuzzer1.hpp" -#include +#include #include -#include #include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace components { struct AffixFuzzer6 { @@ -29,9 +22,15 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer6* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const AffixFuzzer6* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer7.cpp b/rerun_cpp/src/components/affix_fuzzer7.cpp index b4f935b6efa4..30d4bb200b15 100644 --- a/rerun_cpp/src/components/affix_fuzzer7.cpp +++ b/rerun_cpp/src/components/affix_fuzzer7.cpp @@ -11,20 +11,41 @@ namespace rr { namespace components { std::shared_ptr AffixFuzzer7::to_arrow_datatype() { return arrow::list(arrow::field( - "item", rr::datatypes::AffixFuzzer1::to_arrow_datatype(), true, nullptr)); + "item", + rr::datatypes::AffixFuzzer1::to_arrow_datatype(), + true, + nullptr + )); } - arrow::Result> AffixFuzzer7::to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer7* elements, size_t num_elements) { + arrow::Result> AffixFuzzer7::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result(std::make_shared( + memory_pool, + rr::datatypes::AffixFuzzer1::new_arrow_array_builder(memory_pool).ValueOrDie() + )); + } + + arrow::Status AffixFuzzer7::fill_arrow_array_builder( + arrow::ListBuilder* builder, const AffixFuzzer7* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - auto builder = std::make_shared(memory_pool); - return builder; + return arrow::Status::NotImplemented( + "TODO(andreas): custom data types in lists/fixedsizelist are not yet implemented" + ); + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer7.hpp b/rerun_cpp/src/components/affix_fuzzer7.hpp index b4b2905575fa..1e9cd32bef98 100644 --- a/rerun_cpp/src/components/affix_fuzzer7.hpp +++ b/rerun_cpp/src/components/affix_fuzzer7.hpp @@ -5,19 +5,12 @@ #include "../datatypes/affix_fuzzer1.hpp" -#include +#include #include -#include #include #include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace components { struct AffixFuzzer7 { @@ -30,9 +23,15 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer7* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::ListBuilder* builder, const AffixFuzzer7* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer8.cpp b/rerun_cpp/src/components/affix_fuzzer8.cpp index 91d9e4ac6ee1..2ac5850087c3 100644 --- a/rerun_cpp/src/components/affix_fuzzer8.cpp +++ b/rerun_cpp/src/components/affix_fuzzer8.cpp @@ -11,17 +11,37 @@ namespace rr { return arrow::float32(); } - arrow::Result> AffixFuzzer8::to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer8* elements, size_t num_elements) { + arrow::Result> AffixFuzzer8::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result(std::make_shared(memory_pool)); + } + + arrow::Status AffixFuzzer8::fill_arrow_array_builder( + arrow::FloatBuilder* builder, const AffixFuzzer8* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - auto builder = std::make_shared(memory_pool); - return builder; + ARROW_RETURN_NOT_OK(builder->Reserve(num_elements)); + for (auto elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto& element = elements[elem_idx]; + if (element.single_float_optional.has_value()) { + ARROW_RETURN_NOT_OK(builder->Append(element.single_float_optional.value())); + } else { + ARROW_RETURN_NOT_OK(builder->AppendNull()); + } + } + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer8.hpp b/rerun_cpp/src/components/affix_fuzzer8.hpp index a9ef965c0203..63a0925591f3 100644 --- a/rerun_cpp/src/components/affix_fuzzer8.hpp +++ b/rerun_cpp/src/components/affix_fuzzer8.hpp @@ -3,18 +3,11 @@ #pragma once -#include +#include #include -#include #include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace components { struct AffixFuzzer8 { @@ -27,9 +20,15 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer8* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::FloatBuilder* builder, const AffixFuzzer8* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer9.cpp b/rerun_cpp/src/components/affix_fuzzer9.cpp index 43482faa1f71..815fb20a74dc 100644 --- a/rerun_cpp/src/components/affix_fuzzer9.cpp +++ b/rerun_cpp/src/components/affix_fuzzer9.cpp @@ -11,17 +11,32 @@ namespace rr { return arrow::utf8(); } - arrow::Result> AffixFuzzer9::to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer9* elements, size_t num_elements) { + arrow::Result> AffixFuzzer9::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result(std::make_shared(memory_pool)); + } + + arrow::Status AffixFuzzer9::fill_arrow_array_builder( + arrow::StringBuilder* builder, const AffixFuzzer9* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - auto builder = std::make_shared(memory_pool); - return builder; + ARROW_RETURN_NOT_OK(builder->Reserve(num_elements)); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + ARROW_RETURN_NOT_OK(builder->Append(elements[elem_idx].single_string_required)); + } + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/affix_fuzzer9.hpp b/rerun_cpp/src/components/affix_fuzzer9.hpp index 3fb7fa0150c8..c65fb7d5f5a3 100644 --- a/rerun_cpp/src/components/affix_fuzzer9.hpp +++ b/rerun_cpp/src/components/affix_fuzzer9.hpp @@ -3,18 +3,11 @@ #pragma once -#include +#include #include -#include #include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace components { struct AffixFuzzer9 { @@ -27,9 +20,15 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer9* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StringBuilder* builder, const AffixFuzzer9* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/class_id.cpp b/rerun_cpp/src/components/class_id.cpp index 2a9e2ad59bff..91e5835cfc5f 100644 --- a/rerun_cpp/src/components/class_id.cpp +++ b/rerun_cpp/src/components/class_id.cpp @@ -11,17 +11,30 @@ namespace rr { return arrow::uint16(); } - arrow::Result> ClassId::to_arrow( - arrow::MemoryPool* memory_pool, const ClassId* elements, size_t num_elements) { + arrow::Result> ClassId::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result(std::make_shared(memory_pool)); + } + + arrow::Status ClassId::fill_arrow_array_builder( + arrow::UInt16Builder* builder, const ClassId* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - auto builder = std::make_shared(memory_pool); - return builder; + static_assert(sizeof(*elements) == sizeof(elements->id)); + ARROW_RETURN_NOT_OK(builder->AppendValues(&elements->id, num_elements)); + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/class_id.hpp b/rerun_cpp/src/components/class_id.hpp index e16885dc6693..7836c98f0077 100644 --- a/rerun_cpp/src/components/class_id.hpp +++ b/rerun_cpp/src/components/class_id.hpp @@ -3,17 +3,10 @@ #pragma once -#include +#include #include -#include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace components { /// A 16-bit ID representing a type of semantic class. @@ -26,9 +19,15 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const ClassId* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::UInt16Builder* builder, const ClassId* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/color.cpp b/rerun_cpp/src/components/color.cpp index a42c4d39d6d1..a40b178607e2 100644 --- a/rerun_cpp/src/components/color.cpp +++ b/rerun_cpp/src/components/color.cpp @@ -11,17 +11,30 @@ namespace rr { return arrow::uint32(); } - arrow::Result> Color::to_arrow( - arrow::MemoryPool* memory_pool, const Color* elements, size_t num_elements) { + arrow::Result> Color::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result(std::make_shared(memory_pool)); + } + + arrow::Status Color::fill_arrow_array_builder( + arrow::UInt32Builder* builder, const Color* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - auto builder = std::make_shared(memory_pool); - return builder; + static_assert(sizeof(*elements) == sizeof(elements->rgba)); + ARROW_RETURN_NOT_OK(builder->AppendValues(&elements->rgba, num_elements)); + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/color.hpp b/rerun_cpp/src/components/color.hpp index ffe3fce5cbea..0210c902283a 100644 --- a/rerun_cpp/src/components/color.hpp +++ b/rerun_cpp/src/components/color.hpp @@ -3,17 +3,10 @@ #pragma once -#include +#include #include -#include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace components { /// An RGBA color tuple with unmultiplied/separate alpha, in sRGB gamma space with linear @@ -27,9 +20,15 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const Color* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::UInt32Builder* builder, const Color* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/draw_order.cpp b/rerun_cpp/src/components/draw_order.cpp index 39efbcc9e3b4..fdd4b8d7922b 100644 --- a/rerun_cpp/src/components/draw_order.cpp +++ b/rerun_cpp/src/components/draw_order.cpp @@ -11,17 +11,30 @@ namespace rr { return arrow::float32(); } - arrow::Result> DrawOrder::to_arrow( - arrow::MemoryPool* memory_pool, const DrawOrder* elements, size_t num_elements) { + arrow::Result> DrawOrder::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result(std::make_shared(memory_pool)); + } + + arrow::Status DrawOrder::fill_arrow_array_builder( + arrow::FloatBuilder* builder, const DrawOrder* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - auto builder = std::make_shared(memory_pool); - return builder; + static_assert(sizeof(*elements) == sizeof(elements->value)); + ARROW_RETURN_NOT_OK(builder->AppendValues(&elements->value, num_elements)); + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/draw_order.hpp b/rerun_cpp/src/components/draw_order.hpp index a3e9fc586364..417d4aad965d 100644 --- a/rerun_cpp/src/components/draw_order.hpp +++ b/rerun_cpp/src/components/draw_order.hpp @@ -3,17 +3,10 @@ #pragma once -#include +#include #include -#include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace components { /// Draw order used for the display order of 2D elements. @@ -32,9 +25,15 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const DrawOrder* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::FloatBuilder* builder, const DrawOrder* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/instance_key.cpp b/rerun_cpp/src/components/instance_key.cpp index 3de7de491f2f..1c428100087f 100644 --- a/rerun_cpp/src/components/instance_key.cpp +++ b/rerun_cpp/src/components/instance_key.cpp @@ -11,17 +11,30 @@ namespace rr { return arrow::uint64(); } - arrow::Result> InstanceKey::to_arrow( - arrow::MemoryPool* memory_pool, const InstanceKey* elements, size_t num_elements) { + arrow::Result> InstanceKey::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result(std::make_shared(memory_pool)); + } + + arrow::Status InstanceKey::fill_arrow_array_builder( + arrow::UInt64Builder* builder, const InstanceKey* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - auto builder = std::make_shared(memory_pool); - return builder; + static_assert(sizeof(*elements) == sizeof(elements->value)); + ARROW_RETURN_NOT_OK(builder->AppendValues(&elements->value, num_elements)); + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/instance_key.hpp b/rerun_cpp/src/components/instance_key.hpp index f1ad7ea5245a..fcae04055fed 100644 --- a/rerun_cpp/src/components/instance_key.hpp +++ b/rerun_cpp/src/components/instance_key.hpp @@ -3,17 +3,10 @@ #pragma once -#include +#include #include -#include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace components { /// A unique numeric identifier for each individual instance within a batch. @@ -26,9 +19,15 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const InstanceKey* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::UInt64Builder* builder, const InstanceKey* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/keypoint_id.cpp b/rerun_cpp/src/components/keypoint_id.cpp index 89642da4b68f..6871f4989368 100644 --- a/rerun_cpp/src/components/keypoint_id.cpp +++ b/rerun_cpp/src/components/keypoint_id.cpp @@ -11,17 +11,30 @@ namespace rr { return arrow::uint16(); } - arrow::Result> KeypointId::to_arrow( - arrow::MemoryPool* memory_pool, const KeypointId* elements, size_t num_elements) { + arrow::Result> KeypointId::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result(std::make_shared(memory_pool)); + } + + arrow::Status KeypointId::fill_arrow_array_builder( + arrow::UInt16Builder* builder, const KeypointId* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - auto builder = std::make_shared(memory_pool); - return builder; + static_assert(sizeof(*elements) == sizeof(elements->id)); + ARROW_RETURN_NOT_OK(builder->AppendValues(&elements->id, num_elements)); + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/keypoint_id.hpp b/rerun_cpp/src/components/keypoint_id.hpp index 9d9839155a72..ff29000da457 100644 --- a/rerun_cpp/src/components/keypoint_id.hpp +++ b/rerun_cpp/src/components/keypoint_id.hpp @@ -3,17 +3,10 @@ #pragma once -#include +#include #include -#include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace components { /// A 16-bit ID representing a type of semantic keypoint within a class. @@ -26,9 +19,15 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const KeypointId* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::UInt16Builder* builder, const KeypointId* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/label.cpp b/rerun_cpp/src/components/label.cpp index 2ae584f899ea..f2c1fb409f78 100644 --- a/rerun_cpp/src/components/label.cpp +++ b/rerun_cpp/src/components/label.cpp @@ -11,17 +11,32 @@ namespace rr { return arrow::utf8(); } - arrow::Result> Label::to_arrow( - arrow::MemoryPool* memory_pool, const Label* elements, size_t num_elements) { + arrow::Result> Label::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result(std::make_shared(memory_pool)); + } + + arrow::Status Label::fill_arrow_array_builder( + arrow::StringBuilder* builder, const Label* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - auto builder = std::make_shared(memory_pool); - return builder; + ARROW_RETURN_NOT_OK(builder->Reserve(num_elements)); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + ARROW_RETURN_NOT_OK(builder->Append(elements[elem_idx].value)); + } + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/label.hpp b/rerun_cpp/src/components/label.hpp index bb8b4efe74b9..297343f68a6d 100644 --- a/rerun_cpp/src/components/label.hpp +++ b/rerun_cpp/src/components/label.hpp @@ -3,18 +3,11 @@ #pragma once -#include +#include #include -#include #include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace components { /// A String label component. @@ -27,9 +20,15 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const Label* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StringBuilder* builder, const Label* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/point2d.cpp b/rerun_cpp/src/components/point2d.cpp index e951cfeda2c2..d55ee3798564 100644 --- a/rerun_cpp/src/components/point2d.cpp +++ b/rerun_cpp/src/components/point2d.cpp @@ -13,22 +13,36 @@ namespace rr { return rr::datatypes::Point2D::to_arrow_datatype(); } - arrow::Result> Point2D::to_arrow( - arrow::MemoryPool *memory_pool, const Point2D *elements, size_t num_elements) { + arrow::Result> Point2D::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result( + rr::datatypes::Point2D::new_arrow_array_builder(memory_pool).ValueOrDie() + ); + } + + arrow::Status Point2D::fill_arrow_array_builder( + arrow::StructBuilder *builder, const Point2D *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - static_assert(sizeof(Point2D) == sizeof(rr::datatypes::Point2D), - "Expected fully transparent type."); - auto builder = rr::datatypes::Point2D::to_arrow( - memory_pool, + static_assert(sizeof(rr::datatypes::Point2D) == sizeof(Point2D)); + ARROW_RETURN_NOT_OK(rr::datatypes::Point2D::fill_arrow_array_builder( + builder, reinterpret_cast(elements), - num_elements); - return builder; + num_elements + )); + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/point2d.hpp b/rerun_cpp/src/components/point2d.hpp index fabdc2a54a5f..c729d8478b76 100644 --- a/rerun_cpp/src/components/point2d.hpp +++ b/rerun_cpp/src/components/point2d.hpp @@ -5,17 +5,10 @@ #include "../datatypes/point2d.hpp" -#include +#include #include -#include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace components { /// A point in 2D space. @@ -28,9 +21,15 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const Point2D* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const Point2D* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/point3d.cpp b/rerun_cpp/src/components/point3d.cpp index 9bf9e0f5a244..3566bac48ae0 100644 --- a/rerun_cpp/src/components/point3d.cpp +++ b/rerun_cpp/src/components/point3d.cpp @@ -13,22 +13,36 @@ namespace rr { return rr::datatypes::Point3D::to_arrow_datatype(); } - arrow::Result> Point3D::to_arrow( - arrow::MemoryPool *memory_pool, const Point3D *elements, size_t num_elements) { + arrow::Result> Point3D::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result( + rr::datatypes::Point3D::new_arrow_array_builder(memory_pool).ValueOrDie() + ); + } + + arrow::Status Point3D::fill_arrow_array_builder( + arrow::StructBuilder *builder, const Point3D *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - static_assert(sizeof(Point3D) == sizeof(rr::datatypes::Point3D), - "Expected fully transparent type."); - auto builder = rr::datatypes::Point3D::to_arrow( - memory_pool, + static_assert(sizeof(rr::datatypes::Point3D) == sizeof(Point3D)); + ARROW_RETURN_NOT_OK(rr::datatypes::Point3D::fill_arrow_array_builder( + builder, reinterpret_cast(elements), - num_elements); - return builder; + num_elements + )); + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/point3d.hpp b/rerun_cpp/src/components/point3d.hpp index 166f64dbad75..72e196ff194a 100644 --- a/rerun_cpp/src/components/point3d.hpp +++ b/rerun_cpp/src/components/point3d.hpp @@ -5,17 +5,10 @@ #include "../datatypes/point3d.hpp" -#include +#include #include -#include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace components { /// A point in 3D space. @@ -28,9 +21,15 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const Point3D* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const Point3D* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/radius.cpp b/rerun_cpp/src/components/radius.cpp index f73f437f6708..30980e6119a6 100644 --- a/rerun_cpp/src/components/radius.cpp +++ b/rerun_cpp/src/components/radius.cpp @@ -11,17 +11,30 @@ namespace rr { return arrow::float32(); } - arrow::Result> Radius::to_arrow( - arrow::MemoryPool* memory_pool, const Radius* elements, size_t num_elements) { + arrow::Result> Radius::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result(std::make_shared(memory_pool)); + } + + arrow::Status Radius::fill_arrow_array_builder( + arrow::FloatBuilder* builder, const Radius* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - auto builder = std::make_shared(memory_pool); - return builder; + static_assert(sizeof(*elements) == sizeof(elements->value)); + ARROW_RETURN_NOT_OK(builder->AppendValues(&elements->value, num_elements)); + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/radius.hpp b/rerun_cpp/src/components/radius.hpp index 6ab8800695dc..12841b6c5240 100644 --- a/rerun_cpp/src/components/radius.hpp +++ b/rerun_cpp/src/components/radius.hpp @@ -3,17 +3,10 @@ #pragma once -#include +#include #include -#include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace components { /// A Radius component. @@ -26,9 +19,15 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const Radius* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::FloatBuilder* builder, const Radius* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/transform3d.cpp b/rerun_cpp/src/components/transform3d.cpp index 085796a837d7..8678683c86a1 100644 --- a/rerun_cpp/src/components/transform3d.cpp +++ b/rerun_cpp/src/components/transform3d.cpp @@ -13,22 +13,35 @@ namespace rr { return rr::datatypes::Transform3D::to_arrow_datatype(); } - arrow::Result> Transform3D::to_arrow( - arrow::MemoryPool *memory_pool, const Transform3D *elements, size_t num_elements) { + arrow::Result> + Transform3D::new_arrow_array_builder(arrow::MemoryPool *memory_pool) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result( + rr::datatypes::Transform3D::new_arrow_array_builder(memory_pool).ValueOrDie() + ); + } + + arrow::Status Transform3D::fill_arrow_array_builder( + arrow::DenseUnionBuilder *builder, const Transform3D *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - static_assert(sizeof(Transform3D) == sizeof(rr::datatypes::Transform3D), - "Expected fully transparent type."); - auto builder = rr::datatypes::Transform3D::to_arrow( - memory_pool, + static_assert(sizeof(rr::datatypes::Transform3D) == sizeof(Transform3D)); + ARROW_RETURN_NOT_OK(rr::datatypes::Transform3D::fill_arrow_array_builder( + builder, reinterpret_cast(elements), - num_elements); - return builder; + num_elements + )); + + return arrow::Status::OK(); } } // namespace components } // namespace rr diff --git a/rerun_cpp/src/components/transform3d.hpp b/rerun_cpp/src/components/transform3d.hpp index e8e3e3060d08..0853d1bc7b2d 100644 --- a/rerun_cpp/src/components/transform3d.hpp +++ b/rerun_cpp/src/components/transform3d.hpp @@ -5,17 +5,10 @@ #include "../datatypes/transform3d.hpp" -#include +#include #include -#include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace components { /// An affine transform between two 3D spaces, represented in a given direction. @@ -29,9 +22,15 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const Transform3D* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::DenseUnionBuilder* builder, const Transform3D* elements, size_t num_elements + ); }; } // namespace components } // namespace rr diff --git a/rerun_cpp/src/datatypes/affix_fuzzer1.cpp b/rerun_cpp/src/datatypes/affix_fuzzer1.cpp index 619e67ddf23a..c5423b18c8bc 100644 --- a/rerun_cpp/src/datatypes/affix_fuzzer1.cpp +++ b/rerun_cpp/src/datatypes/affix_fuzzer1.cpp @@ -14,42 +14,156 @@ namespace rr { arrow::field("single_float_optional", arrow::float32(), true, nullptr), arrow::field("single_string_required", arrow::utf8(), false, nullptr), arrow::field("single_string_optional", arrow::utf8(), true, nullptr), - arrow::field("many_floats_optional", - arrow::list(arrow::field("item", arrow::float32(), true, nullptr)), - true, - nullptr), - arrow::field("many_strings_required", - arrow::list(arrow::field("item", arrow::utf8(), false, nullptr)), - false, - nullptr), - arrow::field("many_strings_optional", - arrow::list(arrow::field("item", arrow::utf8(), true, nullptr)), - true, - nullptr), + arrow::field( + "many_floats_optional", + arrow::list(arrow::field("item", arrow::float32(), true, nullptr)), + true, + nullptr + ), + arrow::field( + "many_strings_required", + arrow::list(arrow::field("item", arrow::utf8(), false, nullptr)), + false, + nullptr + ), + arrow::field( + "many_strings_optional", + arrow::list(arrow::field("item", arrow::utf8(), true, nullptr)), + true, + nullptr + ), arrow::field("flattened_scalar", arrow::float32(), false, nullptr), - arrow::field("almost_flattened_scalar", - rr::datatypes::FlattenedScalar::to_arrow_datatype(), - false, - nullptr), + arrow::field( + "almost_flattened_scalar", + rr::datatypes::FlattenedScalar::to_arrow_datatype(), + false, + nullptr + ), arrow::field("from_parent", arrow::boolean(), true, nullptr), }); } - arrow::Result> AffixFuzzer1::to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer1* elements, size_t num_elements) { + arrow::Result> AffixFuzzer1::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result(std::make_shared( + to_arrow_datatype(), + memory_pool, + std::vector>({ + std::make_shared(memory_pool), + std::make_shared(memory_pool), + std::make_shared(memory_pool), + std::make_shared( + memory_pool, + std::make_shared(memory_pool) + ), + std::make_shared( + memory_pool, + std::make_shared(memory_pool) + ), + std::make_shared( + memory_pool, + std::make_shared(memory_pool) + ), + std::make_shared(memory_pool), + rr::datatypes::FlattenedScalar::new_arrow_array_builder(memory_pool) + .ValueOrDie(), + std::make_shared(memory_pool), + }) + )); + } + + arrow::Status AffixFuzzer1::fill_arrow_array_builder( + arrow::StructBuilder *builder, const AffixFuzzer1 *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - auto datatype = AffixFuzzer1::to_arrow_datatype(); - let builder = - std::make_shared(datatype, memory_pool, {}, - // TODO(#2647): code-gen for C++ - ); - return builder; + { + auto element_builder = + static_cast(builder->field_builder(0)); + ARROW_RETURN_NOT_OK(element_builder->Reserve(num_elements)); + for (auto elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto &element = elements[elem_idx]; + if (element.single_float_optional.has_value()) { + ARROW_RETURN_NOT_OK( + element_builder->Append(element.single_float_optional.value()) + ); + } else { + ARROW_RETURN_NOT_OK(element_builder->AppendNull()); + } + } + } + { + auto element_builder = + static_cast(builder->field_builder(1)); + ARROW_RETURN_NOT_OK(element_builder->Reserve(num_elements)); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + ARROW_RETURN_NOT_OK( + element_builder->Append(elements[elem_idx].single_string_required) + ); + } + } + { + auto element_builder = + static_cast(builder->field_builder(2)); + ARROW_RETURN_NOT_OK(element_builder->Reserve(num_elements)); + for (auto elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto &element = elements[elem_idx]; + if (element.single_string_optional.has_value()) { + ARROW_RETURN_NOT_OK( + element_builder->Append(element.single_string_optional.value()) + ); + } else { + ARROW_RETURN_NOT_OK(element_builder->AppendNull()); + } + } + } + return arrow::Status::NotImplemented( + "TODO(andreas): lists in structs are not yet supported" + ); + return arrow::Status::NotImplemented( + "TODO(andreas): lists in structs are not yet supported" + ); + return arrow::Status::NotImplemented( + "TODO(andreas): lists in structs are not yet supported" + ); + { + auto element_builder = + static_cast(builder->field_builder(6)); + ARROW_RETURN_NOT_OK(element_builder->Reserve(num_elements)); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + ARROW_RETURN_NOT_OK(element_builder->Append(elements[elem_idx].flattened_scalar) + ); + } + } + return arrow::Status::NotImplemented( + "TODO(andreas): extensions in structs are not yet supported" + ); + { + auto element_builder = + static_cast(builder->field_builder(8)); + ARROW_RETURN_NOT_OK(element_builder->Reserve(num_elements)); + for (auto elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto &element = elements[elem_idx]; + if (element.from_parent.has_value()) { + ARROW_RETURN_NOT_OK(element_builder->Append(element.from_parent.value())); + } else { + ARROW_RETURN_NOT_OK(element_builder->AppendNull()); + } + } + } + ARROW_RETURN_NOT_OK(builder->AppendValues(num_elements, nullptr)); + + return arrow::Status::OK(); } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/affix_fuzzer1.hpp b/rerun_cpp/src/datatypes/affix_fuzzer1.hpp index ed8c62a4cdcf..6626e3314f8b 100644 --- a/rerun_cpp/src/datatypes/affix_fuzzer1.hpp +++ b/rerun_cpp/src/datatypes/affix_fuzzer1.hpp @@ -5,19 +5,12 @@ #include "../datatypes/flattened_scalar.hpp" -#include +#include #include -#include #include #include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace datatypes { struct AffixFuzzer1 { @@ -43,9 +36,15 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer1* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const AffixFuzzer1* elements, size_t num_elements + ); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/affix_fuzzer2.cpp b/rerun_cpp/src/datatypes/affix_fuzzer2.cpp index 35ee82382f52..20a26bd384b8 100644 --- a/rerun_cpp/src/datatypes/affix_fuzzer2.cpp +++ b/rerun_cpp/src/datatypes/affix_fuzzer2.cpp @@ -11,17 +11,37 @@ namespace rr { return arrow::float32(); } - arrow::Result> AffixFuzzer2::to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer2* elements, size_t num_elements) { + arrow::Result> AffixFuzzer2::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result(std::make_shared(memory_pool)); + } + + arrow::Status AffixFuzzer2::fill_arrow_array_builder( + arrow::FloatBuilder* builder, const AffixFuzzer2* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - auto builder = std::make_shared(memory_pool); - return builder; + ARROW_RETURN_NOT_OK(builder->Reserve(num_elements)); + for (auto elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto& element = elements[elem_idx]; + if (element.single_float_optional.has_value()) { + ARROW_RETURN_NOT_OK(builder->Append(element.single_float_optional.value())); + } else { + ARROW_RETURN_NOT_OK(builder->AppendNull()); + } + } + + return arrow::Status::OK(); } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/affix_fuzzer2.hpp b/rerun_cpp/src/datatypes/affix_fuzzer2.hpp index a79eaa88cd1e..b0c9d0ca3dd2 100644 --- a/rerun_cpp/src/datatypes/affix_fuzzer2.hpp +++ b/rerun_cpp/src/datatypes/affix_fuzzer2.hpp @@ -3,18 +3,11 @@ #pragma once -#include +#include #include -#include #include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace datatypes { struct AffixFuzzer2 { @@ -27,9 +20,15 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer2* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::FloatBuilder* builder, const AffixFuzzer2* elements, size_t num_elements + ); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/affix_fuzzer3.cpp b/rerun_cpp/src/datatypes/affix_fuzzer3.cpp index f84a05cbc457..d76727c7be3c 100644 --- a/rerun_cpp/src/datatypes/affix_fuzzer3.cpp +++ b/rerun_cpp/src/datatypes/affix_fuzzer3.cpp @@ -17,15 +17,69 @@ namespace rr { arrow::field( "craziness", arrow::list(arrow::field( - "item", rr::datatypes::AffixFuzzer1::to_arrow_datatype(), false, nullptr)), + "item", + rr::datatypes::AffixFuzzer1::to_arrow_datatype(), + false, + nullptr + )), false, - nullptr), - arrow::field("fixed_size_shenanigans", - arrow::fixed_size_list( - arrow::field("item", arrow::float32(), false, nullptr), 3), - false, - nullptr), + nullptr + ), + arrow::field( + "fixed_size_shenanigans", + arrow::fixed_size_list( + arrow::field("item", arrow::float32(), false, nullptr), + 3 + ), + false, + nullptr + ), }); } + + arrow::Result> + AffixFuzzer3::new_arrow_array_builder(arrow::MemoryPool* memory_pool) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + memory_pool, + std::vector>({ + std::make_shared(memory_pool), + std::make_shared(memory_pool), + std::make_shared(memory_pool), + std::make_shared( + memory_pool, + rr::datatypes::AffixFuzzer1::new_arrow_array_builder(memory_pool) + .ValueOrDie() + ), + std::make_shared( + memory_pool, + std::make_shared(memory_pool), + 3 + ), + }), + to_arrow_datatype() + )); + } + + arrow::Status AffixFuzzer3::fill_arrow_array_builder( + arrow::DenseUnionBuilder* builder, const AffixFuzzer3* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto& element = elements[elem_idx]; + } + return arrow::Status::NotImplemented("TODO(andreas): unions are not yet implemented"); + + return arrow::Status::OK(); + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/affix_fuzzer3.hpp b/rerun_cpp/src/datatypes/affix_fuzzer3.hpp index e5cbabd1f772..fd12d4917c96 100644 --- a/rerun_cpp/src/datatypes/affix_fuzzer3.hpp +++ b/rerun_cpp/src/datatypes/affix_fuzzer3.hpp @@ -5,9 +5,9 @@ #include "../datatypes/affix_fuzzer1.hpp" +#include #include #include -#include #include #include #include @@ -125,6 +125,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::DenseUnionBuilder* builder, const AffixFuzzer3* elements, size_t num_elements + ); + void swap(AffixFuzzer3& other) noexcept { auto tag_temp = this->_tag; this->_tag = other._tag; diff --git a/rerun_cpp/src/datatypes/affix_fuzzer4.cpp b/rerun_cpp/src/datatypes/affix_fuzzer4.cpp index ca241ac777a3..e39e3fbd3a96 100644 --- a/rerun_cpp/src/datatypes/affix_fuzzer4.cpp +++ b/rerun_cpp/src/datatypes/affix_fuzzer4.cpp @@ -12,23 +12,79 @@ namespace rr { std::shared_ptr AffixFuzzer4::to_arrow_datatype() { return arrow::dense_union({ arrow::field("_null_markers", arrow::null(), true, nullptr), - arrow::field("single_required", - rr::datatypes::AffixFuzzer3::to_arrow_datatype(), - false, - nullptr), + arrow::field( + "single_required", + rr::datatypes::AffixFuzzer3::to_arrow_datatype(), + false, + nullptr + ), arrow::field( "many_required", arrow::list(arrow::field( - "item", rr::datatypes::AffixFuzzer3::to_arrow_datatype(), false, nullptr)), + "item", + rr::datatypes::AffixFuzzer3::to_arrow_datatype(), + false, + nullptr + )), false, - nullptr), + nullptr + ), arrow::field( "many_optional", arrow::list(arrow::field( - "item", rr::datatypes::AffixFuzzer3::to_arrow_datatype(), true, nullptr)), + "item", + rr::datatypes::AffixFuzzer3::to_arrow_datatype(), + true, + nullptr + )), false, - nullptr), + nullptr + ), }); } + + arrow::Result> + AffixFuzzer4::new_arrow_array_builder(arrow::MemoryPool* memory_pool) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + memory_pool, + std::vector>({ + std::make_shared(memory_pool), + rr::datatypes::AffixFuzzer3::new_arrow_array_builder(memory_pool).ValueOrDie(), + std::make_shared( + memory_pool, + rr::datatypes::AffixFuzzer3::new_arrow_array_builder(memory_pool) + .ValueOrDie() + ), + std::make_shared( + memory_pool, + rr::datatypes::AffixFuzzer3::new_arrow_array_builder(memory_pool) + .ValueOrDie() + ), + }), + to_arrow_datatype() + )); + } + + arrow::Status AffixFuzzer4::fill_arrow_array_builder( + arrow::DenseUnionBuilder* builder, const AffixFuzzer4* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto& element = elements[elem_idx]; + } + return arrow::Status::NotImplemented("TODO(andreas): unions are not yet implemented"); + + return arrow::Status::OK(); + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/affix_fuzzer4.hpp b/rerun_cpp/src/datatypes/affix_fuzzer4.hpp index cb60e5e9a36e..2c1fc11f799e 100644 --- a/rerun_cpp/src/datatypes/affix_fuzzer4.hpp +++ b/rerun_cpp/src/datatypes/affix_fuzzer4.hpp @@ -5,9 +5,9 @@ #include "../datatypes/affix_fuzzer3.hpp" +#include #include #include -#include #include #include #include @@ -96,8 +96,8 @@ namespace rr { return std::move(self); } - static AffixFuzzer4 many_required( - std::vector many_required) { + static AffixFuzzer4 many_required(std::vector many_required + ) { typedef std::vector TypeAlias; AffixFuzzer4 self; self._tag = detail::AffixFuzzer4Tag::many_required; @@ -106,7 +106,8 @@ namespace rr { } static AffixFuzzer4 many_optional( - std::optional> many_optional) { + std::optional> many_optional + ) { typedef std::optional> TypeAlias; AffixFuzzer4 self; self._tag = detail::AffixFuzzer4Tag::many_optional; @@ -117,6 +118,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::DenseUnionBuilder* builder, const AffixFuzzer4* elements, size_t num_elements + ); + void swap(AffixFuzzer4& other) noexcept { auto tag_temp = this->_tag; this->_tag = other._tag; diff --git a/rerun_cpp/src/datatypes/affix_fuzzer5.cpp b/rerun_cpp/src/datatypes/affix_fuzzer5.cpp index e592df705313..bcf69f78142b 100644 --- a/rerun_cpp/src/datatypes/affix_fuzzer5.cpp +++ b/rerun_cpp/src/datatypes/affix_fuzzer5.cpp @@ -11,28 +11,47 @@ namespace rr { namespace datatypes { std::shared_ptr AffixFuzzer5::to_arrow_datatype() { return arrow::struct_({ - arrow::field("single_optional_union", - rr::datatypes::AffixFuzzer4::to_arrow_datatype(), - true, - nullptr), + arrow::field( + "single_optional_union", + rr::datatypes::AffixFuzzer4::to_arrow_datatype(), + true, + nullptr + ), }); } - arrow::Result> AffixFuzzer5::to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer5* elements, size_t num_elements) { + arrow::Result> AffixFuzzer5::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result(std::make_shared( + to_arrow_datatype(), + memory_pool, + std::vector>({ + rr::datatypes::AffixFuzzer4::new_arrow_array_builder(memory_pool).ValueOrDie(), + }) + )); + } + + arrow::Status AffixFuzzer5::fill_arrow_array_builder( + arrow::StructBuilder* builder, const AffixFuzzer5* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - auto datatype = AffixFuzzer5::to_arrow_datatype(); - let builder = - std::make_shared(datatype, memory_pool, {}, - // TODO(#2647): code-gen for C++ - ); - return builder; + return arrow::Status::NotImplemented( + "TODO(andreas): extensions in structs are not yet supported" + ); + ARROW_RETURN_NOT_OK(builder->AppendValues(num_elements, nullptr)); + + return arrow::Status::OK(); } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/affix_fuzzer5.hpp b/rerun_cpp/src/datatypes/affix_fuzzer5.hpp index f829aa48381c..b68bcf488589 100644 --- a/rerun_cpp/src/datatypes/affix_fuzzer5.hpp +++ b/rerun_cpp/src/datatypes/affix_fuzzer5.hpp @@ -5,18 +5,11 @@ #include "../datatypes/affix_fuzzer4.hpp" -#include +#include #include -#include #include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace datatypes { struct AffixFuzzer5 { @@ -29,9 +22,15 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const AffixFuzzer5* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const AffixFuzzer5* elements, size_t num_elements + ); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/angle.cpp b/rerun_cpp/src/datatypes/angle.cpp index 25517f5efcb3..7371cebbc935 100644 --- a/rerun_cpp/src/datatypes/angle.cpp +++ b/rerun_cpp/src/datatypes/angle.cpp @@ -14,5 +14,41 @@ namespace rr { arrow::field("Degrees", arrow::float32(), false, nullptr), }); } + + arrow::Result> Angle::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + memory_pool, + std::vector>({ + std::make_shared(memory_pool), + std::make_shared(memory_pool), + std::make_shared(memory_pool), + }), + to_arrow_datatype() + )); + } + + arrow::Status Angle::fill_arrow_array_builder( + arrow::DenseUnionBuilder* builder, const Angle* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto& element = elements[elem_idx]; + } + return arrow::Status::NotImplemented("TODO(andreas): unions are not yet implemented"); + + return arrow::Status::OK(); + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/angle.hpp b/rerun_cpp/src/datatypes/angle.hpp index 26d880318fec..59bf5a92cd63 100644 --- a/rerun_cpp/src/datatypes/angle.hpp +++ b/rerun_cpp/src/datatypes/angle.hpp @@ -3,9 +3,9 @@ #pragma once +#include #include #include -#include #include namespace rr { @@ -75,6 +75,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::DenseUnionBuilder* builder, const Angle* elements, size_t num_elements + ); + void swap(Angle& other) noexcept { auto tag_temp = this->_tag; this->_tag = other._tag; diff --git a/rerun_cpp/src/datatypes/flattened_scalar.cpp b/rerun_cpp/src/datatypes/flattened_scalar.cpp index 1eba191539f2..1fa7c7b7e034 100644 --- a/rerun_cpp/src/datatypes/flattened_scalar.cpp +++ b/rerun_cpp/src/datatypes/flattened_scalar.cpp @@ -13,21 +13,42 @@ namespace rr { }); } - arrow::Result> FlattenedScalar::to_arrow( - arrow::MemoryPool* memory_pool, const FlattenedScalar* elements, size_t num_elements) { + arrow::Result> + FlattenedScalar::new_arrow_array_builder(arrow::MemoryPool *memory_pool) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result(std::make_shared( + to_arrow_datatype(), + memory_pool, + std::vector>({ + std::make_shared(memory_pool), + }) + )); + } + + arrow::Status FlattenedScalar::fill_arrow_array_builder( + arrow::StructBuilder *builder, const FlattenedScalar *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - auto datatype = FlattenedScalar::to_arrow_datatype(); - let builder = - std::make_shared(datatype, memory_pool, {}, - // TODO(#2647): code-gen for C++ - ); - return builder; + { + auto element_builder = + static_cast(builder->field_builder(0)); + ARROW_RETURN_NOT_OK(element_builder->Reserve(num_elements)); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + ARROW_RETURN_NOT_OK(element_builder->Append(elements[elem_idx].value)); + } + } + ARROW_RETURN_NOT_OK(builder->AppendValues(num_elements, nullptr)); + + return arrow::Status::OK(); } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/flattened_scalar.hpp b/rerun_cpp/src/datatypes/flattened_scalar.hpp index b7e4b52016e0..6dcd047a807c 100644 --- a/rerun_cpp/src/datatypes/flattened_scalar.hpp +++ b/rerun_cpp/src/datatypes/flattened_scalar.hpp @@ -3,17 +3,10 @@ #pragma once -#include +#include #include -#include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace datatypes { struct FlattenedScalar { @@ -25,10 +18,15 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const FlattenedScalar* elements, - size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const FlattenedScalar* elements, size_t num_elements + ); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/mat3x3.cpp b/rerun_cpp/src/datatypes/mat3x3.cpp index 61d9ab50d838..4ef61f626513 100644 --- a/rerun_cpp/src/datatypes/mat3x3.cpp +++ b/rerun_cpp/src/datatypes/mat3x3.cpp @@ -8,21 +8,45 @@ namespace rr { namespace datatypes { std::shared_ptr Mat3x3::to_arrow_datatype() { - return arrow::fixed_size_list(arrow::field("item", arrow::float32(), false, nullptr), - 9); + return arrow::fixed_size_list( + arrow::field("item", arrow::float32(), false, nullptr), + 9 + ); } - arrow::Result> Mat3x3::to_arrow( - arrow::MemoryPool* memory_pool, const Mat3x3* elements, size_t num_elements) { + arrow::Result> Mat3x3::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result(std::make_shared( + memory_pool, + std::make_shared(memory_pool), + 9 + )); + } + + arrow::Status Mat3x3::fill_arrow_array_builder( + arrow::FixedSizeListBuilder *builder, const Mat3x3 *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - auto builder = std::make_shared(memory_pool); - return builder; + auto value_builder = static_cast(builder->value_builder()); + + static_assert(sizeof(elements[0].coeffs) == sizeof(elements[0])); + ARROW_RETURN_NOT_OK( + value_builder->AppendValues(elements[0].coeffs, num_elements * 9, nullptr) + ); + ARROW_RETURN_NOT_OK(builder->AppendValues(num_elements)); + + return arrow::Status::OK(); } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/mat3x3.hpp b/rerun_cpp/src/datatypes/mat3x3.hpp index e4839179b111..bc3ecee49130 100644 --- a/rerun_cpp/src/datatypes/mat3x3.hpp +++ b/rerun_cpp/src/datatypes/mat3x3.hpp @@ -3,15 +3,8 @@ #pragma once -#include +#include #include -#include - -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow namespace rr { namespace datatypes { @@ -23,9 +16,14 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const Mat3x3* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> + new_arrow_array_builder(arrow::MemoryPool* memory_pool); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::FixedSizeListBuilder* builder, const Mat3x3* elements, size_t num_elements + ); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/mat4x4.cpp b/rerun_cpp/src/datatypes/mat4x4.cpp index 7240cedaed30..7d8454e3b430 100644 --- a/rerun_cpp/src/datatypes/mat4x4.cpp +++ b/rerun_cpp/src/datatypes/mat4x4.cpp @@ -8,21 +8,45 @@ namespace rr { namespace datatypes { std::shared_ptr Mat4x4::to_arrow_datatype() { - return arrow::fixed_size_list(arrow::field("item", arrow::float32(), false, nullptr), - 16); + return arrow::fixed_size_list( + arrow::field("item", arrow::float32(), false, nullptr), + 16 + ); } - arrow::Result> Mat4x4::to_arrow( - arrow::MemoryPool* memory_pool, const Mat4x4* elements, size_t num_elements) { + arrow::Result> Mat4x4::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result(std::make_shared( + memory_pool, + std::make_shared(memory_pool), + 16 + )); + } + + arrow::Status Mat4x4::fill_arrow_array_builder( + arrow::FixedSizeListBuilder *builder, const Mat4x4 *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - auto builder = std::make_shared(memory_pool); - return builder; + auto value_builder = static_cast(builder->value_builder()); + + static_assert(sizeof(elements[0].coeffs) == sizeof(elements[0])); + ARROW_RETURN_NOT_OK( + value_builder->AppendValues(elements[0].coeffs, num_elements * 16, nullptr) + ); + ARROW_RETURN_NOT_OK(builder->AppendValues(num_elements)); + + return arrow::Status::OK(); } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/mat4x4.hpp b/rerun_cpp/src/datatypes/mat4x4.hpp index d90db72311a7..43519801a0b9 100644 --- a/rerun_cpp/src/datatypes/mat4x4.hpp +++ b/rerun_cpp/src/datatypes/mat4x4.hpp @@ -3,15 +3,8 @@ #pragma once -#include +#include #include -#include - -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow namespace rr { namespace datatypes { @@ -23,9 +16,14 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const Mat4x4* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> + new_arrow_array_builder(arrow::MemoryPool* memory_pool); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::FixedSizeListBuilder* builder, const Mat4x4* elements, size_t num_elements + ); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/point2d.cpp b/rerun_cpp/src/datatypes/point2d.cpp index 6b11a3230c9f..2835da471d52 100644 --- a/rerun_cpp/src/datatypes/point2d.cpp +++ b/rerun_cpp/src/datatypes/point2d.cpp @@ -14,21 +14,52 @@ namespace rr { }); } - arrow::Result> Point2D::to_arrow( - arrow::MemoryPool* memory_pool, const Point2D* elements, size_t num_elements) { + arrow::Result> Point2D::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result(std::make_shared( + to_arrow_datatype(), + memory_pool, + std::vector>({ + std::make_shared(memory_pool), + std::make_shared(memory_pool), + }) + )); + } + + arrow::Status Point2D::fill_arrow_array_builder( + arrow::StructBuilder *builder, const Point2D *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - auto datatype = Point2D::to_arrow_datatype(); - let builder = - std::make_shared(datatype, memory_pool, {}, - // TODO(#2647): code-gen for C++ - ); - return builder; + { + auto element_builder = + static_cast(builder->field_builder(0)); + ARROW_RETURN_NOT_OK(element_builder->Reserve(num_elements)); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + ARROW_RETURN_NOT_OK(element_builder->Append(elements[elem_idx].x)); + } + } + { + auto element_builder = + static_cast(builder->field_builder(1)); + ARROW_RETURN_NOT_OK(element_builder->Reserve(num_elements)); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + ARROW_RETURN_NOT_OK(element_builder->Append(elements[elem_idx].y)); + } + } + ARROW_RETURN_NOT_OK(builder->AppendValues(num_elements, nullptr)); + + return arrow::Status::OK(); } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/point2d.hpp b/rerun_cpp/src/datatypes/point2d.hpp index 84d9fb562143..0cf76ab2062e 100644 --- a/rerun_cpp/src/datatypes/point2d.hpp +++ b/rerun_cpp/src/datatypes/point2d.hpp @@ -3,15 +3,8 @@ #pragma once -#include +#include #include -#include - -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow namespace rr { namespace datatypes { @@ -25,9 +18,15 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const Point2D* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const Point2D* elements, size_t num_elements + ); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/point3d.cpp b/rerun_cpp/src/datatypes/point3d.cpp index b55b32fc8fe8..bc24c2203cc1 100644 --- a/rerun_cpp/src/datatypes/point3d.cpp +++ b/rerun_cpp/src/datatypes/point3d.cpp @@ -15,21 +15,61 @@ namespace rr { }); } - arrow::Result> Point3D::to_arrow( - arrow::MemoryPool* memory_pool, const Point3D* elements, size_t num_elements) { + arrow::Result> Point3D::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result(std::make_shared( + to_arrow_datatype(), + memory_pool, + std::vector>({ + std::make_shared(memory_pool), + std::make_shared(memory_pool), + std::make_shared(memory_pool), + }) + )); + } + + arrow::Status Point3D::fill_arrow_array_builder( + arrow::StructBuilder *builder, const Point3D *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - auto datatype = Point3D::to_arrow_datatype(); - let builder = - std::make_shared(datatype, memory_pool, {}, - // TODO(#2647): code-gen for C++ - ); - return builder; + { + auto element_builder = + static_cast(builder->field_builder(0)); + ARROW_RETURN_NOT_OK(element_builder->Reserve(num_elements)); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + ARROW_RETURN_NOT_OK(element_builder->Append(elements[elem_idx].x)); + } + } + { + auto element_builder = + static_cast(builder->field_builder(1)); + ARROW_RETURN_NOT_OK(element_builder->Reserve(num_elements)); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + ARROW_RETURN_NOT_OK(element_builder->Append(elements[elem_idx].y)); + } + } + { + auto element_builder = + static_cast(builder->field_builder(2)); + ARROW_RETURN_NOT_OK(element_builder->Reserve(num_elements)); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + ARROW_RETURN_NOT_OK(element_builder->Append(elements[elem_idx].z)); + } + } + ARROW_RETURN_NOT_OK(builder->AppendValues(num_elements, nullptr)); + + return arrow::Status::OK(); } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/point3d.hpp b/rerun_cpp/src/datatypes/point3d.hpp index 534289e63bc4..437c60ed84ea 100644 --- a/rerun_cpp/src/datatypes/point3d.hpp +++ b/rerun_cpp/src/datatypes/point3d.hpp @@ -3,15 +3,8 @@ #pragma once -#include +#include #include -#include - -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow namespace rr { namespace datatypes { @@ -27,9 +20,15 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const Point3D* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const Point3D* elements, size_t num_elements + ); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/quaternion.cpp b/rerun_cpp/src/datatypes/quaternion.cpp index f16f3689ce12..6f2f7bd2d395 100644 --- a/rerun_cpp/src/datatypes/quaternion.cpp +++ b/rerun_cpp/src/datatypes/quaternion.cpp @@ -8,21 +8,44 @@ namespace rr { namespace datatypes { std::shared_ptr Quaternion::to_arrow_datatype() { - return arrow::fixed_size_list(arrow::field("item", arrow::float32(), false, nullptr), - 4); + return arrow::fixed_size_list( + arrow::field("item", arrow::float32(), false, nullptr), + 4 + ); } - arrow::Result> Quaternion::to_arrow( - arrow::MemoryPool* memory_pool, const Quaternion* elements, size_t num_elements) { + arrow::Result> + Quaternion::new_arrow_array_builder(arrow::MemoryPool *memory_pool) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result(std::make_shared( + memory_pool, + std::make_shared(memory_pool), + 4 + )); + } + + arrow::Status Quaternion::fill_arrow_array_builder( + arrow::FixedSizeListBuilder *builder, const Quaternion *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - auto builder = std::make_shared(memory_pool); - return builder; + auto value_builder = static_cast(builder->value_builder()); + + static_assert(sizeof(elements[0].xyzw) == sizeof(elements[0])); + ARROW_RETURN_NOT_OK( + value_builder->AppendValues(elements[0].xyzw, num_elements * 4, nullptr) + ); + ARROW_RETURN_NOT_OK(builder->AppendValues(num_elements)); + + return arrow::Status::OK(); } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/quaternion.hpp b/rerun_cpp/src/datatypes/quaternion.hpp index 23007d2387e2..40a4587ec56d 100644 --- a/rerun_cpp/src/datatypes/quaternion.hpp +++ b/rerun_cpp/src/datatypes/quaternion.hpp @@ -3,15 +3,8 @@ #pragma once -#include +#include #include -#include - -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow namespace rr { namespace datatypes { @@ -23,9 +16,15 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const Quaternion* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> + new_arrow_array_builder(arrow::MemoryPool* memory_pool); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::FixedSizeListBuilder* builder, const Quaternion* elements, + size_t num_elements + ); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/rotation3d.cpp b/rerun_cpp/src/datatypes/rotation3d.cpp index 72075a72c98b..86980103a742 100644 --- a/rerun_cpp/src/datatypes/rotation3d.cpp +++ b/rerun_cpp/src/datatypes/rotation3d.cpp @@ -14,12 +14,54 @@ namespace rr { return arrow::dense_union({ arrow::field("_null_markers", arrow::null(), true, nullptr), arrow::field( - "Quaternion", rr::datatypes::Quaternion::to_arrow_datatype(), false, nullptr), - arrow::field("AxisAngle", - rr::datatypes::RotationAxisAngle::to_arrow_datatype(), - false, - nullptr), + "Quaternion", + rr::datatypes::Quaternion::to_arrow_datatype(), + false, + nullptr + ), + arrow::field( + "AxisAngle", + rr::datatypes::RotationAxisAngle::to_arrow_datatype(), + false, + nullptr + ), }); } + + arrow::Result> + Rotation3D::new_arrow_array_builder(arrow::MemoryPool* memory_pool) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + memory_pool, + std::vector>({ + std::make_shared(memory_pool), + rr::datatypes::Quaternion::new_arrow_array_builder(memory_pool).ValueOrDie(), + rr::datatypes::RotationAxisAngle::new_arrow_array_builder(memory_pool) + .ValueOrDie(), + }), + to_arrow_datatype() + )); + } + + arrow::Status Rotation3D::fill_arrow_array_builder( + arrow::DenseUnionBuilder* builder, const Rotation3D* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto& element = elements[elem_idx]; + } + return arrow::Status::NotImplemented("TODO(andreas): unions are not yet implemented"); + + return arrow::Status::OK(); + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/rotation3d.hpp b/rerun_cpp/src/datatypes/rotation3d.hpp index b0617a856285..1852cdbdac80 100644 --- a/rerun_cpp/src/datatypes/rotation3d.hpp +++ b/rerun_cpp/src/datatypes/rotation3d.hpp @@ -6,9 +6,9 @@ #include "../datatypes/quaternion.hpp" #include "../datatypes/rotation_axis_angle.hpp" +#include #include #include -#include #include namespace rr { @@ -92,6 +92,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::DenseUnionBuilder* builder, const Rotation3D* elements, size_t num_elements + ); + void swap(Rotation3D& other) noexcept { auto tag_temp = this->_tag; this->_tag = other._tag; diff --git a/rerun_cpp/src/datatypes/rotation_axis_angle.cpp b/rerun_cpp/src/datatypes/rotation_axis_angle.cpp index f5ffa9926b06..f46caad3761a 100644 --- a/rerun_cpp/src/datatypes/rotation_axis_angle.cpp +++ b/rerun_cpp/src/datatypes/rotation_axis_angle.cpp @@ -17,22 +17,41 @@ namespace rr { }); } - arrow::Result> RotationAxisAngle::to_arrow( - arrow::MemoryPool* memory_pool, const RotationAxisAngle* elements, - size_t num_elements) { + arrow::Result> + RotationAxisAngle::new_arrow_array_builder(arrow::MemoryPool* memory_pool) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result(std::make_shared( + to_arrow_datatype(), + memory_pool, + std::vector>({ + rr::datatypes::Vec3D::new_arrow_array_builder(memory_pool).ValueOrDie(), + rr::datatypes::Angle::new_arrow_array_builder(memory_pool).ValueOrDie(), + }) + )); + } + + arrow::Status RotationAxisAngle::fill_arrow_array_builder( + arrow::StructBuilder* builder, const RotationAxisAngle* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - auto datatype = RotationAxisAngle::to_arrow_datatype(); - let builder = - std::make_shared(datatype, memory_pool, {}, - // TODO(#2647): code-gen for C++ - ); - return builder; + return arrow::Status::NotImplemented( + "TODO(andreas): extensions in structs are not yet supported" + ); + return arrow::Status::NotImplemented( + "TODO(andreas): extensions in structs are not yet supported" + ); + ARROW_RETURN_NOT_OK(builder->AppendValues(num_elements, nullptr)); + + return arrow::Status::OK(); } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/rotation_axis_angle.hpp b/rerun_cpp/src/datatypes/rotation_axis_angle.hpp index d689df4d10b7..b80e9870e573 100644 --- a/rerun_cpp/src/datatypes/rotation_axis_angle.hpp +++ b/rerun_cpp/src/datatypes/rotation_axis_angle.hpp @@ -6,15 +6,8 @@ #include "../datatypes/angle.hpp" #include "../datatypes/vec3d.hpp" -#include +#include #include -#include - -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow namespace rr { namespace datatypes { @@ -34,10 +27,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const RotationAxisAngle* elements, - size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const RotationAxisAngle* elements, + size_t num_elements + ); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/scale3d.cpp b/rerun_cpp/src/datatypes/scale3d.cpp index 6398dbcb5a74..0c839ea1189c 100644 --- a/rerun_cpp/src/datatypes/scale3d.cpp +++ b/rerun_cpp/src/datatypes/scale3d.cpp @@ -16,5 +16,41 @@ namespace rr { arrow::field("Uniform", arrow::float32(), false, nullptr), }); } + + arrow::Result> Scale3D::new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + memory_pool, + std::vector>({ + std::make_shared(memory_pool), + rr::datatypes::Vec3D::new_arrow_array_builder(memory_pool).ValueOrDie(), + std::make_shared(memory_pool), + }), + to_arrow_datatype() + )); + } + + arrow::Status Scale3D::fill_arrow_array_builder( + arrow::DenseUnionBuilder* builder, const Scale3D* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto& element = elements[elem_idx]; + } + return arrow::Status::NotImplemented("TODO(andreas): unions are not yet implemented"); + + return arrow::Status::OK(); + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/scale3d.hpp b/rerun_cpp/src/datatypes/scale3d.hpp index 2d1be4ca6729..d18ba6f6c016 100644 --- a/rerun_cpp/src/datatypes/scale3d.hpp +++ b/rerun_cpp/src/datatypes/scale3d.hpp @@ -5,9 +5,9 @@ #include "../datatypes/vec3d.hpp" +#include #include #include -#include #include namespace rr { @@ -91,6 +91,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::DenseUnionBuilder* builder, const Scale3D* elements, size_t num_elements + ); + void swap(Scale3D& other) noexcept { auto tag_temp = this->_tag; this->_tag = other._tag; diff --git a/rerun_cpp/src/datatypes/transform3d.cpp b/rerun_cpp/src/datatypes/transform3d.cpp index 1c24cda5390f..4943990e83dc 100644 --- a/rerun_cpp/src/datatypes/transform3d.cpp +++ b/rerun_cpp/src/datatypes/transform3d.cpp @@ -13,15 +13,56 @@ namespace rr { std::shared_ptr Transform3D::to_arrow_datatype() { return arrow::dense_union({ arrow::field("_null_markers", arrow::null(), true, nullptr), - arrow::field("TranslationAndMat3x3", - rr::datatypes::TranslationAndMat3x3::to_arrow_datatype(), - false, - nullptr), - arrow::field("TranslationRotationScale", - rr::datatypes::TranslationRotationScale3D::to_arrow_datatype(), - false, - nullptr), + arrow::field( + "TranslationAndMat3x3", + rr::datatypes::TranslationAndMat3x3::to_arrow_datatype(), + false, + nullptr + ), + arrow::field( + "TranslationRotationScale", + rr::datatypes::TranslationRotationScale3D::to_arrow_datatype(), + false, + nullptr + ), }); } + + arrow::Result> + Transform3D::new_arrow_array_builder(arrow::MemoryPool* memory_pool) { + if (!memory_pool) { + return arrow::Status::Invalid("Memory pool is null."); + } + + return arrow::Result(std::make_shared( + memory_pool, + std::vector>({ + std::make_shared(memory_pool), + rr::datatypes::TranslationAndMat3x3::new_arrow_array_builder(memory_pool) + .ValueOrDie(), + rr::datatypes::TranslationRotationScale3D::new_arrow_array_builder(memory_pool) + .ValueOrDie(), + }), + to_arrow_datatype() + )); + } + + arrow::Status Transform3D::fill_arrow_array_builder( + arrow::DenseUnionBuilder* builder, const Transform3D* elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } + if (!elements) { + return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); + } + + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto& element = elements[elem_idx]; + } + return arrow::Status::NotImplemented("TODO(andreas): unions are not yet implemented"); + + return arrow::Status::OK(); + } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/transform3d.hpp b/rerun_cpp/src/datatypes/transform3d.hpp index fe61b8a8025f..302e366b116f 100644 --- a/rerun_cpp/src/datatypes/transform3d.hpp +++ b/rerun_cpp/src/datatypes/transform3d.hpp @@ -6,9 +6,9 @@ #include "../datatypes/translation_and_mat3x3.hpp" #include "../datatypes/translation_rotation_scale3d.hpp" +#include #include #include -#include #include namespace rr { @@ -62,7 +62,8 @@ namespace rr { } static Transform3D translation_and_mat3x3( - rr::datatypes::TranslationAndMat3x3 translation_and_mat3x3) { + rr::datatypes::TranslationAndMat3x3 translation_and_mat3x3 + ) { Transform3D self; self._tag = detail::Transform3DTag::TranslationAndMat3x3; self._data.translation_and_mat3x3 = std::move(translation_and_mat3x3); @@ -70,7 +71,8 @@ namespace rr { } static Transform3D translation_rotation_scale( - rr::datatypes::TranslationRotationScale3D translation_rotation_scale) { + rr::datatypes::TranslationRotationScale3D translation_rotation_scale + ) { Transform3D self; self._tag = detail::Transform3DTag::TranslationRotationScale; self._data.translation_rotation_scale = std::move(translation_rotation_scale); @@ -89,6 +91,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::DenseUnionBuilder* builder, const Transform3D* elements, size_t num_elements + ); + void swap(Transform3D& other) noexcept { auto tag_temp = this->_tag; this->_tag = other._tag; diff --git a/rerun_cpp/src/datatypes/translation_and_mat3x3.cpp b/rerun_cpp/src/datatypes/translation_and_mat3x3.cpp index bfe83f477d49..31b8eee2c8aa 100644 --- a/rerun_cpp/src/datatypes/translation_and_mat3x3.cpp +++ b/rerun_cpp/src/datatypes/translation_and_mat3x3.cpp @@ -13,28 +13,60 @@ namespace rr { std::shared_ptr TranslationAndMat3x3::to_arrow_datatype() { return arrow::struct_({ arrow::field( - "translation", rr::datatypes::Vec3D::to_arrow_datatype(), true, nullptr), + "translation", + rr::datatypes::Vec3D::to_arrow_datatype(), + true, + nullptr + ), arrow::field("matrix", rr::datatypes::Mat3x3::to_arrow_datatype(), true, nullptr), arrow::field("from_parent", arrow::boolean(), false, nullptr), }); } - arrow::Result> TranslationAndMat3x3::to_arrow( - arrow::MemoryPool* memory_pool, const TranslationAndMat3x3* elements, - size_t num_elements) { + arrow::Result> + TranslationAndMat3x3::new_arrow_array_builder(arrow::MemoryPool *memory_pool) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result(std::make_shared( + to_arrow_datatype(), + memory_pool, + std::vector>({ + rr::datatypes::Vec3D::new_arrow_array_builder(memory_pool).ValueOrDie(), + rr::datatypes::Mat3x3::new_arrow_array_builder(memory_pool).ValueOrDie(), + std::make_shared(memory_pool), + }) + )); + } + + arrow::Status TranslationAndMat3x3::fill_arrow_array_builder( + arrow::StructBuilder *builder, const TranslationAndMat3x3 *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - auto datatype = TranslationAndMat3x3::to_arrow_datatype(); - let builder = - std::make_shared(datatype, memory_pool, {}, - // TODO(#2647): code-gen for C++ - ); - return builder; + return arrow::Status::NotImplemented( + "TODO(andreas): extensions in structs are not yet supported" + ); + return arrow::Status::NotImplemented( + "TODO(andreas): extensions in structs are not yet supported" + ); + { + auto element_builder = + static_cast(builder->field_builder(2)); + ARROW_RETURN_NOT_OK(element_builder->Reserve(num_elements)); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + ARROW_RETURN_NOT_OK(element_builder->Append(elements[elem_idx].from_parent)); + } + } + ARROW_RETURN_NOT_OK(builder->AppendValues(num_elements, nullptr)); + + return arrow::Status::OK(); } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/translation_and_mat3x3.hpp b/rerun_cpp/src/datatypes/translation_and_mat3x3.hpp index 157bec963561..0abf367df8d9 100644 --- a/rerun_cpp/src/datatypes/translation_and_mat3x3.hpp +++ b/rerun_cpp/src/datatypes/translation_and_mat3x3.hpp @@ -6,17 +6,10 @@ #include "../datatypes/mat3x3.hpp" #include "../datatypes/vec3d.hpp" -#include +#include #include -#include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace datatypes { /// Representation of an affine transform via a 3x3 affine matrix paired with a translation. @@ -37,10 +30,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const TranslationAndMat3x3* elements, - size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const TranslationAndMat3x3* elements, + size_t num_elements + ); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/translation_rotation_scale3d.cpp b/rerun_cpp/src/datatypes/translation_rotation_scale3d.cpp index b56f45ccd3de..3ba54c30e625 100644 --- a/rerun_cpp/src/datatypes/translation_rotation_scale3d.cpp +++ b/rerun_cpp/src/datatypes/translation_rotation_scale3d.cpp @@ -14,30 +14,71 @@ namespace rr { std::shared_ptr TranslationRotationScale3D::to_arrow_datatype() { return arrow::struct_({ arrow::field( - "translation", rr::datatypes::Vec3D::to_arrow_datatype(), true, nullptr), + "translation", + rr::datatypes::Vec3D::to_arrow_datatype(), + true, + nullptr + ), arrow::field( - "rotation", rr::datatypes::Rotation3D::to_arrow_datatype(), true, nullptr), + "rotation", + rr::datatypes::Rotation3D::to_arrow_datatype(), + true, + nullptr + ), arrow::field("scale", rr::datatypes::Scale3D::to_arrow_datatype(), true, nullptr), arrow::field("from_parent", arrow::boolean(), false, nullptr), }); } - arrow::Result> TranslationRotationScale3D::to_arrow( - arrow::MemoryPool* memory_pool, const TranslationRotationScale3D* elements, - size_t num_elements) { + arrow::Result> + TranslationRotationScale3D::new_arrow_array_builder(arrow::MemoryPool *memory_pool) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result(std::make_shared( + to_arrow_datatype(), + memory_pool, + std::vector>({ + rr::datatypes::Vec3D::new_arrow_array_builder(memory_pool).ValueOrDie(), + rr::datatypes::Rotation3D::new_arrow_array_builder(memory_pool).ValueOrDie(), + rr::datatypes::Scale3D::new_arrow_array_builder(memory_pool).ValueOrDie(), + std::make_shared(memory_pool), + }) + )); + } + + arrow::Status TranslationRotationScale3D::fill_arrow_array_builder( + arrow::StructBuilder *builder, const TranslationRotationScale3D *elements, + size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - auto datatype = TranslationRotationScale3D::to_arrow_datatype(); - let builder = - std::make_shared(datatype, memory_pool, {}, - // TODO(#2647): code-gen for C++ - ); - return builder; + return arrow::Status::NotImplemented( + "TODO(andreas): extensions in structs are not yet supported" + ); + return arrow::Status::NotImplemented( + "TODO(andreas): extensions in structs are not yet supported" + ); + return arrow::Status::NotImplemented( + "TODO(andreas): extensions in structs are not yet supported" + ); + { + auto element_builder = + static_cast(builder->field_builder(3)); + ARROW_RETURN_NOT_OK(element_builder->Reserve(num_elements)); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + ARROW_RETURN_NOT_OK(element_builder->Append(elements[elem_idx].from_parent)); + } + } + ARROW_RETURN_NOT_OK(builder->AppendValues(num_elements, nullptr)); + + return arrow::Status::OK(); } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/translation_rotation_scale3d.hpp b/rerun_cpp/src/datatypes/translation_rotation_scale3d.hpp index 8b2d621da71f..8a8e11cadca9 100644 --- a/rerun_cpp/src/datatypes/translation_rotation_scale3d.hpp +++ b/rerun_cpp/src/datatypes/translation_rotation_scale3d.hpp @@ -7,17 +7,10 @@ #include "../datatypes/scale3d.hpp" #include "../datatypes/vec3d.hpp" -#include +#include #include -#include #include -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow - namespace rr { namespace datatypes { /// Representation of an affine transform via separate translation, rotation & scale. @@ -39,10 +32,16 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const TranslationRotationScale3D* elements, - size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> new_arrow_array_builder( + arrow::MemoryPool* memory_pool + ); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::StructBuilder* builder, const TranslationRotationScale3D* elements, + size_t num_elements + ); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/vec2d.cpp b/rerun_cpp/src/datatypes/vec2d.cpp index d2178481aa3e..7af078821fd4 100644 --- a/rerun_cpp/src/datatypes/vec2d.cpp +++ b/rerun_cpp/src/datatypes/vec2d.cpp @@ -8,21 +8,45 @@ namespace rr { namespace datatypes { std::shared_ptr Vec2D::to_arrow_datatype() { - return arrow::fixed_size_list(arrow::field("item", arrow::float32(), false, nullptr), - 2); + return arrow::fixed_size_list( + arrow::field("item", arrow::float32(), false, nullptr), + 2 + ); } - arrow::Result> Vec2D::to_arrow( - arrow::MemoryPool* memory_pool, const Vec2D* elements, size_t num_elements) { + arrow::Result> Vec2D::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result(std::make_shared( + memory_pool, + std::make_shared(memory_pool), + 2 + )); + } + + arrow::Status Vec2D::fill_arrow_array_builder( + arrow::FixedSizeListBuilder *builder, const Vec2D *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - auto builder = std::make_shared(memory_pool); - return builder; + auto value_builder = static_cast(builder->value_builder()); + + static_assert(sizeof(elements[0].xy) == sizeof(elements[0])); + ARROW_RETURN_NOT_OK( + value_builder->AppendValues(elements[0].xy, num_elements * 2, nullptr) + ); + ARROW_RETURN_NOT_OK(builder->AppendValues(num_elements)); + + return arrow::Status::OK(); } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/vec2d.hpp b/rerun_cpp/src/datatypes/vec2d.hpp index 0051c5a44f06..6f3438d30242 100644 --- a/rerun_cpp/src/datatypes/vec2d.hpp +++ b/rerun_cpp/src/datatypes/vec2d.hpp @@ -3,15 +3,8 @@ #pragma once -#include +#include #include -#include - -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow namespace rr { namespace datatypes { @@ -23,9 +16,14 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const Vec2D* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> + new_arrow_array_builder(arrow::MemoryPool* memory_pool); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::FixedSizeListBuilder* builder, const Vec2D* elements, size_t num_elements + ); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/vec3d.cpp b/rerun_cpp/src/datatypes/vec3d.cpp index 7b136afdae37..be277e718291 100644 --- a/rerun_cpp/src/datatypes/vec3d.cpp +++ b/rerun_cpp/src/datatypes/vec3d.cpp @@ -8,21 +8,45 @@ namespace rr { namespace datatypes { std::shared_ptr Vec3D::to_arrow_datatype() { - return arrow::fixed_size_list(arrow::field("item", arrow::float32(), false, nullptr), - 3); + return arrow::fixed_size_list( + arrow::field("item", arrow::float32(), false, nullptr), + 3 + ); } - arrow::Result> Vec3D::to_arrow( - arrow::MemoryPool* memory_pool, const Vec3D* elements, size_t num_elements) { + arrow::Result> Vec3D::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result(std::make_shared( + memory_pool, + std::make_shared(memory_pool), + 3 + )); + } + + arrow::Status Vec3D::fill_arrow_array_builder( + arrow::FixedSizeListBuilder *builder, const Vec3D *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - auto builder = std::make_shared(memory_pool); - return builder; + auto value_builder = static_cast(builder->value_builder()); + + static_assert(sizeof(elements[0].xyz) == sizeof(elements[0])); + ARROW_RETURN_NOT_OK( + value_builder->AppendValues(elements[0].xyz, num_elements * 3, nullptr) + ); + ARROW_RETURN_NOT_OK(builder->AppendValues(num_elements)); + + return arrow::Status::OK(); } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/vec3d.hpp b/rerun_cpp/src/datatypes/vec3d.hpp index c9ba8bf3b240..92bd757db95a 100644 --- a/rerun_cpp/src/datatypes/vec3d.hpp +++ b/rerun_cpp/src/datatypes/vec3d.hpp @@ -3,15 +3,8 @@ #pragma once -#include +#include #include -#include - -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow namespace rr { namespace datatypes { @@ -23,9 +16,14 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const Vec3D* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> + new_arrow_array_builder(arrow::MemoryPool* memory_pool); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::FixedSizeListBuilder* builder, const Vec3D* elements, size_t num_elements + ); }; } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/vec4d.cpp b/rerun_cpp/src/datatypes/vec4d.cpp index 058e1ea267cb..dff32d481cf0 100644 --- a/rerun_cpp/src/datatypes/vec4d.cpp +++ b/rerun_cpp/src/datatypes/vec4d.cpp @@ -8,21 +8,45 @@ namespace rr { namespace datatypes { std::shared_ptr Vec4D::to_arrow_datatype() { - return arrow::fixed_size_list(arrow::field("item", arrow::float32(), false, nullptr), - 4); + return arrow::fixed_size_list( + arrow::field("item", arrow::float32(), false, nullptr), + 4 + ); } - arrow::Result> Vec4D::to_arrow( - arrow::MemoryPool* memory_pool, const Vec4D* elements, size_t num_elements) { + arrow::Result> Vec4D::new_arrow_array_builder( + arrow::MemoryPool *memory_pool + ) { if (!memory_pool) { return arrow::Status::Invalid("Memory pool is null."); } + + return arrow::Result(std::make_shared( + memory_pool, + std::make_shared(memory_pool), + 4 + )); + } + + arrow::Status Vec4D::fill_arrow_array_builder( + arrow::FixedSizeListBuilder *builder, const Vec4D *elements, size_t num_elements + ) { + if (!builder) { + return arrow::Status::Invalid("Passed array builder is null."); + } if (!elements) { return arrow::Status::Invalid("Cannot serialize null pointer to arrow array."); } - auto builder = std::make_shared(memory_pool); - return builder; + auto value_builder = static_cast(builder->value_builder()); + + static_assert(sizeof(elements[0].xyzw) == sizeof(elements[0])); + ARROW_RETURN_NOT_OK( + value_builder->AppendValues(elements[0].xyzw, num_elements * 4, nullptr) + ); + ARROW_RETURN_NOT_OK(builder->AppendValues(num_elements)); + + return arrow::Status::OK(); } } // namespace datatypes } // namespace rr diff --git a/rerun_cpp/src/datatypes/vec4d.hpp b/rerun_cpp/src/datatypes/vec4d.hpp index 0086942146fe..7ab3e76a2283 100644 --- a/rerun_cpp/src/datatypes/vec4d.hpp +++ b/rerun_cpp/src/datatypes/vec4d.hpp @@ -3,15 +3,8 @@ #pragma once -#include +#include #include -#include - -namespace arrow { - class ArrayBuilder; - class DataType; - class MemoryPool; -} // namespace arrow namespace rr { namespace datatypes { @@ -23,9 +16,14 @@ namespace rr { /// Returns the arrow data type this type corresponds to. static std::shared_ptr to_arrow_datatype(); - /// Fills out an arrow array builder with an array of this type. - static arrow::Result> to_arrow( - arrow::MemoryPool* memory_pool, const Vec4D* elements, size_t num_elements); + /// Creates a new array builder with an array of this type. + static arrow::Result> + new_arrow_array_builder(arrow::MemoryPool* memory_pool); + + /// Fills an arrow array builder with an array of this type. + static arrow::Status fill_arrow_array_builder( + arrow::FixedSizeListBuilder* builder, const Vec4D* elements, size_t num_elements + ); }; } // namespace datatypes } // namespace rr From a012c8c8ac400f887bbe0f290c2fe5a83161dc05 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Wed, 26 Jul 2023 12:00:07 +0200 Subject: [PATCH 20/20] formatting --- examples/cpp/minimal/main.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/cpp/minimal/main.cpp b/examples/cpp/minimal/main.cpp index 43ec5d30e4ee..1a7e24b33c12 100644 --- a/examples/cpp/minimal/main.cpp +++ b/examples/cpp/minimal/main.cpp @@ -9,7 +9,8 @@ arrow::Result> points2(size_t num_points, const fl ARROW_ASSIGN_OR_RAISE(auto builder, rr::components::Point2D::new_arrow_array_builder(pool)); ARROW_RETURN_NOT_OK(rr::components::Point2D::fill_arrow_array_builder( builder.get(), - (const rr::components::Point2D*)xy, // TODO(andreas): Hack to get Points2D C-style array in an easy fashion + (const rr::components::Point2D*) + xy, // TODO(andreas): Hack to get Points2D C-style array in an easy fashion num_points ));