From b45a5bf36c05ffeed0903b0e502e8aa8600ee735 Mon Sep 17 00:00:00 2001 From: Clement Rey Date: Fri, 30 Jun 2023 15:26:28 +0200 Subject: [PATCH] Codegen'd Rust/Arrow (de)ser 6: serialization (#2549) **Best reviewed on a commit-by-commit basis; in particular the `rerun codegen` commit is nothing but generated code.** Implements serialization for the Rust codegen backend. --- - #2484 - #2485 - #2487 - #2545 - #2546 - #2549 - #2554 - #2570 - #2571 --- PR Build Summary: https://build.rerun.io/pr/2487 Docs preview: https://rerun.io/preview/87ca70a/docs Examples preview: https://rerun.io/preview/87ca70a/examples --- Cargo.lock | 2 + _typos.toml | 7 +- crates/re_types/Cargo.toml | 3 + crates/re_types/source_hash.txt | 2 +- crates/re_types/src/archetypes/fuzzy.rs | 418 ++++++- crates/re_types/src/archetypes/points2d.rs | 137 +- crates/re_types/src/components/class_id.rs | 48 + crates/re_types/src/components/color.rs | 48 + crates/re_types/src/components/draw_order.rs | 48 + crates/re_types/src/components/fuzzy.rs | 1106 +++++++++++++++++ .../re_types/src/components/instance_key.rs | 48 + crates/re_types/src/components/keypoint_id.rs | 48 + crates/re_types/src/components/label.rs | 63 + crates/re_types/src/components/point2d.rs | 106 ++ crates/re_types/src/components/radius.rs | 48 + crates/re_types/src/datatypes/fuzzy.rs | 519 ++++++++ crates/re_types/src/datatypes/vec2d.rs | 78 ++ crates/re_types/src/lib.rs | 191 ++- crates/re_types/tests/fuzzy.rs | 12 +- crates/re_types/tests/points2d.rs | 32 +- crates/re_types_builder/src/codegen/rust.rs | 391 +++++- 21 files changed, 3325 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b50ed20b3230..a1d1bee8809f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4521,6 +4521,7 @@ dependencies = [ name = "re_types" version = "0.8.0-alpha.0" dependencies = [ + "anyhow", "arrow2", "bytemuck", "document-features", @@ -4531,6 +4532,7 @@ dependencies = [ "re_build_tools", "re_types_builder", "similar-asserts", + "thiserror", "xshell", ] diff --git a/_typos.toml b/_typos.toml index c943f61426b5..adb9d5b658d4 100644 --- a/_typos.toml +++ b/_typos.toml @@ -11,9 +11,10 @@ extend-exclude = [ [default.extend-words] -lod = "lod" # level-of-detail -teh = "teh" # part of @teh-cmc -ND = "ND" # np.NDArray +lod = "lod" # level-of-detail +teh = "teh" # part of @teh-cmc +ND = "ND" # np.NDArray +somes = "somes" # many `Some` # American English: grey = "gray" diff --git a/crates/re_types/Cargo.toml b/crates/re_types/Cargo.toml index d596ff1b9347..e3abc22738eb 100644 --- a/crates/re_types/Cargo.toml +++ b/crates/re_types/Cargo.toml @@ -35,8 +35,11 @@ arrow2 = { workspace = true, features = [ "io_print", "compute_concatenate", ] } +anyhow.workspace = true bytemuck = { version = "1.11", features = ["derive", "extern_crate_alloc"] } document-features = "0.2" +itertools.workspace = true +thiserror.workspace = true # External (optional) ecolor = { workspace = true, optional = true } diff --git a/crates/re_types/source_hash.txt b/crates/re_types/source_hash.txt index 15cae3431078..2e71ceaad9cd 100644 --- a/crates/re_types/source_hash.txt +++ b/crates/re_types/source_hash.txt @@ -1,4 +1,4 @@ # This is a sha256 hash for all direct and indirect dependencies of this crate's build script. # It can be safely removed at anytime to force the build script to run again. # Check out build.rs to see how it's computed. -32b9eccc275777c29b252374ea6dcd69a3f1a48c42ae79aa5138ec2556526b52 \ No newline at end of file +70d271f9866784d9f0d25b818cd0d45fff1df72aec62ceae9f167f523a5c2d08 \ No newline at end of file diff --git a/crates/re_types/src/archetypes/fuzzy.rs b/crates/re_types/src/archetypes/fuzzy.rs index a4cb29782f0b..ec710d0d1499 100644 --- a/crates/re_types/src/archetypes/fuzzy.rs +++ b/crates/re_types/src/archetypes/fuzzy.rs @@ -110,25 +110,437 @@ impl AffixFuzzer1 { } impl crate::Archetype for AffixFuzzer1 { + #[inline] fn name() -> crate::ArchetypeName { crate::ArchetypeName::Borrowed("rerun.testing.archetypes.AffixFuzzer1") } + #[inline] fn required_components() -> Vec { Self::REQUIRED_COMPONENTS.to_vec() } + #[inline] fn recommended_components() -> Vec { Self::RECOMMENDED_COMPONENTS.to_vec() } + #[inline] fn optional_components() -> Vec { Self::OPTIONAL_COMPONENTS.to_vec() } - #[allow(clippy::todo)] - fn to_arrow_datatypes() -> Vec { - todo!("query the registry for all fqnames"); + #[inline] + fn try_to_arrow( + &self, + ) -> crate::SerializationResult< + Vec<(::arrow2::datatypes::Field, Box)>, + > { + use crate::Component as _; + Ok([ + { + Some({ + let array = ::try_to_arrow([&self.fuzz1001]); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("fuzz1001", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + Some({ + let array = ::try_to_arrow([&self.fuzz1002]); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("fuzz1002", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + Some({ + let array = ::try_to_arrow([&self.fuzz1003]); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("fuzz1003", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + Some({ + let array = ::try_to_arrow([&self.fuzz1004]); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("fuzz1004", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + Some({ + let array = ::try_to_arrow([&self.fuzz1005]); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("fuzz1005", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + Some({ + let array = ::try_to_arrow([&self.fuzz1006]); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("fuzz1006", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + Some({ + let array = ::try_to_arrow([&self.fuzz1007]); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("fuzz1007", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + Some({ + let array = + ::try_to_arrow(self.fuzz1101.iter()); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("fuzz1101", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + Some({ + let array = + ::try_to_arrow(self.fuzz1102.iter()); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("fuzz1102", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + Some({ + let array = + ::try_to_arrow(self.fuzz1103.iter()); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("fuzz1103", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + Some({ + let array = + ::try_to_arrow(self.fuzz1104.iter()); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("fuzz1104", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + Some({ + let array = + ::try_to_arrow(self.fuzz1105.iter()); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("fuzz1105", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + Some({ + let array = + ::try_to_arrow(self.fuzz1106.iter()); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("fuzz1106", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + Some({ + let array = + ::try_to_arrow(self.fuzz1107.iter()); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("fuzz1107", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.fuzz2001 + .as_ref() + .map(|single| { + let array = ::try_to_arrow([single]); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("fuzz2001", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.fuzz2002 + .as_ref() + .map(|single| { + let array = ::try_to_arrow([single]); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("fuzz2002", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.fuzz2003 + .as_ref() + .map(|single| { + let array = ::try_to_arrow([single]); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("fuzz2003", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.fuzz2004 + .as_ref() + .map(|single| { + let array = ::try_to_arrow([single]); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("fuzz2004", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.fuzz2005 + .as_ref() + .map(|single| { + let array = ::try_to_arrow([single]); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("fuzz2005", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.fuzz2006 + .as_ref() + .map(|single| { + let array = ::try_to_arrow([single]); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("fuzz2006", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.fuzz2007 + .as_ref() + .map(|single| { + let array = ::try_to_arrow([single]); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("fuzz2007", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.fuzz2101 + .as_ref() + .map(|many| { + let array = ::try_to_arrow(many.iter()); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("fuzz2101", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.fuzz2102 + .as_ref() + .map(|many| { + let array = ::try_to_arrow(many.iter()); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("fuzz2102", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.fuzz2103 + .as_ref() + .map(|many| { + let array = ::try_to_arrow(many.iter()); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("fuzz2103", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.fuzz2104 + .as_ref() + .map(|many| { + let array = ::try_to_arrow(many.iter()); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("fuzz2104", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.fuzz2105 + .as_ref() + .map(|many| { + let array = ::try_to_arrow(many.iter()); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("fuzz2105", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.fuzz2106 + .as_ref() + .map(|many| { + let array = ::try_to_arrow(many.iter()); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("fuzz2106", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.fuzz2107 + .as_ref() + .map(|many| { + let array = ::try_to_arrow(many.iter()); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("fuzz2107", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + ] + .into_iter() + .flatten() + .collect()) } } diff --git a/crates/re_types/src/archetypes/points2d.rs b/crates/re_types/src/archetypes/points2d.rs index b3e03ac72e49..c30a200665ca 100644 --- a/crates/re_types/src/archetypes/points2d.rs +++ b/crates/re_types/src/archetypes/points2d.rs @@ -78,25 +78,156 @@ impl Points2D { } impl crate::Archetype for Points2D { + #[inline] fn name() -> crate::ArchetypeName { crate::ArchetypeName::Borrowed("rerun.archetypes.Points2D") } + #[inline] fn required_components() -> Vec { Self::REQUIRED_COMPONENTS.to_vec() } + #[inline] fn recommended_components() -> Vec { Self::RECOMMENDED_COMPONENTS.to_vec() } + #[inline] fn optional_components() -> Vec { Self::OPTIONAL_COMPONENTS.to_vec() } - #[allow(clippy::todo)] - fn to_arrow_datatypes() -> Vec { - todo!("query the registry for all fqnames"); + #[inline] + fn try_to_arrow( + &self, + ) -> crate::SerializationResult< + Vec<(::arrow2::datatypes::Field, Box)>, + > { + use crate::Component as _; + Ok([ + { + Some({ + let array = ::try_to_arrow(self.points.iter()); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("points", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.radii + .as_ref() + .map(|many| { + let array = ::try_to_arrow(many.iter()); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("radii", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.colors + .as_ref() + .map(|many| { + let array = ::try_to_arrow(many.iter()); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("colors", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.labels + .as_ref() + .map(|many| { + let array = ::try_to_arrow(many.iter()); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("labels", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.draw_order + .as_ref() + .map(|single| { + let array = ::try_to_arrow([single]); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("draw_order", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.class_ids + .as_ref() + .map(|many| { + let array = ::try_to_arrow(many.iter()); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("class_ids", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.keypoint_ids + .as_ref() + .map(|many| { + let array = ::try_to_arrow(many.iter()); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("keypoint_ids", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.instance_keys + .as_ref() + .map(|many| { + let array = ::try_to_arrow(many.iter()); + array.map(|array| { + let datatype = array.data_type().clone(); + ( + ::arrow2::datatypes::Field::new("instance_keys", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + ] + .into_iter() + .flatten() + .collect()) } } diff --git a/crates/re_types/src/components/class_id.rs b/crates/re_types/src/components/class_id.rs index 64775ff5c49b..d60ebaa318a0 100644 --- a/crates/re_types/src/components/class_id.rs +++ b/crates/re_types/src/components/class_id.rs @@ -14,6 +14,20 @@ #[derive(Copy, Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct ClassId(pub u16); +impl<'a> From for ::std::borrow::Cow<'a, ClassId> { + #[inline] + fn from(value: ClassId) -> Self { + std::borrow::Cow::Owned(value) + } +} + +impl<'a> From<&'a ClassId> for ::std::borrow::Cow<'a, ClassId> { + #[inline] + fn from(value: &'a ClassId) -> Self { + std::borrow::Cow::Borrowed(value) + } +} + impl crate::Component for ClassId { #[inline] fn name() -> crate::ComponentName { @@ -30,4 +44,38 @@ impl crate::Component for ClassId { None, ) } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_to_arrow_opt<'a>( + data: impl IntoIterator>>>, + ) -> crate::SerializationResult> + where + Self: Clone + 'a, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let (somes, data0): (Vec<_>, Vec<_>) = data + .into_iter() + .map(|datum| { + let datum: Option<::std::borrow::Cow<'a, Self>> = datum.map(Into::into); + let datum = datum.map(|datum| { + let Self(data0) = datum.into_owned(); + data0 + }); + (datum.is_some(), datum) + }) + .unzip(); + let data0_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + PrimitiveArray::new( + DataType::UInt16, + data0.into_iter().map(|v| v.unwrap_or_default()).collect(), + data0_bitmap, + ) + .boxed() + }) + } } diff --git a/crates/re_types/src/components/color.rs b/crates/re_types/src/components/color.rs index 2acd13309b03..1b0a91ffe06b 100644 --- a/crates/re_types/src/components/color.rs +++ b/crates/re_types/src/components/color.rs @@ -24,6 +24,20 @@ #[repr(transparent)] pub struct Color(pub u32); +impl<'a> From for ::std::borrow::Cow<'a, Color> { + #[inline] + fn from(value: Color) -> Self { + std::borrow::Cow::Owned(value) + } +} + +impl<'a> From<&'a Color> for ::std::borrow::Cow<'a, Color> { + #[inline] + fn from(value: &'a Color) -> Self { + std::borrow::Cow::Borrowed(value) + } +} + impl crate::Component for Color { #[inline] fn name() -> crate::ComponentName { @@ -40,4 +54,38 @@ impl crate::Component for Color { None, ) } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_to_arrow_opt<'a>( + data: impl IntoIterator>>>, + ) -> crate::SerializationResult> + where + Self: Clone + 'a, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let (somes, data0): (Vec<_>, Vec<_>) = data + .into_iter() + .map(|datum| { + let datum: Option<::std::borrow::Cow<'a, Self>> = datum.map(Into::into); + let datum = datum.map(|datum| { + let Self(data0) = datum.into_owned(); + data0 + }); + (datum.is_some(), datum) + }) + .unzip(); + let data0_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + PrimitiveArray::new( + DataType::UInt32, + data0.into_iter().map(|v| v.unwrap_or_default()).collect(), + data0_bitmap, + ) + .boxed() + }) + } } diff --git a/crates/re_types/src/components/draw_order.rs b/crates/re_types/src/components/draw_order.rs index 17e541b1fe2a..1b494ce5ba19 100644 --- a/crates/re_types/src/components/draw_order.rs +++ b/crates/re_types/src/components/draw_order.rs @@ -19,6 +19,20 @@ #[repr(transparent)] pub struct DrawOrder(pub f32); +impl<'a> From for ::std::borrow::Cow<'a, DrawOrder> { + #[inline] + fn from(value: DrawOrder) -> Self { + std::borrow::Cow::Owned(value) + } +} + +impl<'a> From<&'a DrawOrder> for ::std::borrow::Cow<'a, DrawOrder> { + #[inline] + fn from(value: &'a DrawOrder) -> Self { + std::borrow::Cow::Borrowed(value) + } +} + impl crate::Component for DrawOrder { #[inline] fn name() -> crate::ComponentName { @@ -35,4 +49,38 @@ impl crate::Component for DrawOrder { None, ) } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_to_arrow_opt<'a>( + data: impl IntoIterator>>>, + ) -> crate::SerializationResult> + where + Self: Clone + 'a, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let (somes, data0): (Vec<_>, Vec<_>) = data + .into_iter() + .map(|datum| { + let datum: Option<::std::borrow::Cow<'a, Self>> = datum.map(Into::into); + let datum = datum.map(|datum| { + let Self(data0) = datum.into_owned(); + data0 + }); + (datum.is_some(), datum) + }) + .unzip(); + let data0_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + PrimitiveArray::new( + DataType::Float32, + data0.into_iter().map(|v| v.unwrap_or_default()).collect(), + data0_bitmap, + ) + .boxed() + }) + } } diff --git a/crates/re_types/src/components/fuzzy.rs b/crates/re_types/src/components/fuzzy.rs index f977e38fefd5..232383ea7175 100644 --- a/crates/re_types/src/components/fuzzy.rs +++ b/crates/re_types/src/components/fuzzy.rs @@ -13,6 +13,20 @@ pub struct AffixFuzzer1 { pub single_required: crate::datatypes::AffixFuzzer1, } +impl<'a> From for ::std::borrow::Cow<'a, AffixFuzzer1> { + #[inline] + fn from(value: AffixFuzzer1) -> Self { + std::borrow::Cow::Owned(value) + } +} + +impl<'a> From<&'a AffixFuzzer1> for ::std::borrow::Cow<'a, AffixFuzzer1> { + #[inline] + fn from(value: &'a AffixFuzzer1) -> Self { + std::borrow::Cow::Borrowed(value) + } +} + impl crate::Component for AffixFuzzer1 { #[inline] fn name() -> crate::ComponentName { @@ -85,11 +99,57 @@ impl crate::Component for AffixFuzzer1 { None, ) } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_to_arrow_opt<'a>( + data: impl IntoIterator>>>, + ) -> crate::SerializationResult> + where + Self: Clone + 'a, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let (somes, single_required): (Vec<_>, Vec<_>) = data + .into_iter() + .map(|datum| { + let datum: Option<::std::borrow::Cow<'a, Self>> = datum.map(Into::into); + let datum = datum.map(|datum| { + let Self { single_required } = datum.into_owned(); + single_required + }); + (datum.is_some(), datum) + }) + .unzip(); + let single_required_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + { + _ = single_required_bitmap; + crate::datatypes::AffixFuzzer1::try_to_arrow_opt(single_required)? + } + }) + } } #[derive(Debug, Clone, PartialEq)] pub struct AffixFuzzer2(pub crate::datatypes::AffixFuzzer1); +impl<'a> From for ::std::borrow::Cow<'a, AffixFuzzer2> { + #[inline] + fn from(value: AffixFuzzer2) -> Self { + std::borrow::Cow::Owned(value) + } +} + +impl<'a> From<&'a AffixFuzzer2> for ::std::borrow::Cow<'a, AffixFuzzer2> { + #[inline] + fn from(value: &'a AffixFuzzer2) -> Self { + std::borrow::Cow::Borrowed(value) + } +} + impl crate::Component for AffixFuzzer2 { #[inline] fn name() -> crate::ComponentName { @@ -162,6 +222,38 @@ impl crate::Component for AffixFuzzer2 { None, ) } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_to_arrow_opt<'a>( + data: impl IntoIterator>>>, + ) -> crate::SerializationResult> + where + Self: Clone + 'a, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let (somes, data0): (Vec<_>, Vec<_>) = data + .into_iter() + .map(|datum| { + let datum: Option<::std::borrow::Cow<'a, Self>> = datum.map(Into::into); + let datum = datum.map(|datum| { + let Self(data0) = datum.into_owned(); + data0 + }); + (datum.is_some(), datum) + }) + .unzip(); + let data0_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + { + _ = data0_bitmap; + crate::datatypes::AffixFuzzer1::try_to_arrow_opt(data0)? + } + }) + } } #[derive(Debug, Clone, PartialEq)] @@ -169,6 +261,20 @@ pub struct AffixFuzzer3 { pub single_required: crate::datatypes::AffixFuzzer1, } +impl<'a> From for ::std::borrow::Cow<'a, AffixFuzzer3> { + #[inline] + fn from(value: AffixFuzzer3) -> Self { + std::borrow::Cow::Owned(value) + } +} + +impl<'a> From<&'a AffixFuzzer3> for ::std::borrow::Cow<'a, AffixFuzzer3> { + #[inline] + fn from(value: &'a AffixFuzzer3) -> Self { + std::borrow::Cow::Borrowed(value) + } +} + impl crate::Component for AffixFuzzer3 { #[inline] fn name() -> crate::ComponentName { @@ -246,6 +352,122 @@ impl crate::Component for AffixFuzzer3 { None, ) } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_to_arrow_opt<'a>( + data: impl IntoIterator>>>, + ) -> crate::SerializationResult> + where + Self: Clone + 'a, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let (somes, data): (Vec<_>, Vec<_>) = data + .into_iter() + .map(|datum| { + let datum: Option<::std::borrow::Cow<'a, Self>> = datum.map(Into::into); + (datum.is_some(), datum) + }) + .unzip(); + let bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + StructArray::new( + DataType::Extension( + "rerun.testing.components.AffixFuzzer3".to_owned(), + Box::new(DataType::Struct(vec![Field { + name: "single_required".to_owned(), + data_type: DataType::Extension( + "rerun.testing.datatypes.AffixFuzzer1".to_owned(), + Box::new(DataType::Struct(vec![ + Field { + name: "single_float_optional".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "single_string_required".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "single_string_optional".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_floats_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_strings_required".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + })), + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "many_strings_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + ])), + None, + ), + is_nullable: false, + metadata: [].into(), + }])), + None, + ), + vec![{ + let (somes, single_required): (Vec<_>, Vec<_>) = data + .iter() + .map(|datum| { + let datum = datum.as_ref().map(|datum| { + let Self { + single_required, .. + } = &**datum; + single_required.clone() + }); + (datum.is_some(), datum) + }) + .unzip(); + let single_required_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + { + _ = single_required_bitmap; + crate::datatypes::AffixFuzzer1::try_to_arrow_opt(single_required)? + } + }], + bitmap, + ) + .boxed() + }) + } } #[derive(Debug, Clone, PartialEq)] @@ -253,6 +475,20 @@ pub struct AffixFuzzer4 { pub single_optional: Option, } +impl<'a> From for ::std::borrow::Cow<'a, AffixFuzzer4> { + #[inline] + fn from(value: AffixFuzzer4) -> Self { + std::borrow::Cow::Owned(value) + } +} + +impl<'a> From<&'a AffixFuzzer4> for ::std::borrow::Cow<'a, AffixFuzzer4> { + #[inline] + fn from(value: &'a AffixFuzzer4) -> Self { + std::borrow::Cow::Borrowed(value) + } +} + impl crate::Component for AffixFuzzer4 { #[inline] fn name() -> crate::ComponentName { @@ -325,11 +561,59 @@ impl crate::Component for AffixFuzzer4 { None, ) } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_to_arrow_opt<'a>( + data: impl IntoIterator>>>, + ) -> crate::SerializationResult> + where + Self: Clone + 'a, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let (somes, single_optional): (Vec<_>, Vec<_>) = data + .into_iter() + .map(|datum| { + let datum: Option<::std::borrow::Cow<'a, Self>> = datum.map(Into::into); + let datum = datum + .map(|datum| { + let Self { single_optional } = datum.into_owned(); + single_optional + }) + .flatten(); + (datum.is_some(), datum) + }) + .unzip(); + let single_optional_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + { + _ = single_optional_bitmap; + crate::datatypes::AffixFuzzer1::try_to_arrow_opt(single_optional)? + } + }) + } } #[derive(Debug, Clone, PartialEq)] pub struct AffixFuzzer5(pub Option); +impl<'a> From for ::std::borrow::Cow<'a, AffixFuzzer5> { + #[inline] + fn from(value: AffixFuzzer5) -> Self { + std::borrow::Cow::Owned(value) + } +} + +impl<'a> From<&'a AffixFuzzer5> for ::std::borrow::Cow<'a, AffixFuzzer5> { + #[inline] + fn from(value: &'a AffixFuzzer5) -> Self { + std::borrow::Cow::Borrowed(value) + } +} + impl crate::Component for AffixFuzzer5 { #[inline] fn name() -> crate::ComponentName { @@ -402,6 +686,40 @@ impl crate::Component for AffixFuzzer5 { None, ) } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_to_arrow_opt<'a>( + data: impl IntoIterator>>>, + ) -> crate::SerializationResult> + where + Self: Clone + 'a, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let (somes, data0): (Vec<_>, Vec<_>) = data + .into_iter() + .map(|datum| { + let datum: Option<::std::borrow::Cow<'a, Self>> = datum.map(Into::into); + let datum = datum + .map(|datum| { + let Self(data0) = datum.into_owned(); + data0 + }) + .flatten(); + (datum.is_some(), datum) + }) + .unzip(); + let data0_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + { + _ = data0_bitmap; + crate::datatypes::AffixFuzzer1::try_to_arrow_opt(data0)? + } + }) + } } #[derive(Debug, Clone, PartialEq)] @@ -409,6 +727,20 @@ pub struct AffixFuzzer6 { pub single_optional: Option, } +impl<'a> From for ::std::borrow::Cow<'a, AffixFuzzer6> { + #[inline] + fn from(value: AffixFuzzer6) -> Self { + std::borrow::Cow::Owned(value) + } +} + +impl<'a> From<&'a AffixFuzzer6> for ::std::borrow::Cow<'a, AffixFuzzer6> { + #[inline] + fn from(value: &'a AffixFuzzer6) -> Self { + std::borrow::Cow::Borrowed(value) + } +} + impl crate::Component for AffixFuzzer6 { #[inline] fn name() -> crate::ComponentName { @@ -486,6 +818,125 @@ impl crate::Component for AffixFuzzer6 { None, ) } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_to_arrow_opt<'a>( + data: impl IntoIterator>>>, + ) -> crate::SerializationResult> + where + Self: Clone + 'a, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let (somes, data): (Vec<_>, Vec<_>) = data + .into_iter() + .map(|datum| { + let datum: Option<::std::borrow::Cow<'a, Self>> = datum.map(Into::into); + (datum.is_some(), datum) + }) + .unzip(); + let bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + StructArray::new( + DataType::Extension( + "rerun.testing.components.AffixFuzzer6".to_owned(), + Box::new(DataType::Struct(vec![Field { + name: "single_optional".to_owned(), + data_type: DataType::Extension( + "rerun.testing.datatypes.AffixFuzzer1".to_owned(), + Box::new(DataType::Struct(vec![ + Field { + name: "single_float_optional".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "single_string_required".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "single_string_optional".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_floats_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_strings_required".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + })), + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "many_strings_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + ])), + None, + ), + is_nullable: true, + metadata: [].into(), + }])), + None, + ), + vec![{ + let (somes, single_optional): (Vec<_>, Vec<_>) = data + .iter() + .map(|datum| { + let datum = datum + .as_ref() + .map(|datum| { + let Self { + single_optional, .. + } = &**datum; + single_optional.clone() + }) + .flatten(); + (datum.is_some(), datum) + }) + .unzip(); + let single_optional_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + { + _ = single_optional_bitmap; + crate::datatypes::AffixFuzzer1::try_to_arrow_opt(single_optional)? + } + }], + bitmap, + ) + .boxed() + }) + } } #[derive(Debug, Clone, PartialEq)] @@ -499,6 +950,20 @@ pub struct AffixFuzzer7 { pub many_strings_optional: Option>, } +impl<'a> From for ::std::borrow::Cow<'a, AffixFuzzer7> { + #[inline] + fn from(value: AffixFuzzer7) -> Self { + std::borrow::Cow::Owned(value) + } +} + +impl<'a> From<&'a AffixFuzzer7> for ::std::borrow::Cow<'a, AffixFuzzer7> { + #[inline] + fn from(value: &'a AffixFuzzer7) -> Self { + std::borrow::Cow::Borrowed(value) + } +} + impl crate::Component for AffixFuzzer7 { #[inline] fn name() -> crate::ComponentName { @@ -634,4 +1099,645 @@ impl crate::Component for AffixFuzzer7 { None, ) } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_to_arrow_opt<'a>( + data: impl IntoIterator>>>, + ) -> crate::SerializationResult> + where + Self: Clone + 'a, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let (somes, data): (Vec<_>, Vec<_>) = data + .into_iter() + .map(|datum| { + let datum: Option<::std::borrow::Cow<'a, Self>> = datum.map(Into::into); + (datum.is_some(), datum) + }) + .unzip(); + let bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + StructArray::new( + DataType::Extension( + "rerun.testing.components.AffixFuzzer7".to_owned(), + Box::new(DataType::Struct(vec![ + Field { + name: "many_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Extension( + "rerun.testing.datatypes.AffixFuzzer1".to_owned(), + Box::new(DataType::Struct(vec![ + Field { + name: "single_float_optional".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "single_string_required".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "single_string_optional".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_floats_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_strings_required".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + })), + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "many_strings_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + ])), + None, + ), + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "single_float_optional".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "single_string_required".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "single_string_optional".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_floats_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_strings_required".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + })), + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "many_strings_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + ])), + None, + ), + vec![ + { + let (somes, many_optional): (Vec<_>, Vec<_>) = data + .iter() + .map(|datum| { + let datum = datum + .as_ref() + .map(|datum| { + let Self { many_optional, .. } = &**datum; + many_optional.clone() + }) + .flatten(); + (datum.is_some(), datum) + }) + .unzip(); + let many_optional_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + { + use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; + let many_optional_inner_data: Vec<_> = many_optional + .iter() + .flatten() + .flatten() + .map(ToOwned::to_owned) + .map(Some) + .collect(); + let many_optional_inner_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = + many_optional_inner_data.iter().any(|v| v.is_none()); + any_nones.then(|| { + many_optional_inner_data + .iter() + .map(|v| v.is_some()) + .collect() + }) + }; + let offsets = ::arrow2::offset::Offsets::::try_from_lengths( + many_optional.iter().map(|opt| { + opt.as_ref().map(|datum| datum.len()).unwrap_or_default() + }), + ) + .unwrap() + .into(); + ListArray::new( + DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Extension( + "rerun.testing.datatypes.AffixFuzzer1".to_owned(), + Box::new(DataType::Struct(vec![ + Field { + name: "single_float_optional".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "single_string_required".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "single_string_optional".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_floats_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_strings_required".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + })), + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "many_strings_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + ])), + None, + ), + is_nullable: true, + metadata: [].into(), + })), + offsets, + { + _ = many_optional_inner_bitmap; + crate::datatypes::AffixFuzzer1::try_to_arrow_opt( + many_optional_inner_data, + )? + }, + many_optional_bitmap, + ) + .boxed() + } + }, + { + let (somes, single_float_optional): (Vec<_>, Vec<_>) = data + .iter() + .map(|datum| { + let datum = datum + .as_ref() + .map(|datum| { + let Self { + single_float_optional, + .. + } = &**datum; + single_float_optional.clone() + }) + .flatten(); + (datum.is_some(), datum) + }) + .unzip(); + let single_float_optional_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + PrimitiveArray::new( + DataType::Float32, + single_float_optional + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect(), + single_float_optional_bitmap, + ) + .boxed() + }, + { + let (somes, single_string_required): (Vec<_>, Vec<_>) = data + .iter() + .map(|datum| { + let datum = datum.as_ref().map(|datum| { + let Self { + single_string_required, + .. + } = &**datum; + single_string_required.clone() + }); + (datum.is_some(), datum) + }) + .unzip(); + let single_string_required_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + { + let inner_data: ::arrow2::buffer::Buffer = single_string_required + .iter() + .flatten() + .flat_map(|s| s.bytes()) + .collect(); + let offsets = ::arrow2::offset::Offsets::::try_from_lengths( + single_string_required.iter().map(|opt| { + opt.as_ref().map(|datum| datum.len()).unwrap_or_default() + }), + ) + .unwrap() + .into(); + #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] + unsafe { + Utf8Array::::new_unchecked( + DataType::Utf8, + offsets, + inner_data, + single_string_required_bitmap, + ) + } + .boxed() + } + }, + { + let (somes, single_string_optional): (Vec<_>, Vec<_>) = data + .iter() + .map(|datum| { + let datum = datum + .as_ref() + .map(|datum| { + let Self { + single_string_optional, + .. + } = &**datum; + single_string_optional.clone() + }) + .flatten(); + (datum.is_some(), datum) + }) + .unzip(); + let single_string_optional_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + { + let inner_data: ::arrow2::buffer::Buffer = single_string_optional + .iter() + .flatten() + .flat_map(|s| s.bytes()) + .collect(); + let offsets = ::arrow2::offset::Offsets::::try_from_lengths( + single_string_optional.iter().map(|opt| { + opt.as_ref().map(|datum| datum.len()).unwrap_or_default() + }), + ) + .unwrap() + .into(); + #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] + unsafe { + Utf8Array::::new_unchecked( + DataType::Utf8, + offsets, + inner_data, + single_string_optional_bitmap, + ) + } + .boxed() + } + }, + { + let (somes, many_floats_optional): (Vec<_>, Vec<_>) = data + .iter() + .map(|datum| { + let datum = datum + .as_ref() + .map(|datum| { + let Self { + many_floats_optional, + .. + } = &**datum; + many_floats_optional.clone() + }) + .flatten(); + (datum.is_some(), datum) + }) + .unzip(); + let many_floats_optional_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + { + use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; + let many_floats_optional_inner_data: Vec<_> = many_floats_optional + .iter() + .flatten() + .flatten() + .map(ToOwned::to_owned) + .map(Some) + .collect(); + let many_floats_optional_inner_bitmap: Option< + ::arrow2::bitmap::Bitmap, + > = { + let any_nones = + many_floats_optional_inner_data.iter().any(|v| v.is_none()); + any_nones.then(|| { + many_floats_optional_inner_data + .iter() + .map(|v| v.is_some()) + .collect() + }) + }; + let offsets = ::arrow2::offset::Offsets::::try_from_lengths( + many_floats_optional.iter().map(|opt| { + opt.as_ref().map(|datum| datum.len()).unwrap_or_default() + }), + ) + .unwrap() + .into(); + ListArray::new( + DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + })), + offsets, + PrimitiveArray::new( + DataType::Float32, + many_floats_optional_inner_data + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect(), + many_floats_optional_inner_bitmap, + ) + .boxed(), + many_floats_optional_bitmap, + ) + .boxed() + } + }, + { + let (somes, many_strings_required): (Vec<_>, Vec<_>) = data + .iter() + .map(|datum| { + let datum = datum.as_ref().map(|datum| { + let Self { + many_strings_required, + .. + } = &**datum; + many_strings_required.clone() + }); + (datum.is_some(), datum) + }) + .unzip(); + let many_strings_required_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + { + use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; + let many_strings_required_inner_data: Vec<_> = many_strings_required + .iter() + .flatten() + .flatten() + .map(ToOwned::to_owned) + .map(Some) + .collect(); + let many_strings_required_inner_bitmap: Option< + ::arrow2::bitmap::Bitmap, + > = { + let any_nones = + many_strings_required_inner_data.iter().any(|v| v.is_none()); + any_nones.then(|| { + many_strings_required_inner_data + .iter() + .map(|v| v.is_some()) + .collect() + }) + }; + let offsets = ::arrow2::offset::Offsets::::try_from_lengths( + many_strings_required.iter().map(|opt| { + opt.as_ref().map(|datum| datum.len()).unwrap_or_default() + }), + ) + .unwrap() + .into(); + ListArray::new( + DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + })), + offsets, + { + let inner_data: ::arrow2::buffer::Buffer = + many_strings_required_inner_data + .iter() + .flatten() + .flat_map(|s| s.bytes()) + .collect(); + let offsets = + ::arrow2::offset::Offsets::::try_from_lengths( + many_strings_required_inner_data.iter().map(|opt| { + opt.as_ref() + .map(|datum| datum.len()) + .unwrap_or_default() + }), + ) + .unwrap() + .into(); + #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] + unsafe { + Utf8Array::::new_unchecked( + DataType::Utf8, + offsets, + inner_data, + many_strings_required_inner_bitmap, + ) + } + .boxed() + }, + many_strings_required_bitmap, + ) + .boxed() + } + }, + { + let (somes, many_strings_optional): (Vec<_>, Vec<_>) = data + .iter() + .map(|datum| { + let datum = datum + .as_ref() + .map(|datum| { + let Self { + many_strings_optional, + .. + } = &**datum; + many_strings_optional.clone() + }) + .flatten(); + (datum.is_some(), datum) + }) + .unzip(); + let many_strings_optional_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + { + use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; + let many_strings_optional_inner_data: Vec<_> = many_strings_optional + .iter() + .flatten() + .flatten() + .map(ToOwned::to_owned) + .map(Some) + .collect(); + let many_strings_optional_inner_bitmap: Option< + ::arrow2::bitmap::Bitmap, + > = { + let any_nones = + many_strings_optional_inner_data.iter().any(|v| v.is_none()); + any_nones.then(|| { + many_strings_optional_inner_data + .iter() + .map(|v| v.is_some()) + .collect() + }) + }; + let offsets = ::arrow2::offset::Offsets::::try_from_lengths( + many_strings_optional.iter().map(|opt| { + opt.as_ref().map(|datum| datum.len()).unwrap_or_default() + }), + ) + .unwrap() + .into(); + ListArray::new( + DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + })), + offsets, + { + let inner_data: ::arrow2::buffer::Buffer = + many_strings_optional_inner_data + .iter() + .flatten() + .flat_map(|s| s.bytes()) + .collect(); + let offsets = + ::arrow2::offset::Offsets::::try_from_lengths( + many_strings_optional_inner_data.iter().map(|opt| { + opt.as_ref() + .map(|datum| datum.len()) + .unwrap_or_default() + }), + ) + .unwrap() + .into(); + #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] + unsafe { + Utf8Array::::new_unchecked( + DataType::Utf8, + offsets, + inner_data, + many_strings_optional_inner_bitmap, + ) + } + .boxed() + }, + many_strings_optional_bitmap, + ) + .boxed() + } + }, + ], + bitmap, + ) + .boxed() + }) + } } diff --git a/crates/re_types/src/components/instance_key.rs b/crates/re_types/src/components/instance_key.rs index b24fb1ee9ff8..71ca6cd5209d 100644 --- a/crates/re_types/src/components/instance_key.rs +++ b/crates/re_types/src/components/instance_key.rs @@ -12,6 +12,20 @@ #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct InstanceKey(pub u64); +impl<'a> From for ::std::borrow::Cow<'a, InstanceKey> { + #[inline] + fn from(value: InstanceKey) -> Self { + std::borrow::Cow::Owned(value) + } +} + +impl<'a> From<&'a InstanceKey> for ::std::borrow::Cow<'a, InstanceKey> { + #[inline] + fn from(value: &'a InstanceKey) -> Self { + std::borrow::Cow::Borrowed(value) + } +} + impl crate::Component for InstanceKey { #[inline] fn name() -> crate::ComponentName { @@ -28,4 +42,38 @@ impl crate::Component for InstanceKey { None, ) } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_to_arrow_opt<'a>( + data: impl IntoIterator>>>, + ) -> crate::SerializationResult> + where + Self: Clone + 'a, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let (somes, data0): (Vec<_>, Vec<_>) = data + .into_iter() + .map(|datum| { + let datum: Option<::std::borrow::Cow<'a, Self>> = datum.map(Into::into); + let datum = datum.map(|datum| { + let Self(data0) = datum.into_owned(); + data0 + }); + (datum.is_some(), datum) + }) + .unzip(); + let data0_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + PrimitiveArray::new( + DataType::UInt64, + data0.into_iter().map(|v| v.unwrap_or_default()).collect(), + data0_bitmap, + ) + .boxed() + }) + } } diff --git a/crates/re_types/src/components/keypoint_id.rs b/crates/re_types/src/components/keypoint_id.rs index 413858197e7e..c26328d4e4cd 100644 --- a/crates/re_types/src/components/keypoint_id.rs +++ b/crates/re_types/src/components/keypoint_id.rs @@ -16,6 +16,20 @@ #[derive(Copy, Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct KeypointId(pub u16); +impl<'a> From for ::std::borrow::Cow<'a, KeypointId> { + #[inline] + fn from(value: KeypointId) -> Self { + std::borrow::Cow::Owned(value) + } +} + +impl<'a> From<&'a KeypointId> for ::std::borrow::Cow<'a, KeypointId> { + #[inline] + fn from(value: &'a KeypointId) -> Self { + std::borrow::Cow::Borrowed(value) + } +} + impl crate::Component for KeypointId { #[inline] fn name() -> crate::ComponentName { @@ -32,4 +46,38 @@ impl crate::Component for KeypointId { None, ) } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_to_arrow_opt<'a>( + data: impl IntoIterator>>>, + ) -> crate::SerializationResult> + where + Self: Clone + 'a, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let (somes, data0): (Vec<_>, Vec<_>) = data + .into_iter() + .map(|datum| { + let datum: Option<::std::borrow::Cow<'a, Self>> = datum.map(Into::into); + let datum = datum.map(|datum| { + let Self(data0) = datum.into_owned(); + data0 + }); + (datum.is_some(), datum) + }) + .unzip(); + let data0_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + PrimitiveArray::new( + DataType::UInt16, + data0.into_iter().map(|v| v.unwrap_or_default()).collect(), + data0_bitmap, + ) + .boxed() + }) + } } diff --git a/crates/re_types/src/components/label.rs b/crates/re_types/src/components/label.rs index 44618ce60bf3..c7472f727280 100644 --- a/crates/re_types/src/components/label.rs +++ b/crates/re_types/src/components/label.rs @@ -13,6 +13,20 @@ #[repr(transparent)] pub struct Label(pub String); +impl<'a> From