diff --git a/crates/re_types/source_hash.txt b/crates/re_types/source_hash.txt index 2e71ceaad9cd..e82e7d43705a 100644 --- a/crates/re_types/source_hash.txt +++ b/crates/re_types/source_hash.txt @@ -1,4 +1,4 @@ # This is a sha256 hash for all direct and indirect dependencies of this crate's build script. # It can be safely removed at anytime to force the build script to run again. # Check out build.rs to see how it's computed. -70d271f9866784d9f0d25b818cd0d45fff1df72aec62ceae9f167f523a5c2d08 \ No newline at end of file +7d42813c538b5c4716e75e225572409d5799dc142ede36243043ef82fc90399e \ No newline at end of file diff --git a/crates/re_types/src/archetypes/fuzzy.rs b/crates/re_types/src/archetypes/fuzzy.rs index ec710d0d1499..6750b5887290 100644 --- a/crates/re_types/src/archetypes/fuzzy.rs +++ b/crates/re_types/src/archetypes/fuzzy.rs @@ -542,6 +542,439 @@ impl crate::Archetype for AffixFuzzer1 { .flatten() .collect()) } + + #[inline] + fn try_from_arrow( + data: impl IntoIterator)>, + ) -> crate::DeserializationResult { + use crate::Component as _; + let arrays_by_name: ::std::collections::HashMap<_, _> = data + .into_iter() + .map(|(field, array)| (field.name, array)) + .collect(); + let fuzz1001 = { + let array = arrays_by_name.get("fuzz1001").ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + } + })?; + ::try_from_arrow_opt(&**array)? + .into_iter() + .next() + .flatten() + .ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + })? + }; + let fuzz1002 = { + let array = arrays_by_name.get("fuzz1002").ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + } + })?; + ::try_from_arrow_opt(&**array)? + .into_iter() + .next() + .flatten() + .ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + })? + }; + let fuzz1003 = { + let array = arrays_by_name.get("fuzz1003").ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + } + })?; + ::try_from_arrow_opt(&**array)? + .into_iter() + .next() + .flatten() + .ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + })? + }; + let fuzz1004 = { + let array = arrays_by_name.get("fuzz1004").ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + } + })?; + ::try_from_arrow_opt(&**array)? + .into_iter() + .next() + .flatten() + .ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + })? + }; + let fuzz1005 = { + let array = arrays_by_name.get("fuzz1005").ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + } + })?; + ::try_from_arrow_opt(&**array)? + .into_iter() + .next() + .flatten() + .ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + })? + }; + let fuzz1006 = { + let array = arrays_by_name.get("fuzz1006").ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + } + })?; + ::try_from_arrow_opt(&**array)? + .into_iter() + .next() + .flatten() + .ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + })? + }; + let fuzz1007 = { + let array = arrays_by_name.get("fuzz1007").ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + } + })?; + ::try_from_arrow_opt(&**array)? + .into_iter() + .next() + .flatten() + .ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + })? + }; + let fuzz1101 = { + let array = arrays_by_name.get("fuzz1101").ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + } + })?; + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()? + }; + let fuzz1102 = { + let array = arrays_by_name.get("fuzz1102").ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + } + })?; + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()? + }; + let fuzz1103 = { + let array = arrays_by_name.get("fuzz1103").ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + } + })?; + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()? + }; + let fuzz1104 = { + let array = arrays_by_name.get("fuzz1104").ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + } + })?; + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()? + }; + let fuzz1105 = { + let array = arrays_by_name.get("fuzz1105").ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + } + })?; + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()? + }; + let fuzz1106 = { + let array = arrays_by_name.get("fuzz1106").ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + } + })?; + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()? + }; + let fuzz1107 = { + let array = arrays_by_name.get("fuzz1107").ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + } + })?; + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()? + }; + let fuzz2001 = if let Some(array) = arrays_by_name.get("fuzz2001") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .next() + .flatten() + .ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + })?, + ) + } else { + None + }; + let fuzz2002 = if let Some(array) = arrays_by_name.get("fuzz2002") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .next() + .flatten() + .ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + })?, + ) + } else { + None + }; + let fuzz2003 = if let Some(array) = arrays_by_name.get("fuzz2003") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .next() + .flatten() + .ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + })?, + ) + } else { + None + }; + let fuzz2004 = if let Some(array) = arrays_by_name.get("fuzz2004") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .next() + .flatten() + .ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + })?, + ) + } else { + None + }; + let fuzz2005 = if let Some(array) = arrays_by_name.get("fuzz2005") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .next() + .flatten() + .ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + })?, + ) + } else { + None + }; + let fuzz2006 = if let Some(array) = arrays_by_name.get("fuzz2006") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .next() + .flatten() + .ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + })?, + ) + } else { + None + }; + let fuzz2007 = if let Some(array) = arrays_by_name.get("fuzz2007") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .next() + .flatten() + .ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + })?, + ) + } else { + None + }; + let fuzz2101 = if let Some(array) = arrays_by_name.get("fuzz2101") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()?, + ) + } else { + None + }; + let fuzz2102 = if let Some(array) = arrays_by_name.get("fuzz2102") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()?, + ) + } else { + None + }; + let fuzz2103 = if let Some(array) = arrays_by_name.get("fuzz2103") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()?, + ) + } else { + None + }; + let fuzz2104 = if let Some(array) = arrays_by_name.get("fuzz2104") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()?, + ) + } else { + None + }; + let fuzz2105 = if let Some(array) = arrays_by_name.get("fuzz2105") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()?, + ) + } else { + None + }; + let fuzz2106 = if let Some(array) = arrays_by_name.get("fuzz2106") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()?, + ) + } else { + None + }; + let fuzz2107 = if let Some(array) = arrays_by_name.get("fuzz2107") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()?, + ) + } else { + None + }; + Ok(Self { + fuzz1001, + fuzz1002, + fuzz1003, + fuzz1004, + fuzz1005, + fuzz1006, + fuzz1007, + fuzz1101, + fuzz1102, + fuzz1103, + fuzz1104, + fuzz1105, + fuzz1106, + fuzz1107, + fuzz2001, + fuzz2002, + fuzz2003, + fuzz2004, + fuzz2005, + fuzz2006, + fuzz2007, + fuzz2101, + fuzz2102, + fuzz2103, + fuzz2104, + fuzz2105, + fuzz2106, + fuzz2107, + }) + } } impl AffixFuzzer1 { diff --git a/crates/re_types/src/archetypes/points2d.rs b/crates/re_types/src/archetypes/points2d.rs index c30a200665ca..777e980a339a 100644 --- a/crates/re_types/src/archetypes/points2d.rs +++ b/crates/re_types/src/archetypes/points2d.rs @@ -229,6 +229,139 @@ impl crate::Archetype for Points2D { .flatten() .collect()) } + + #[inline] + fn try_from_arrow( + data: impl IntoIterator)>, + ) -> crate::DeserializationResult { + use crate::Component as _; + let arrays_by_name: ::std::collections::HashMap<_, _> = data + .into_iter() + .map(|(field, array)| (field.name, array)) + .collect(); + let points = { + let array = arrays_by_name.get("points").ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + } + })?; + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()? + }; + let radii = if let Some(array) = arrays_by_name.get("radii") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()?, + ) + } else { + None + }; + let colors = if let Some(array) = arrays_by_name.get("colors") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()?, + ) + } else { + None + }; + let labels = if let Some(array) = arrays_by_name.get("labels") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()?, + ) + } else { + None + }; + let draw_order = if let Some(array) = arrays_by_name.get("draw_order") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .next() + .flatten() + .ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + })?, + ) + } else { + None + }; + let class_ids = if let Some(array) = arrays_by_name.get("class_ids") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()?, + ) + } else { + None + }; + let keypoint_ids = if let Some(array) = arrays_by_name.get("keypoint_ids") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()?, + ) + } else { + None + }; + let instance_keys = if let Some(array) = arrays_by_name.get("instance_keys") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()?, + ) + } else { + None + }; + Ok(Self { + points, + radii, + colors, + labels, + draw_order, + class_ids, + keypoint_ids, + instance_keys, + }) + } } impl Points2D { diff --git a/crates/re_types/src/components/class_id.rs b/crates/re_types/src/components/class_id.rs index d60ebaa318a0..a919d261aee8 100644 --- a/crates/re_types/src/components/class_id.rs +++ b/crates/re_types/src/components/class_id.rs @@ -78,4 +78,32 @@ impl crate::Component for ClassId { .boxed() }) } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_from_arrow_opt( + data: &dyn ::arrow2::array::Array, + ) -> crate::DeserializationResult>> + where + Self: Sized, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok(data + .as_any() + .downcast_ref::() + .unwrap() + .into_iter() + .map(|v| v.copied()) + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: DataType::Extension( + "rerun.components.ClassId".to_owned(), + Box::new(DataType::UInt16), + None, + ), + }) + }) + .map(|res| res.map(|v| Some(Self(v)))) + .collect::>>>()?) + } } diff --git a/crates/re_types/src/components/color.rs b/crates/re_types/src/components/color.rs index 1b0a91ffe06b..ca6ae6cd403d 100644 --- a/crates/re_types/src/components/color.rs +++ b/crates/re_types/src/components/color.rs @@ -88,4 +88,32 @@ impl crate::Component for Color { .boxed() }) } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_from_arrow_opt( + data: &dyn ::arrow2::array::Array, + ) -> crate::DeserializationResult>> + where + Self: Sized, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok(data + .as_any() + .downcast_ref::() + .unwrap() + .into_iter() + .map(|v| v.copied()) + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: DataType::Extension( + "rerun.components.Color".to_owned(), + Box::new(DataType::UInt32), + None, + ), + }) + }) + .map(|res| res.map(|v| Some(Self(v)))) + .collect::>>>()?) + } } diff --git a/crates/re_types/src/components/draw_order.rs b/crates/re_types/src/components/draw_order.rs index 1b494ce5ba19..b97a818dac38 100644 --- a/crates/re_types/src/components/draw_order.rs +++ b/crates/re_types/src/components/draw_order.rs @@ -83,4 +83,32 @@ impl crate::Component for DrawOrder { .boxed() }) } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_from_arrow_opt( + data: &dyn ::arrow2::array::Array, + ) -> crate::DeserializationResult>> + where + Self: Sized, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok(data + .as_any() + .downcast_ref::() + .unwrap() + .into_iter() + .map(|v| v.copied()) + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: DataType::Extension( + "rerun.components.DrawOrder".to_owned(), + Box::new(DataType::Float32), + None, + ), + }) + }) + .map(|res| res.map(|v| Some(Self(v)))) + .collect::>>>()?) + } } diff --git a/crates/re_types/src/components/fuzzy.rs b/crates/re_types/src/components/fuzzy.rs index 232383ea7175..0c48d75fc572 100644 --- a/crates/re_types/src/components/fuzzy.rs +++ b/crates/re_types/src/components/fuzzy.rs @@ -131,6 +131,86 @@ impl crate::Component for AffixFuzzer1 { } }) } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_from_arrow_opt( + data: &dyn ::arrow2::array::Array, + ) -> crate::DeserializationResult>> + where + Self: Sized, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok(crate::datatypes::AffixFuzzer1::try_from_arrow_opt(data)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: DataType::Extension( + "rerun.testing.components.AffixFuzzer1".to_owned(), + Box::new(DataType::Extension( + "rerun.testing.datatypes.AffixFuzzer1".to_owned(), + Box::new(DataType::Struct(vec![ + Field { + name: "single_float_optional".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "single_string_required".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "single_string_optional".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_floats_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_strings_required".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + })), + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "many_strings_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + ])), + None, + )), + None, + ), + }) + }) + .map(|res| res.map(|single_required| Some(Self { single_required }))) + .collect::>>>()?) + } } #[derive(Debug, Clone, PartialEq)] @@ -254,6 +334,86 @@ impl crate::Component for AffixFuzzer2 { } }) } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_from_arrow_opt( + data: &dyn ::arrow2::array::Array, + ) -> crate::DeserializationResult>> + where + Self: Sized, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok(crate::datatypes::AffixFuzzer1::try_from_arrow_opt(data)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: DataType::Extension( + "rerun.testing.components.AffixFuzzer2".to_owned(), + Box::new(DataType::Extension( + "rerun.testing.datatypes.AffixFuzzer1".to_owned(), + Box::new(DataType::Struct(vec![ + Field { + name: "single_float_optional".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "single_string_required".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "single_string_optional".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_floats_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_strings_required".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + })), + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "many_strings_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + ])), + None, + )), + None, + ), + }) + }) + .map(|res| res.map(|v| Some(Self(v)))) + .collect::>>>()?) + } } #[derive(Debug, Clone, PartialEq)] @@ -468,6 +628,197 @@ impl crate::Component for AffixFuzzer3 { .boxed() }) } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_from_arrow_opt( + data: &dyn ::arrow2::array::Array, + ) -> crate::DeserializationResult>> + where + Self: Sized, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let data = data + .as_any() + .downcast_ref::<::arrow2::array::StructArray>() + .ok_or_else(|| crate::DeserializationError::SchemaMismatch { + expected: DataType::Extension( + "rerun.testing.components.AffixFuzzer3".to_owned(), + Box::new(DataType::Struct(vec![Field { + name: "single_required".to_owned(), + data_type: DataType::Extension( + "rerun.testing.datatypes.AffixFuzzer1".to_owned(), + Box::new(DataType::Struct(vec![ + Field { + name: "single_float_optional".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "single_string_required".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "single_string_optional".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_floats_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_strings_required".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + })), + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "many_strings_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + ])), + None, + ), + is_nullable: false, + metadata: [].into(), + }])), + None, + ), + got: data.data_type().clone(), + })?; + let (data_fields, data_arrays, data_bitmap) = + (data.fields(), data.values(), data.validity()); + let is_valid = |i| data_bitmap.map_or(true, |bitmap| bitmap.get_bit(i)); + let arrays_by_name: ::std::collections::HashMap<_, _> = data_fields + .iter() + .map(|field| field.name.as_str()) + .zip(data_arrays) + .collect(); + let single_required = { + let data = &**arrays_by_name["single_required"]; + + crate::datatypes::AffixFuzzer1::try_from_arrow_opt(data)?.into_iter() + }; + ::itertools::izip!(single_required) + .enumerate() + .map(|(i, (single_required))| { + is_valid(i) + .then(|| { + Ok(Self { + single_required: single_required.ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: DataType::Extension( + "rerun.testing.components.AffixFuzzer3".to_owned(), + Box::new(DataType::Struct(vec![Field { + name: "single_required".to_owned(), + data_type: DataType::Extension( + "rerun.testing.datatypes.AffixFuzzer1" + .to_owned(), + Box::new(DataType::Struct(vec![ + Field { + name: "single_float_optional" + .to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "single_string_required" + .to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "single_string_optional" + .to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_floats_optional".to_owned(), + data_type: DataType::List(Box::new( + Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }, + )), + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_strings_required" + .to_owned(), + data_type: DataType::List(Box::new( + Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + }, + )), + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "many_strings_optional" + .to_owned(), + data_type: DataType::List(Box::new( + Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + }, + )), + is_nullable: true, + metadata: [].into(), + }, + ])), + None, + ), + is_nullable: false, + metadata: [].into(), + }])), + None, + ), + } + })?, + }) + }) + .transpose() + }) + .collect::>>()? + }) + } } #[derive(Debug, Clone, PartialEq)] @@ -595,6 +946,22 @@ impl crate::Component for AffixFuzzer4 { } }) } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_from_arrow_opt( + data: &dyn ::arrow2::array::Array, + ) -> crate::DeserializationResult>> + where + Self: Sized, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok(crate::datatypes::AffixFuzzer1::try_from_arrow_opt(data)? + .into_iter() + .map(Ok) + .map(|res| res.map(|single_optional| Some(Self { single_optional }))) + .collect::>>>()?) + } } #[derive(Debug, Clone, PartialEq)] @@ -720,6 +1087,22 @@ impl crate::Component for AffixFuzzer5 { } }) } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_from_arrow_opt( + data: &dyn ::arrow2::array::Array, + ) -> crate::DeserializationResult>> + where + Self: Sized, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok(crate::datatypes::AffixFuzzer1::try_from_arrow_opt(data)? + .into_iter() + .map(Ok) + .map(|res| res.map(|v| Some(Self(v)))) + .collect::>>>()?) + } } #[derive(Debug, Clone, PartialEq)] @@ -937,6 +1320,112 @@ impl crate::Component for AffixFuzzer6 { .boxed() }) } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_from_arrow_opt( + data: &dyn ::arrow2::array::Array, + ) -> crate::DeserializationResult>> + where + Self: Sized, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let data = data + .as_any() + .downcast_ref::<::arrow2::array::StructArray>() + .ok_or_else(|| crate::DeserializationError::SchemaMismatch { + expected: DataType::Extension( + "rerun.testing.components.AffixFuzzer6".to_owned(), + Box::new(DataType::Struct(vec![Field { + name: "single_optional".to_owned(), + data_type: DataType::Extension( + "rerun.testing.datatypes.AffixFuzzer1".to_owned(), + Box::new(DataType::Struct(vec![ + Field { + name: "single_float_optional".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "single_string_required".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "single_string_optional".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_floats_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_strings_required".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + })), + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "many_strings_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + ])), + None, + ), + is_nullable: true, + metadata: [].into(), + }])), + None, + ), + got: data.data_type().clone(), + })?; + let (data_fields, data_arrays, data_bitmap) = + (data.fields(), data.values(), data.validity()); + let is_valid = |i| data_bitmap.map_or(true, |bitmap| bitmap.get_bit(i)); + let arrays_by_name: ::std::collections::HashMap<_, _> = data_fields + .iter() + .map(|field| field.name.as_str()) + .zip(data_arrays) + .collect(); + let single_optional = { + let data = &**arrays_by_name["single_optional"]; + + crate::datatypes::AffixFuzzer1::try_from_arrow_opt(data)?.into_iter() + }; + ::itertools::izip!(single_optional) + .enumerate() + .map(|(i, (single_optional))| { + is_valid(i) + .then(|| Ok(Self { single_optional })) + .transpose() + }) + .collect::>>()? + }) + } } #[derive(Debug, Clone, PartialEq)] @@ -1740,4 +2229,849 @@ impl crate::Component for AffixFuzzer7 { .boxed() }) } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_from_arrow_opt( + data: &dyn ::arrow2::array::Array, + ) -> crate::DeserializationResult>> + where + Self: Sized, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let data = data + .as_any() + .downcast_ref::<::arrow2::array::StructArray>() + .ok_or_else(|| crate::DeserializationError::SchemaMismatch { + expected: DataType::Extension( + "rerun.testing.components.AffixFuzzer7".to_owned(), + Box::new(DataType::Struct(vec![ + Field { + name: "many_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Extension( + "rerun.testing.datatypes.AffixFuzzer1".to_owned(), + Box::new(DataType::Struct(vec![ + Field { + name: "single_float_optional".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "single_string_required".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "single_string_optional".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_floats_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_strings_required".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + })), + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "many_strings_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + ])), + None, + ), + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "single_float_optional".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "single_string_required".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "single_string_optional".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_floats_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_strings_required".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + })), + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "many_strings_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + ])), + None, + ), + got: data.data_type().clone(), + })?; + let (data_fields, data_arrays, data_bitmap) = + (data.fields(), data.values(), data.validity()); + let is_valid = |i| data_bitmap.map_or(true, |bitmap| bitmap.get_bit(i)); + let arrays_by_name: ::std::collections::HashMap<_, _> = data_fields + .iter() + .map(|field| field.name.as_str()) + .zip(data_arrays) + .collect(); + let many_optional = { + let data = &**arrays_by_name["many_optional"]; + + { + let data = data + .as_any() + .downcast_ref::<::arrow2::array::ListArray>() + .unwrap(); + let bitmap = data.validity().cloned(); + let offsets = { + let offsets = data.offsets(); + offsets.iter().copied().zip(offsets.iter().copied().skip(1)) + }; + let data = &**data.values(); + let data = crate::datatypes::AffixFuzzer1::try_from_arrow_opt(data)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: DataType::Extension( + "rerun.testing.datatypes.AffixFuzzer1".to_owned(), + Box::new(DataType::Struct(vec![ + Field { + name: "single_float_optional".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "single_string_required".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "single_string_optional".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_floats_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_strings_required".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + })), + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "many_strings_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + ])), + None, + ), + }) + }) + .collect::>>()?; + offsets + .enumerate() + .map(move |(i, (start, end))| { + bitmap + .as_ref() + .map_or(true, |bitmap| bitmap.get_bit(i)) + .then(|| { + Ok(data + .get(start as usize..end as usize) + .ok_or_else(|| { + crate::DeserializationError::OffsetsMismatch { + bounds: (start as usize, end as usize), + len: data.len(), + datatype: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Extension( + "rerun.testing.datatypes.AffixFuzzer1" + .to_owned(), + Box::new(DataType::Struct(vec![ + Field { + name: "single_float_optional" + .to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "single_string_required" + .to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "single_string_optional" + .to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_floats_optional" + .to_owned(), + data_type: DataType::List( + Box::new(Field { + name: "item".to_owned(), + data_type: + DataType::Float32, + is_nullable: true, + metadata: [].into(), + }), + ), + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_strings_required" + .to_owned(), + data_type: DataType::List( + Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + }), + ), + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "many_strings_optional" + .to_owned(), + data_type: DataType::List( + Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + }), + ), + is_nullable: true, + metadata: [].into(), + }, + ])), + None, + ), + is_nullable: true, + metadata: [].into(), + })), + } + })? + .to_vec()) + }) + .transpose() + }) + .collect::>>>()? + .into_iter() + } + }; + let single_float_optional = { + let data = &**arrays_by_name["single_float_optional"]; + + data.as_any() + .downcast_ref::() + .unwrap() + .into_iter() + .map(|v| v.copied()) + }; + let single_string_required = { + let data = &**arrays_by_name["single_string_required"]; + + data.as_any() + .downcast_ref::>() + .unwrap() + .into_iter() + .map(|v| v.map(ToOwned::to_owned)) + }; + let single_string_optional = { + let data = &**arrays_by_name["single_string_optional"]; + + data.as_any() + .downcast_ref::>() + .unwrap() + .into_iter() + .map(|v| v.map(ToOwned::to_owned)) + }; + let many_floats_optional = { + let data = &**arrays_by_name["many_floats_optional"]; + + { + let data = data + .as_any() + .downcast_ref::<::arrow2::array::ListArray>() + .unwrap(); + let bitmap = data.validity().cloned(); + let offsets = { + let offsets = data.offsets(); + offsets.iter().copied().zip(offsets.iter().copied().skip(1)) + }; + let data = &**data.values(); + let data = data + .as_any() + .downcast_ref::() + .unwrap() + .into_iter() + .map(|v| v.copied()) + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: DataType::Float32, + }) + }) + .collect::>>()?; + offsets + .enumerate() + .map(move |(i, (start, end))| { + bitmap + .as_ref() + .map_or(true, |bitmap| bitmap.get_bit(i)) + .then(|| { + Ok(data + .get(start as usize..end as usize) + .ok_or_else(|| { + crate::DeserializationError::OffsetsMismatch { + bounds: (start as usize, end as usize), + len: data.len(), + datatype: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + })), + } + })? + .to_vec()) + }) + .transpose() + }) + .collect::>>>()? + .into_iter() + } + }; + let many_strings_required = { + let data = &**arrays_by_name["many_strings_required"]; + + { + let data = data + .as_any() + .downcast_ref::<::arrow2::array::ListArray>() + .unwrap(); + let bitmap = data.validity().cloned(); + let offsets = { + let offsets = data.offsets(); + offsets.iter().copied().zip(offsets.iter().copied().skip(1)) + }; + let data = &**data.values(); + let data = data + .as_any() + .downcast_ref::>() + .unwrap() + .into_iter() + .map(|v| v.map(ToOwned::to_owned)) + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: DataType::Utf8, + }) + }) + .collect::>>()?; + offsets + .enumerate() + .map(move |(i, (start, end))| { + bitmap + .as_ref() + .map_or(true, |bitmap| bitmap.get_bit(i)) + .then(|| { + Ok(data + .get(start as usize..end as usize) + .ok_or_else(|| { + crate::DeserializationError::OffsetsMismatch { + bounds: (start as usize, end as usize), + len: data.len(), + datatype: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + })), + } + })? + .to_vec()) + }) + .transpose() + }) + .collect::>>>()? + .into_iter() + } + }; + let many_strings_optional = { + let data = &**arrays_by_name["many_strings_optional"]; + + { + let data = data + .as_any() + .downcast_ref::<::arrow2::array::ListArray>() + .unwrap(); + let bitmap = data.validity().cloned(); + let offsets = { + let offsets = data.offsets(); + offsets.iter().copied().zip(offsets.iter().copied().skip(1)) + }; + let data = &**data.values(); + let data = data + .as_any() + .downcast_ref::>() + .unwrap() + .into_iter() + .map(|v| v.map(ToOwned::to_owned)) + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: DataType::Utf8, + }) + }) + .collect::>>()?; + offsets + .enumerate() + .map(move |(i, (start, end))| { + bitmap + .as_ref() + .map_or(true, |bitmap| bitmap.get_bit(i)) + .then(|| { + Ok(data + .get(start as usize..end as usize) + .ok_or_else(|| { + crate::DeserializationError::OffsetsMismatch { + bounds: (start as usize, end as usize), + len: data.len(), + datatype: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + })), + } + })? + .to_vec()) + }) + .transpose() + }) + .collect::>>>()? + .into_iter() + } + }; + ::itertools::izip!( + many_optional, + single_float_optional, + single_string_required, + single_string_optional, + many_floats_optional, + many_strings_required, + many_strings_optional + ) + .enumerate() + .map( + |( + i, + ( + many_optional, + single_float_optional, + single_string_required, + single_string_optional, + many_floats_optional, + many_strings_required, + many_strings_optional, + ), + )| { + is_valid(i) + .then(|| { + Ok(Self { + many_optional, + single_float_optional, + single_string_required: single_string_required.ok_or_else( + || crate::DeserializationError::MissingData { + datatype: DataType::Extension( + "rerun.testing.components.AffixFuzzer7".to_owned(), + Box::new(DataType::Struct(vec![ + Field { + name: "many_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Extension( + "rerun.testing.datatypes.AffixFuzzer1" + .to_owned(), + Box::new(DataType::Struct(vec![ + Field { + name: "single_float_optional" + .to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "single_string_required" + .to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "single_string_optional" + .to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_floats_optional" + .to_owned(), + data_type: DataType::List( + Box::new(Field { + name: "item".to_owned(), + data_type: + DataType::Float32, + is_nullable: true, + metadata: [].into(), + }), + ), + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_strings_required" + .to_owned(), + data_type: DataType::List( + Box::new(Field { + name: "item".to_owned(), + data_type: + DataType::Utf8, + is_nullable: false, + metadata: [].into(), + }), + ), + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "many_strings_optional" + .to_owned(), + data_type: DataType::List( + Box::new(Field { + name: "item".to_owned(), + data_type: + DataType::Utf8, + is_nullable: true, + metadata: [].into(), + }), + ), + is_nullable: true, + metadata: [].into(), + }, + ])), + None, + ), + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "single_float_optional".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "single_string_required".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "single_string_optional".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_floats_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_strings_required".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + })), + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "many_strings_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + ])), + None, + ), + }, + )?, + single_string_optional, + many_floats_optional, + many_strings_required: many_strings_required.ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: DataType::Extension( + "rerun.testing.components.AffixFuzzer7".to_owned(), + Box::new(DataType::Struct(vec![ + Field { + name: "many_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Extension( + "rerun.testing.datatypes.AffixFuzzer1" + .to_owned(), + Box::new(DataType::Struct(vec![ + Field { + name: "single_float_optional" + .to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "single_string_required" + .to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "single_string_optional" + .to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_floats_optional" + .to_owned(), + data_type: DataType::List( + Box::new(Field { + name: "item".to_owned(), + data_type: + DataType::Float32, + is_nullable: true, + metadata: [].into(), + }), + ), + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_strings_required" + .to_owned(), + data_type: DataType::List( + Box::new(Field { + name: "item".to_owned(), + data_type: + DataType::Utf8, + is_nullable: false, + metadata: [].into(), + }), + ), + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "many_strings_optional" + .to_owned(), + data_type: DataType::List( + Box::new(Field { + name: "item".to_owned(), + data_type: + DataType::Utf8, + is_nullable: true, + metadata: [].into(), + }), + ), + is_nullable: true, + metadata: [].into(), + }, + ])), + None, + ), + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "single_float_optional".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "single_string_required".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "single_string_optional".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_floats_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_strings_required".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + })), + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "many_strings_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + ])), + None, + ), + } + })?, + many_strings_optional, + }) + }) + .transpose() + }, + ) + .collect::>>()? + }) + } } diff --git a/crates/re_types/src/components/instance_key.rs b/crates/re_types/src/components/instance_key.rs index 71ca6cd5209d..feb8939dc39e 100644 --- a/crates/re_types/src/components/instance_key.rs +++ b/crates/re_types/src/components/instance_key.rs @@ -76,4 +76,32 @@ impl crate::Component for InstanceKey { .boxed() }) } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_from_arrow_opt( + data: &dyn ::arrow2::array::Array, + ) -> crate::DeserializationResult>> + where + Self: Sized, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok(data + .as_any() + .downcast_ref::() + .unwrap() + .into_iter() + .map(|v| v.copied()) + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: DataType::Extension( + "rerun.components.InstanceKey".to_owned(), + Box::new(DataType::UInt64), + None, + ), + }) + }) + .map(|res| res.map(|v| Some(Self(v)))) + .collect::>>>()?) + } } diff --git a/crates/re_types/src/components/keypoint_id.rs b/crates/re_types/src/components/keypoint_id.rs index c26328d4e4cd..678890ccd866 100644 --- a/crates/re_types/src/components/keypoint_id.rs +++ b/crates/re_types/src/components/keypoint_id.rs @@ -80,4 +80,32 @@ impl crate::Component for KeypointId { .boxed() }) } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_from_arrow_opt( + data: &dyn ::arrow2::array::Array, + ) -> crate::DeserializationResult>> + where + Self: Sized, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok(data + .as_any() + .downcast_ref::() + .unwrap() + .into_iter() + .map(|v| v.copied()) + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: DataType::Extension( + "rerun.components.KeypointId".to_owned(), + Box::new(DataType::UInt16), + None, + ), + }) + }) + .map(|res| res.map(|v| Some(Self(v)))) + .collect::>>>()?) + } } diff --git a/crates/re_types/src/components/label.rs b/crates/re_types/src/components/label.rs index c7472f727280..559afeeee02c 100644 --- a/crates/re_types/src/components/label.rs +++ b/crates/re_types/src/components/label.rs @@ -92,4 +92,32 @@ impl crate::Component for Label { } }) } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_from_arrow_opt( + data: &dyn ::arrow2::array::Array, + ) -> crate::DeserializationResult>> + where + Self: Sized, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok(data + .as_any() + .downcast_ref::>() + .unwrap() + .into_iter() + .map(|v| v.map(ToOwned::to_owned)) + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: DataType::Extension( + "rerun.components.Label".to_owned(), + Box::new(DataType::Utf8), + None, + ), + }) + }) + .map(|res| res.map(|v| Some(Self(v)))) + .collect::>>>()?) + } } diff --git a/crates/re_types/src/components/point2d.rs b/crates/re_types/src/components/point2d.rs index 4f19a2a09b5b..b4347e32d5e2 100644 --- a/crates/re_types/src/components/point2d.rs +++ b/crates/re_types/src/components/point2d.rs @@ -150,4 +150,118 @@ impl crate::Component for Point2D { .boxed() }) } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_from_arrow_opt( + data: &dyn ::arrow2::array::Array, + ) -> crate::DeserializationResult>> + where + Self: Sized, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let data = data + .as_any() + .downcast_ref::<::arrow2::array::StructArray>() + .ok_or_else(|| crate::DeserializationError::SchemaMismatch { + expected: DataType::Extension( + "rerun.components.Point2D".to_owned(), + Box::new(DataType::Struct(vec![ + Field { + name: "x".to_owned(), + data_type: DataType::Float32, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "y".to_owned(), + data_type: DataType::Float32, + is_nullable: false, + metadata: [].into(), + }, + ])), + None, + ), + got: data.data_type().clone(), + })?; + let (data_fields, data_arrays, data_bitmap) = + (data.fields(), data.values(), data.validity()); + let is_valid = |i| data_bitmap.map_or(true, |bitmap| bitmap.get_bit(i)); + let arrays_by_name: ::std::collections::HashMap<_, _> = data_fields + .iter() + .map(|field| field.name.as_str()) + .zip(data_arrays) + .collect(); + let x = { + let data = &**arrays_by_name["x"]; + + data.as_any() + .downcast_ref::() + .unwrap() + .into_iter() + .map(|v| v.copied()) + }; + let y = { + let data = &**arrays_by_name["y"]; + + data.as_any() + .downcast_ref::() + .unwrap() + .into_iter() + .map(|v| v.copied()) + }; + ::itertools::izip!(x, y) + .enumerate() + .map(|(i, (x, y))| { + is_valid(i) + .then(|| { + Ok(Self { + x: x.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: DataType::Extension( + "rerun.components.Point2D".to_owned(), + Box::new(DataType::Struct(vec![ + Field { + name: "x".to_owned(), + data_type: DataType::Float32, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "y".to_owned(), + data_type: DataType::Float32, + is_nullable: false, + metadata: [].into(), + }, + ])), + None, + ), + })?, + y: y.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: DataType::Extension( + "rerun.components.Point2D".to_owned(), + Box::new(DataType::Struct(vec![ + Field { + name: "x".to_owned(), + data_type: DataType::Float32, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "y".to_owned(), + data_type: DataType::Float32, + is_nullable: false, + metadata: [].into(), + }, + ])), + None, + ), + })?, + }) + }) + .transpose() + }) + .collect::>>()? + }) + } } diff --git a/crates/re_types/src/components/radius.rs b/crates/re_types/src/components/radius.rs index 551ddbf090e8..6226849f4e06 100644 --- a/crates/re_types/src/components/radius.rs +++ b/crates/re_types/src/components/radius.rs @@ -76,4 +76,32 @@ impl crate::Component for Radius { .boxed() }) } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_from_arrow_opt( + data: &dyn ::arrow2::array::Array, + ) -> crate::DeserializationResult>> + where + Self: Sized, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok(data + .as_any() + .downcast_ref::() + .unwrap() + .into_iter() + .map(|v| v.copied()) + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: DataType::Extension( + "rerun.components.Radius".to_owned(), + Box::new(DataType::Float32), + None, + ), + }) + }) + .map(|res| res.map(|v| Some(Self(v)))) + .collect::>>>()?) + } } diff --git a/crates/re_types/src/datatypes/fuzzy.rs b/crates/re_types/src/datatypes/fuzzy.rs index ea9a2a91e84e..cd9610dbf52f 100644 --- a/crates/re_types/src/datatypes/fuzzy.rs +++ b/crates/re_types/src/datatypes/fuzzy.rs @@ -555,6 +555,438 @@ impl crate::Datatype for AffixFuzzer1 { .boxed() }) } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_from_arrow_opt( + data: &dyn ::arrow2::array::Array, + ) -> crate::DeserializationResult>> + where + Self: Sized, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let data = data + .as_any() + .downcast_ref::<::arrow2::array::StructArray>() + .ok_or_else(|| crate::DeserializationError::SchemaMismatch { + expected: DataType::Extension( + "rerun.testing.datatypes.AffixFuzzer1".to_owned(), + Box::new(DataType::Struct(vec![ + Field { + name: "single_float_optional".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "single_string_required".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "single_string_optional".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_floats_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_strings_required".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + })), + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "many_strings_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + ])), + None, + ), + got: data.data_type().clone(), + })?; + let (data_fields, data_arrays, data_bitmap) = + (data.fields(), data.values(), data.validity()); + let is_valid = |i| data_bitmap.map_or(true, |bitmap| bitmap.get_bit(i)); + let arrays_by_name: ::std::collections::HashMap<_, _> = data_fields + .iter() + .map(|field| field.name.as_str()) + .zip(data_arrays) + .collect(); + let single_float_optional = { + let data = &**arrays_by_name["single_float_optional"]; + + data.as_any() + .downcast_ref::() + .unwrap() + .into_iter() + .map(|v| v.copied()) + }; + let single_string_required = { + let data = &**arrays_by_name["single_string_required"]; + + data.as_any() + .downcast_ref::>() + .unwrap() + .into_iter() + .map(|v| v.map(ToOwned::to_owned)) + }; + let single_string_optional = { + let data = &**arrays_by_name["single_string_optional"]; + + data.as_any() + .downcast_ref::>() + .unwrap() + .into_iter() + .map(|v| v.map(ToOwned::to_owned)) + }; + let many_floats_optional = { + let data = &**arrays_by_name["many_floats_optional"]; + + { + let data = data + .as_any() + .downcast_ref::<::arrow2::array::ListArray>() + .unwrap(); + let bitmap = data.validity().cloned(); + let offsets = { + let offsets = data.offsets(); + offsets.iter().copied().zip(offsets.iter().copied().skip(1)) + }; + let data = &**data.values(); + let data = data + .as_any() + .downcast_ref::() + .unwrap() + .into_iter() + .map(|v| v.copied()) + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: DataType::Float32, + }) + }) + .collect::>>()?; + offsets + .enumerate() + .map(move |(i, (start, end))| { + bitmap + .as_ref() + .map_or(true, |bitmap| bitmap.get_bit(i)) + .then(|| { + Ok(data + .get(start as usize..end as usize) + .ok_or_else(|| { + crate::DeserializationError::OffsetsMismatch { + bounds: (start as usize, end as usize), + len: data.len(), + datatype: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + })), + } + })? + .to_vec()) + }) + .transpose() + }) + .collect::>>>()? + .into_iter() + } + }; + let many_strings_required = { + let data = &**arrays_by_name["many_strings_required"]; + + { + let data = data + .as_any() + .downcast_ref::<::arrow2::array::ListArray>() + .unwrap(); + let bitmap = data.validity().cloned(); + let offsets = { + let offsets = data.offsets(); + offsets.iter().copied().zip(offsets.iter().copied().skip(1)) + }; + let data = &**data.values(); + let data = data + .as_any() + .downcast_ref::>() + .unwrap() + .into_iter() + .map(|v| v.map(ToOwned::to_owned)) + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: DataType::Utf8, + }) + }) + .collect::>>()?; + offsets + .enumerate() + .map(move |(i, (start, end))| { + bitmap + .as_ref() + .map_or(true, |bitmap| bitmap.get_bit(i)) + .then(|| { + Ok(data + .get(start as usize..end as usize) + .ok_or_else(|| { + crate::DeserializationError::OffsetsMismatch { + bounds: (start as usize, end as usize), + len: data.len(), + datatype: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + })), + } + })? + .to_vec()) + }) + .transpose() + }) + .collect::>>>()? + .into_iter() + } + }; + let many_strings_optional = { + let data = &**arrays_by_name["many_strings_optional"]; + + { + let data = data + .as_any() + .downcast_ref::<::arrow2::array::ListArray>() + .unwrap(); + let bitmap = data.validity().cloned(); + let offsets = { + let offsets = data.offsets(); + offsets.iter().copied().zip(offsets.iter().copied().skip(1)) + }; + let data = &**data.values(); + let data = data + .as_any() + .downcast_ref::>() + .unwrap() + .into_iter() + .map(|v| v.map(ToOwned::to_owned)) + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: DataType::Utf8, + }) + }) + .collect::>>()?; + offsets + .enumerate() + .map(move |(i, (start, end))| { + bitmap + .as_ref() + .map_or(true, |bitmap| bitmap.get_bit(i)) + .then(|| { + Ok(data + .get(start as usize..end as usize) + .ok_or_else(|| { + crate::DeserializationError::OffsetsMismatch { + bounds: (start as usize, end as usize), + len: data.len(), + datatype: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + })), + } + })? + .to_vec()) + }) + .transpose() + }) + .collect::>>>()? + .into_iter() + } + }; + ::itertools::izip!( + single_float_optional, + single_string_required, + single_string_optional, + many_floats_optional, + many_strings_required, + many_strings_optional + ) + .enumerate() + .map( + |( + i, + ( + single_float_optional, + single_string_required, + single_string_optional, + many_floats_optional, + many_strings_required, + many_strings_optional, + ), + )| { + is_valid(i) + .then(|| { + Ok(Self { + single_float_optional, + single_string_required: single_string_required.ok_or_else( + || crate::DeserializationError::MissingData { + datatype: DataType::Extension( + "rerun.testing.datatypes.AffixFuzzer1".to_owned(), + Box::new(DataType::Struct(vec![ + Field { + name: "single_float_optional".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "single_string_required".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "single_string_optional".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_floats_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_strings_required".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + })), + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "many_strings_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + ])), + None, + ), + }, + )?, + single_string_optional, + many_floats_optional, + many_strings_required: many_strings_required.ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: DataType::Extension( + "rerun.testing.datatypes.AffixFuzzer1".to_owned(), + Box::new(DataType::Struct(vec![ + Field { + name: "single_float_optional".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "single_string_required".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "single_string_optional".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_floats_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_strings_required".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + })), + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "many_strings_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + ])), + None, + ), + } + })?, + many_strings_optional, + }) + }) + .transpose() + }, + ) + .collect::>>()? + }) + } } #[derive(Debug, Clone, PartialEq)] @@ -626,4 +1058,24 @@ impl crate::Datatype for AffixFuzzer2 { .boxed() }) } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_from_arrow_opt( + data: &dyn ::arrow2::array::Array, + ) -> crate::DeserializationResult>> + where + Self: Sized, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok(data + .as_any() + .downcast_ref::() + .unwrap() + .into_iter() + .map(|v| v.copied()) + .map(Ok) + .map(|res| res.map(|v| Some(Self(v)))) + .collect::>>>()?) + } } diff --git a/crates/re_types/src/datatypes/vec2d.rs b/crates/re_types/src/datatypes/vec2d.rs index a6d57c3c3d01..b0569e189178 100644 --- a/crates/re_types/src/datatypes/vec2d.rs +++ b/crates/re_types/src/datatypes/vec2d.rs @@ -114,4 +114,96 @@ impl crate::Datatype for Vec2D { } }) } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_from_arrow_opt( + data: &dyn ::arrow2::array::Array, + ) -> crate::DeserializationResult>> + where + Self: Sized, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let data = data + .as_any() + .downcast_ref::<::arrow2::array::ListArray>() + .unwrap(); + let bitmap = data.validity().cloned(); + let offsets = (0..).step_by(2usize).zip((2usize..).step_by(2usize)); + let data = &**data.values(); + let data = data + .as_any() + .downcast_ref::() + .unwrap() + .into_iter() + .map(|v| v.copied()) + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: DataType::Float32, + }) + }) + .collect::>>()?; + offsets + .enumerate() + .map(move |(i, (start, end))| { + bitmap + .as_ref() + .map_or(true, |bitmap| bitmap.get_bit(i)) + .then(|| { + data.get(start as usize..end as usize) + .ok_or_else(|| crate::DeserializationError::OffsetsMismatch { + bounds: (start as usize, end as usize), + len: data.len(), + datatype: DataType::FixedSizeList( + Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: false, + metadata: [].into(), + }), + 2usize, + ), + })? + .to_vec() + .try_into() + .map_err(|_err| crate::DeserializationError::ArrayLengthMismatch { + expected: 2usize, + got: (end - start) as usize, + datatype: DataType::FixedSizeList( + Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: false, + metadata: [].into(), + }), + 2usize, + ), + }) + }) + .transpose() + }) + .collect::>>>()? + .into_iter() + } + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: DataType::Extension( + "rerun.datatypes.Vec2D".to_owned(), + Box::new(DataType::FixedSizeList( + Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: false, + metadata: [].into(), + }), + 2usize, + )), + None, + ), + }) + }) + .map(|res| res.map(|v| Some(Self(v)))) + .collect::>>>()?) + } } diff --git a/crates/re_types/src/lib.rs b/crates/re_types/src/lib.rs index 449ea497d686..b619fae76d60 100644 --- a/crates/re_types/src/lib.rs +++ b/crates/re_types/src/lib.rs @@ -152,6 +152,67 @@ pub trait Datatype { ) -> SerializationResult> where Self: Clone + 'a; + + // --- + + /// Given an Arrow array, deserializes it into a collection of [`Datatype`]s. + /// + /// Panics if the data schema doesn't match, or if optional entries were missing at runtime. + /// For the non-fallible version, see [`Datatype::try_from_arrow`]. + #[inline] + fn from_arrow(data: &dyn ::arrow2::array::Array) -> Vec + where + Self: Sized, + { + Self::from_arrow_opt(data) + .into_iter() + .map(Option::unwrap) + .collect() + } + + /// Given an Arrow array, deserializes it into a collection of optional [`Datatype`]s. + /// + /// This will _never_ fail for if the Arrow array's datatype matches the one returned by + /// [`Datatype::to_arrow_datatype`]. + /// For the non-fallible version, see [`Datatype::from_arrow_opt`]. + #[inline] + fn try_from_arrow(data: &dyn ::arrow2::array::Array) -> DeserializationResult> + where + Self: Sized, + { + Self::try_from_arrow_opt(data)? + .into_iter() + .map(|v| { + v.ok_or_else(|| DeserializationError::MissingData { + datatype: data.data_type().clone(), + }) + }) + .collect() + } + + /// Given an Arrow array, deserializes it into a collection of optional [`Datatype`]s. + /// + /// This will _never_ fail for if the Arrow array's datatype matches the one returned by + /// [`Datatype::to_arrow_datatype`]. + /// For the fallible version, see [`Datatype::try_from_arrow_opt`]. + #[inline] + fn from_arrow_opt(data: &dyn ::arrow2::array::Array) -> Vec> + where + Self: Sized, + { + Self::try_from_arrow_opt(data).unwrap() + } + + /// Given an Arrow array, deserializes it into a collection of optional [`Datatype`]s. + /// + /// This will _never_ fail for if the Arrow array's datatype matches the one returned by + /// [`Datatype::to_arrow_datatype`]. + /// For the non-fallible version, see [`Datatype::from_arrow_opt`]. + fn try_from_arrow_opt( + data: &dyn ::arrow2::array::Array, + ) -> DeserializationResult>> + where + Self: Sized; } /// The fully-qualified name of a [`Component`], e.g. `rerun.components.Point2D`. @@ -229,6 +290,67 @@ pub trait Component { ) -> SerializationResult> where Self: Clone + 'a; + + // --- + + /// Given an Arrow array, deserializes it into a collection of [`Component`]s. + /// + /// Panics if the data schema doesn't match, or if optional entries were missing at runtime. + /// For the non-fallible version, see [`Component::try_from_arrow`]. + #[inline] + fn from_arrow(data: &dyn ::arrow2::array::Array) -> Vec + where + Self: Sized, + { + Self::from_arrow_opt(data) + .into_iter() + .map(Option::unwrap) + .collect() + } + + /// Given an Arrow array, deserializes it into a collection of optional [`Component`]s. + /// + /// This will _never_ fail for if the Arrow array's datatype matches the one returned by + /// [`Component::to_arrow_datatype`]. + /// For the non-fallible version, see [`Component::from_arrow_opt`]. + #[inline] + fn try_from_arrow(data: &dyn ::arrow2::array::Array) -> DeserializationResult> + where + Self: Sized, + { + Self::try_from_arrow_opt(data)? + .into_iter() + .map(|v| { + v.ok_or_else(|| DeserializationError::MissingData { + datatype: data.data_type().clone(), + }) + }) + .collect() + } + + /// Given an Arrow array, deserializes it into a collection of optional [`Component`]s. + /// + /// This will _never_ fail for if the Arrow array's datatype matches the one returned by + /// [`Component::to_arrow_datatype`]. + /// For the fallible version, see [`Component::try_from_arrow_opt`]. + #[inline] + fn from_arrow_opt(data: &dyn ::arrow2::array::Array) -> Vec> + where + Self: Sized, + { + Self::try_from_arrow_opt(data).unwrap() + } + + /// Given an Arrow array, deserializes it into a collection of optional [`Component`]s. + /// + /// This will _never_ fail for if the Arrow array's datatype matches the one returned by + /// [`Component::to_arrow_datatype`]. + /// For the non-fallible version, see [`Component::from_arrow_opt`]. + fn try_from_arrow_opt( + data: &dyn ::arrow2::array::Array, + ) -> DeserializationResult>> + where + Self: Sized; } /// The fully-qualified name of an [`Archetype`], e.g. `rerun.archetypes.Points2D`. @@ -272,6 +394,39 @@ pub trait Archetype { fn try_to_arrow( &self, ) -> SerializationResult)>>; + + // --- + + /// Given an iterator of Arrow arrays and their respective field metadata, deserializes them + /// into this archetype. + /// + /// Panics on failure. + /// For the fallible version, see [`Archetype::try_from_arrow`]. + /// + /// Arrow arrays that are unknown to this [`Archetype`] will simply be ignored and a warning + /// logged to stderr. + #[inline] + fn from_arrow( + data: impl IntoIterator)>, + ) -> Self + where + Self: Sized, + { + Self::try_from_arrow(data).unwrap() + } + + /// Given an iterator of Arrow arrays and their respective field metadata, deserializes them + /// into this archetype. + /// + /// Arrow arrays that are unknown to this [`Archetype`] will simply be ignored and a warning + /// logged to stderr. + /// + /// For the non-fallible version, see [`Archetype::from_arrow`]. + fn try_from_arrow( + data: impl IntoIterator)>, + ) -> DeserializationResult + where + Self: Sized; } // --- @@ -289,6 +444,44 @@ pub enum SerializationError { pub type SerializationResult = ::std::result::Result; +#[derive(thiserror::Error, Debug)] +pub enum DeserializationError { + #[error("Missing data for {datatype:#?}")] + MissingData { + datatype: ::arrow2::datatypes::DataType, + }, + + #[error("Expected {expected:#?} but found {got:#?} instead")] + SchemaMismatch { + expected: ::arrow2::datatypes::DataType, + got: ::arrow2::datatypes::DataType, + }, + + #[error( + "Offsets were ouf of bounds, trying to read from {bounds:?} in an array of size {len}" + )] + OffsetsMismatch { + bounds: (usize, usize), + len: usize, + datatype: ::arrow2::datatypes::DataType, + }, + + #[error("Expected array of length {expected} but found a length of {got:#?} instead")] + ArrayLengthMismatch { + expected: usize, + got: usize, + datatype: ::arrow2::datatypes::DataType, + }, + + #[error("Expected single-instanced component but found {got} instances instead")] + MonoMismatch { + got: usize, + datatype: ::arrow2::datatypes::DataType, + }, +} + +pub type DeserializationResult = ::std::result::Result; + // --- /// Number of decimals shown for all vector display methods. diff --git a/crates/re_types/tests/fuzzy.rs b/crates/re_types/tests/fuzzy.rs index 8c1424dc1186..bc454452513b 100644 --- a/crates/re_types/tests/fuzzy.rs +++ b/crates/re_types/tests/fuzzy.rs @@ -132,5 +132,6 @@ fn roundtrip() { eprintln!("{} = {array:#?}", field.name); } - // TODO(cmc): deserialize + let deserialized = AffixFuzzer1::from_arrow(serialized); + similar_asserts::assert_eq!(arch, deserialized); } diff --git a/crates/re_types/tests/points2d.rs b/crates/re_types/tests/points2d.rs index 1630ed0252b1..ec0a046b7d35 100644 --- a/crates/re_types/tests/points2d.rs +++ b/crates/re_types/tests/points2d.rs @@ -53,5 +53,6 @@ fn roundtrip() { eprintln!("{} = {array:#?}", field.name); } - // TODO(cmc): deserialize + let deserialized = Points2D::from_arrow(serialized); + similar_asserts::assert_eq!(expected, deserialized); } diff --git a/crates/re_types_builder/src/codegen/rust.rs b/crates/re_types_builder/src/codegen/rust.rs index 831853da7832..0146e4fa31a0 100644 --- a/crates/re_types_builder/src/codegen/rust.rs +++ b/crates/re_types_builder/src/codegen/rust.rs @@ -470,7 +470,7 @@ fn quote_meta_clause_from_obj(obj: &Object, attr: &str, clause: &str) -> TokenSt fn quote_trait_impls_from_obj( arrow_registry: &ArrowRegistry, - _objects: &Objects, + objects: &Objects, obj: &Object, ) -> TokenStream { let Object { @@ -506,6 +506,8 @@ fn quote_trait_impls_from_obj( let quoted_serializer = quote_arrow_serializer(arrow_registry, obj, &format_ident!("data")); + let quoted_deserializer = + quote_arrow_deserializer(arrow_registry, objects, obj, &format_ident!("data")); let into_cow = quote! { // NOTE: We need these so end-user code can effortlessly serialize both iterators @@ -555,6 +557,16 @@ fn quote_trait_impls_from_obj( use crate::{Component as _, Datatype as _}; Ok(#quoted_serializer) } + + // NOTE: Don't inline this, this gets _huge_. + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_from_arrow_opt(data: &dyn ::arrow2::array::Array) -> crate::DeserializationResult>> + where + Self: Sized { + use ::arrow2::{datatypes::*, array::*}; + use crate::{Component as _, Datatype as _}; + Ok(#quoted_deserializer) + } } } } @@ -573,6 +585,12 @@ fn quote_trait_impls_from_obj( let num_all = num_required + num_recommended + num_optional; + let quoted_field_names = obj + .fields + .iter() + .map(|field| format_ident!("{}", field.name)) + .collect::>(); + let all_serializers = { obj.fields.iter().map(|obj_field| { let field_name_str = &obj_field.name; @@ -629,6 +647,64 @@ fn quote_trait_impls_from_obj( }) }; + let all_deserializers = { + obj.fields.iter().map(|obj_field| { + let field_name_str = &obj_field.name; + let field_name = format_ident!("{}", obj_field.name); + + let is_plural = obj_field.typ.is_plural(); + let is_nullable = obj_field.is_nullable; + + // NOTE: unwrapping is safe since the field must point to a component. + let component = obj_field.typ.fqname().unwrap(); + let component = format_ident!("{}", component.rsplit_once('.').unwrap().1); + let component = quote!(crate::components::#component); + + let quoted_collection = if is_plural { + quote! { + .into_iter() + .map(|v| v .ok_or_else(|| crate::DeserializationError::MissingData { + // TODO(cmc): gotta improve this + datatype: ::arrow2::datatypes::DataType::Null, + })) + .collect::>>()? + } + } else { + quote! { + .into_iter() + .next() + .flatten() + .ok_or_else(|| crate::DeserializationError::MissingData { + // TODO(cmc): gotta improve this + datatype: ::arrow2::datatypes::DataType::Null, + })? + } + }; + + let quoted_deser = if is_nullable { + quote! { + if let Some(array) = arrays_by_name.get(#field_name_str) { + Some(<#component>::try_from_arrow_opt(&**array)? #quoted_collection) + } else { + None + } + } + } else { + quote! {{ + let array = arrays_by_name + .get(#field_name_str) + .ok_or_else(|| crate::DeserializationError::MissingData { + // TODO(cmc): gotta improve this + datatype: ::arrow2::datatypes::DataType::Null, + })?; + <#component>::try_from_arrow_opt(&**array)? #quoted_collection + }} + }; + + quote!(let #field_name = #quoted_deser;) + }) + }; + quote! { impl #name { pub const REQUIRED_COMPONENTS: [crate::ComponentName; #num_required] = [#required]; @@ -668,6 +744,24 @@ fn quote_trait_impls_from_obj( use crate::Component as _; Ok([ #({ #all_serializers },)* ].into_iter().flatten().collect()) } + + #[inline] + fn try_from_arrow( + data: impl IntoIterator)>, + ) -> crate::DeserializationResult { + use crate::Component as _; + + let arrays_by_name: ::std::collections::HashMap<_, _> = data + .into_iter() + .map(|(field, array)| (field.name, array)) + .collect(); + + #(#all_deserializers;)* + + Ok(Self { + #(#quoted_field_names,)* + }) + } } } } @@ -1182,6 +1276,295 @@ fn quote_arrow_field_serializer( } } +fn quote_arrow_deserializer( + arrow_registry: &ArrowRegistry, + _objects: &Objects, + obj: &Object, + data_src: &proc_macro2::Ident, +) -> TokenStream { + let datatype = &arrow_registry.get(&obj.fqname); + let quoted_datatype = ArrowDataTypeTokenizer(datatype); + + let is_arrow_transparent = obj.datatype.is_none(); + let is_tuple_struct = is_tuple_struct_from_obj(obj); + + if is_arrow_transparent { + // NOTE: Arrow transparent objects must have a single field, no more no less. + // The semantic pass would have failed already if this wasn't the case. + let obj_field = &obj.fields[0]; + + let data_src = data_src.clone(); + let data_dst = format_ident!( + "{}", + if is_tuple_struct { + "data0" + } else { + obj_field.name.as_str() + } + ); + + let quoted_deserializer = quote_arrow_field_deserializer( + &arrow_registry.get(&obj_field.fqname), + obj_field.is_nullable, + &data_src, + ); + + let quoted_unwrap = if obj_field.is_nullable { + quote!(.map(Ok)) + } else { + quote! { + .map(|v| v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: #quoted_datatype, + })) + } + }; + + let quoted_opt_map = if is_tuple_struct { + quote!(.map(|res| res.map(|v| Some(Self(v))))) + } else { + quote!(.map(|res| res.map(|#data_dst| Some(Self { #data_dst })))) + }; + + quote! { + #quoted_deserializer + #quoted_unwrap + #quoted_opt_map + // NOTE: implicit Vec to Result + .collect::>>>()? + } + } else { + let data_src = data_src.clone(); + + // NOTE: This can only be struct or union/enum at this point. + match datatype.to_logical_type() { + DataType::Struct(_) => { + let data_src_fields = format_ident!("{data_src}_fields"); + let data_src_arrays = format_ident!("{data_src}_arrays"); + let data_src_bitmap = format_ident!("{data_src}_bitmap"); + + let quoted_field_deserializers = obj.fields.iter().map(|obj_field| { + let field_name = &obj_field.name; + let data_dst = format_ident!("{}", obj_field.name); + + let quoted_deserializer = quote_arrow_field_deserializer( + &arrow_registry.get(&obj_field.fqname), + obj_field.is_nullable, + &data_src, + ); + + quote! { + let #data_dst = { + let #data_src = &**arrays_by_name[#field_name]; + #quoted_deserializer + } + } + }); + + // NOTE: Collecting because we need it more than once. + let quoted_field_names = obj + .fields + .iter() + .map(|field| format_ident!("{}", field.name)) + .collect::>(); + + let quoted_unwrappings = obj.fields.iter().map(|obj_field| { + let quoted_obj_field_name = format_ident!("{}", obj_field.name); + if obj_field.is_nullable { + quote!(#quoted_obj_field_name) + } else { + quote! { + #quoted_obj_field_name: #quoted_obj_field_name + .ok_or_else(|| crate::DeserializationError::MissingData { + datatype: #quoted_datatype, + })? + } + } + }); + + quote! { { + let #data_src = #data_src + .as_any() + .downcast_ref::<::arrow2::array::StructArray>() + .ok_or_else(|| crate::DeserializationError::SchemaMismatch { + expected: #quoted_datatype, + got: #data_src.data_type().clone(), + })?; + + let (#data_src_fields, #data_src_arrays, #data_src_bitmap) = + (#data_src.fields(), #data_src.values(), #data_src.validity()); + + let is_valid = |i| #data_src_bitmap.map_or(true, |bitmap| bitmap.get_bit(i)); + + let arrays_by_name: ::std::collections::HashMap<_, _> = #data_src_fields + .iter() + .map(|field| field.name.as_str()) + .zip(#data_src_arrays) + .collect(); + + #(#quoted_field_deserializers;)* + + ::itertools::izip!(#(#quoted_field_names),*) + .enumerate() + .map(|(i, (#(#quoted_field_names),*))| is_valid(i).then(|| Ok(Self { + #(#quoted_unwrappings,)* + })).transpose()) + // NOTE: implicit Vec to Result + .collect::>>()? + } } + } + _ => unimplemented!("{datatype:#?}"), // NOLINT + } + } +} + +fn quote_arrow_field_deserializer( + datatype: &DataType, + is_nullable: bool, + data_src: &proc_macro2::Ident, +) -> TokenStream { + _ = is_nullable; // not yet used, will be needed very soon + let quoted_datatype = ArrowDataTypeTokenizer(datatype); + match datatype.to_logical_type() { + DataType::Int8 + | DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::UInt8 + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 + | DataType::Float16 + | DataType::Float32 + | DataType::Float64 => { + let arrow_type = format!("{:?}", datatype.to_logical_type()).replace("DataType::", ""); + let arrow_type = format_ident!("{arrow_type}Array"); + quote! { + #data_src + .as_any() + .downcast_ref::<#arrow_type>() + .unwrap() // safe + .into_iter() + .map(|v| v.copied()) + } + } + + DataType::Utf8 => { + quote! { + #data_src + .as_any() + .downcast_ref::>() + .unwrap() // safe + .into_iter() + .map(|v| v.map(ToOwned::to_owned)) + } + } + + DataType::FixedSizeList(inner, length) => { + let inner_datatype = inner.data_type(); + let quoted_inner_datatype = ArrowDataTypeTokenizer(inner_datatype); + + let quoted_inner = + quote_arrow_field_deserializer(inner_datatype, inner.is_nullable, data_src); + + quote! { { + let #data_src = #data_src + .as_any() + .downcast_ref::<::arrow2::array::ListArray>() + .unwrap(); // safe + + let bitmap = #data_src.validity().cloned(); + let offsets = (0..).step_by(#length).zip((#length..).step_by(#length)); + + let #data_src = &**#data_src.values(); + + let data = #quoted_inner + .map(|v| v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: #quoted_inner_datatype, + })) + // NOTE: implicit Vec to Result + .collect::>>()?; + + offsets + .enumerate() + .map(move |(i, (start, end))| bitmap.as_ref().map_or(true, |bitmap| bitmap.get_bit(i)).then(|| { + data.get(start as usize .. end as usize) + .ok_or_else(|| crate::DeserializationError::OffsetsMismatch { + bounds: (start as usize, end as usize), + len: data.len(), + datatype: #quoted_datatype, + })? + .to_vec() + .try_into() + .map_err(|_err| crate::DeserializationError::ArrayLengthMismatch { + expected: #length, + got: (end - start) as usize, + datatype: #quoted_datatype, + }) + }).transpose() + ) + // NOTE: implicit Vec to Result + .collect::>>>()? + .into_iter() + } } + } + + DataType::List(inner) => { + let inner_datatype = inner.data_type(); + let quoted_inner_datatype = ArrowDataTypeTokenizer(inner_datatype); + + let quoted_inner = + quote_arrow_field_deserializer(inner_datatype, inner.is_nullable, data_src); + + quote! { { + let #data_src = #data_src + .as_any() + .downcast_ref::<::arrow2::array::ListArray>() + .unwrap(); // safe + + let bitmap = #data_src.validity().cloned(); + let offsets = { + let offsets = #data_src.offsets(); + offsets.iter().copied().zip(offsets.iter().copied().skip(1)) + }; + + let #data_src = &**#data_src.values(); + + let data = #quoted_inner + .map(|v| v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: #quoted_inner_datatype, + })) + // NOTE: implicit Vec to Result + .collect::>>()?; + + offsets + .enumerate() + .map(move |(i, (start, end))| bitmap.as_ref().map_or(true, |bitmap| bitmap.get_bit(i)).then(|| { + Ok(data.get(start as usize .. end as usize) + .ok_or_else(|| crate::DeserializationError::OffsetsMismatch { + bounds: (start as usize, end as usize), + len: data.len(), + datatype: #quoted_datatype, + })? + .to_vec() + ) + }).transpose() + ) + // NOTE: implicit Vec to Result + .collect::>>>()? + .into_iter() + } } + } + + DataType::Struct(_) => { + let DataType::Extension(fqname, _, _) = datatype else { unreachable!() }; + let fqname_use = quote_fqname_as_type_path(fqname); + quote!(#fqname_use::try_from_arrow_opt(#data_src)?.into_iter()) + } + + _ => unimplemented!("{datatype:#?}"), // NOLINT + } +} + // --- Helpers --- fn is_tuple_struct_from_obj(obj: &Object) -> bool { diff --git a/scripts/ci/check_large_files_allow_list.txt b/scripts/ci/check_large_files_allow_list.txt index d3449cbe583d..5a7f3fe8db28 100644 --- a/scripts/ci/check_large_files_allow_list.txt +++ b/scripts/ci/check_large_files_allow_list.txt @@ -1,3 +1,4 @@ Cargo.lock +crates/re_types/src/components/fuzzy.rs crates/re_types_builder/src/reflection.rs crates/re_ui/data/Inter-Medium.otf