From 90c0c9d8da26484d4f04c32989f6be2bc037a674 Mon Sep 17 00:00:00 2001 From: Clement Rey Date: Mon, 25 Sep 2023 09:33:01 +0200 Subject: [PATCH 1/3] annihilate re_components --- Cargo.toml | 1 - crates/re_components/Cargo.toml | 82 ----- crates/re_components/README.md | 10 - crates/re_components/src/datagen.rs | 144 -------- crates/re_components/src/lib.rs | 248 -------------- crates/re_components/src/load_file.rs | 159 --------- crates/re_components/src/mat.rs | 146 -------- crates/re_components/src/quaternion.rs | 133 -------- crates/re_components/src/vec.rs | 281 --------------- crates/re_components/tests/arrow_msg.rs | 31 -- crates/re_components/tests/data_row.rs | 71 ---- crates/re_components/tests/data_table.rs | 149 -------- .../re_components/tests/data_table_batcher.rs | 323 ------------------ 13 files changed, 1778 deletions(-) delete mode 100644 crates/re_components/Cargo.toml delete mode 100644 crates/re_components/README.md delete mode 100644 crates/re_components/src/datagen.rs delete mode 100644 crates/re_components/src/lib.rs delete mode 100644 crates/re_components/src/load_file.rs delete mode 100644 crates/re_components/src/mat.rs delete mode 100644 crates/re_components/src/quaternion.rs delete mode 100644 crates/re_components/src/vec.rs delete mode 100644 crates/re_components/tests/arrow_msg.rs delete mode 100644 crates/re_components/tests/data_row.rs delete mode 100644 crates/re_components/tests/data_table.rs delete mode 100644 crates/re_components/tests/data_table_batcher.rs diff --git a/Cargo.toml b/Cargo.toml index a24d5fd5114b..2214eb7ae75e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,6 @@ re_arrow_store = { path = "crates/re_arrow_store", version = "=0.9.0-alpha.4", d re_build_info = { path = "crates/re_build_info", version = "=0.9.0-alpha.4", default-features = false } re_build_tools = { path = "crates/re_build_tools", version = "=0.9.0-alpha.4", default-features = false } re_build_web_viewer = { path = "crates/re_build_web_viewer", version = "=0.9.0-alpha.4", default-features = false } -re_components = { path = "crates/re_components", version = "=0.9.0-alpha.4", default-features = false } re_crash_handler = { path = "crates/re_crash_handler", version = "=0.9.0-alpha.4", default-features = false } re_data_source = { path = "crates/re_data_source", version = "=0.9.0-alpha.4", default-features = false } re_data_store = { path = "crates/re_data_store", version = "=0.9.0-alpha.4", default-features = false } diff --git a/crates/re_components/Cargo.toml b/crates/re_components/Cargo.toml deleted file mode 100644 index 57f9ede39b11..000000000000 --- a/crates/re_components/Cargo.toml +++ /dev/null @@ -1,82 +0,0 @@ -[package] -name = "re_components" -authors.workspace = true -description = "The standard rerun data types, component types, and archetypes" -edition.workspace = true -homepage.workspace = true -include.workspace = true -license.workspace = true -publish = true -readme = "README.md" -repository.workspace = true -rust-version.workspace = true -version.workspace = true - -[package.metadata.docs.rs] -all-features = true - - -[features] -default = [] - -## Enables the `datagen` module, which exposes a number of tools for generating random data for -## tests and benchmarks. -arrow_datagen = ["dep:rand"] - -## Enable conversions -ecolor = ["dep:ecolor"] - -## Add support for some math operations using [`glam`](https://crates.io/crates/glam/). -glam = ["dep:glam"] - -## Integration with the [`image`](https://crates.io/crates/image/) crate, plus JPEG support.. -image = [ - "dep:ecolor", - "dep:image", - "dep:zune-core", - "dep:zune-jpeg", - "re_types/image", -] - -## Enable (de)serialization using serde. -serde = ["dep:serde", "half/serde", "re_log_types/serde"] - - -[dependencies] - -# Rerun -re_log_types.workspace = true -re_log.workspace = true -re_tracing.workspace = true -re_types.workspace = true - -# External -array-init = "2.1.0" -arrow2.workspace = true -arrow2_convert.workspace = true -bytemuck = "1.11" -document-features = "0.2" -half = { workspace = true, features = ["bytemuck"] } -itertools = { workspace = true } -lazy_static.workspace = true -ndarray.workspace = true -thiserror.workspace = true - - -# Optional dependencies: -ecolor = { workspace = true, optional = true } -glam = { workspace = true, optional = true } -image = { workspace = true, optional = true, default-features = false } -rand = { workspace = true, optional = true, features = ["std", "std_rng"] } -serde = { version = "1", optional = true, features = ["derive", "rc"] } -zune-core = { version = "0.2", optional = true } -zune-jpeg = { version = "0.3", optional = true } - -# Native dependencies: -[target.'cfg(not(target_arch = "wasm32"))'.dependencies] -crossbeam.workspace = true - - -[dev-dependencies] -rmp-serde = "1.1" -similar-asserts = "1.4.2" diff --git a/crates/re_components/README.md b/crates/re_components/README.md deleted file mode 100644 index b08b34f7056d..000000000000 --- a/crates/re_components/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# re_log_types - -Part of the [`rerun`](https://github.com/rerun-io/rerun) family of crates. - -[![Latest version](https://img.shields.io/crates/v/re_log_types.svg)](https://crates.io/crates/re_log_types) -[![Documentation](https://docs.rs/re_log_types/badge.svg)](https://docs.rs/re_log_types) -![MIT](https://img.shields.io/badge/license-MIT-blue.svg) -![Apache](https://img.shields.io/badge/license-Apache-blue.svg) - -The standard Rerun data types, component types, and archetypes. diff --git a/crates/re_components/src/datagen.rs b/crates/re_components/src/datagen.rs deleted file mode 100644 index a41015ce922d..000000000000 --- a/crates/re_components/src/datagen.rs +++ /dev/null @@ -1,144 +0,0 @@ -//! Generate random data for tests and benchmarks. - -// TODO(#1810): It really is time for whole module to disappear. - -use re_log_types::{Time, TimeInt, TimeType, Timeline}; -use re_types::components::InstanceKey; - -/// Create `len` dummy colors -pub fn build_some_colors(len: usize) -> Vec { - (0..len) - .map(|i| re_types::components::Color::from(i as u32)) - .collect() -} - -/// Create `len` dummy `Position2D` -pub fn build_some_positions2d(len: usize) -> Vec { - use rand::Rng as _; - let mut rng = rand::thread_rng(); - - (0..len) - .map(|_| { - re_types::components::Position2D::new( - rng.gen_range(0.0..10.0), - rng.gen_range(0.0..10.0), - ) - }) - .collect() -} - -/// Create `len` dummy `Vec3D` -pub fn build_some_vec3d(len: usize) -> Vec { - use rand::Rng as _; - let mut rng = rand::thread_rng(); - - (0..len) - .map(|_| { - crate::LegacyVec3D::new( - rng.gen_range(0.0..10.0), - rng.gen_range(0.0..10.0), - rng.gen_range(0.0..10.0), - ) - }) - .collect() -} - -/// Build a ([`Timeline`], [`TimeInt`]) tuple from `log_time` suitable for inserting in a [`re_log_types::TimePoint`]. -pub fn build_log_time(log_time: Time) -> (Timeline, TimeInt) { - (Timeline::log_time(), log_time.into()) -} - -/// Build a ([`Timeline`], [`TimeInt`]) tuple from `frame_nr` suitable for inserting in a [`re_log_types::TimePoint`]. -pub fn build_frame_nr(frame_nr: TimeInt) -> (Timeline, TimeInt) { - (Timeline::new("frame_nr", TimeType::Sequence), frame_nr) -} - -/// Create `len` dummy `InstanceKey` keys. These keys will be sorted. -pub fn build_some_instances(num_instances: usize) -> Vec { - use rand::seq::SliceRandom; - let mut rng = rand::thread_rng(); - - // Allocate pool of 10x the potential instance keys, draw a random sampling, and then sort it - let mut instance_pool = (0..(num_instances * 10)).collect::>(); - let (rand_instances, _) = instance_pool.partial_shuffle(&mut rng, num_instances); - let mut sorted_instances = rand_instances.to_vec(); - sorted_instances.sort(); - - sorted_instances - .into_iter() - .map(|id| InstanceKey(id as u64)) - .collect() -} - -pub fn build_some_instances_from(instances: impl IntoIterator) -> Vec { - let mut instances = instances.into_iter().map(InstanceKey).collect::>(); - instances.sort(); - instances -} - -/// Crafts a simple but interesting [`re_log_types::DataTable`]. -#[cfg(not(target_arch = "wasm32"))] -pub fn data_table_example(timeless: bool) -> re_log_types::DataTable { - use re_log_types::{DataRow, DataTable, RowId, TableId, TimePoint}; - use re_types::components::{Color, Position2D, Text}; - - let table_id = TableId::random(); - - let mut tick = 0i64; - let mut timepoint = |frame_nr: i64| { - let tp = if timeless { - TimePoint::timeless() - } else { - TimePoint::from([ - (Timeline::log_time(), Time::now().into()), - (Timeline::log_tick(), tick.into()), - (Timeline::new_sequence("frame_nr"), frame_nr.into()), - ]) - }; - tick += 1; - tp - }; - - let row0 = { - let num_instances = 2; - let positions: &[Position2D] = &[[10.0, 10.0].into(), [20.0, 20.0].into()]; - let colors: &[_] = &[Color::from_rgb(128, 128, 128)]; - let labels: &[Text] = &[]; - - DataRow::from_cells3( - RowId::random(), - "a", - timepoint(1), - num_instances, - (positions, colors, labels), - ) - .unwrap() - }; - - let row1 = { - let num_instances = 0; - let colors: &[Color] = &[]; - - DataRow::from_cells1(RowId::random(), "b", timepoint(1), num_instances, colors).unwrap() - }; - - let row2 = { - let num_instances = 1; - let colors: &[_] = &[Color::from_rgb(255, 255, 255)]; - let labels: &[_] = &[Text("hey".into())]; - - DataRow::from_cells2( - RowId::random(), - "c", - timepoint(2), - num_instances, - (colors, labels), - ) - .unwrap() - }; - - let mut table = DataTable::from_rows(table_id, [row0, row1, row2]); - table.compute_all_size_bytes(); - - table -} diff --git a/crates/re_components/src/lib.rs b/crates/re_components/src/lib.rs deleted file mode 100644 index 0abdf2398312..000000000000 --- a/crates/re_components/src/lib.rs +++ /dev/null @@ -1,248 +0,0 @@ -//! User-facing data types, component types, and archetypes. -//! -//! The SDK is responsible for submitting component columns that conforms to these schemas. The -//! schemas are additionally documented in doctests. -//! -//! ## Feature flags -#![doc = document_features::document_features!()] -//! - -use arrow2::{ - array::{FixedSizeListArray, MutableFixedSizeListArray, PrimitiveArray}, - datatypes::{DataType, Field}, -}; -use arrow2_convert::{ - deserialize::{ArrowArray, ArrowDeserialize}, - field::{ArrowEnableVecForType, ArrowField}, - serialize::ArrowSerialize, -}; -use lazy_static::lazy_static; - -mod mat; -mod quaternion; -mod vec; - -mod load_file; - -#[cfg(feature = "arrow_datagen")] -pub mod datagen; - -// ---------------------------------------------------------------------------- - -// TODO(cmc): get rid of this once every single archetypes depending on those have been migrated. -pub use vec::{LegacyVec2D, LegacyVec3D, LegacyVec4D}; - -pub use self::{mat::LegacyMat3x3, quaternion::Quaternion}; - -#[cfg(not(target_arch = "wasm32"))] -pub use self::load_file::data_cells_from_file_path; - -pub use self::load_file::{data_cells_from_file_contents, FromFileError}; - -// This is very convenient to re-export -pub use re_log_types::LegacyComponent; - -pub mod external { - #[cfg(feature = "glam")] - pub use glam; - - #[cfg(feature = "image")] - pub use image; -} - -// ---------------------------------------------------------------------------- - -lazy_static! { - //TODO(john): use a run-time type registry - static ref FIELDS: [Field; 2] = [ - ::field(), - ::field(), - ]; -} - -/// Iterate over the registered field types -pub fn iter_registered_field_types() -> impl Iterator { - FIELDS.iter() -} - -#[derive(thiserror::Error, Debug)] -pub enum FieldError { - #[error("Encountered bad value")] - BadValue, - - #[error("Slice over bad indices")] - BadSlice(#[from] std::array::TryFromSliceError), -} - -pub type Result = std::result::Result; - -/// `arrow2_convert` helper for fields of type `[T; SIZE]` -/// -/// This allows us to use fields of type `[T; SIZE]` in `arrow2_convert`. Since this is a helper, -/// it must be specified as the type of the field using the `#[arrow_field(type = "FixedSizeArrayField")]` attribute. -/// -/// ## Example: -/// ``` -/// use arrow2_convert::{ArrowField, ArrowSerialize, ArrowDeserialize}; -/// use re_components::FixedSizeArrayField; -/// -/// #[derive(ArrowField, ArrowSerialize, ArrowDeserialize)] -/// pub struct ConvertibleType { -/// #[arrow_field(type = "FixedSizeArrayField")] -/// data: [u32; 2], -/// } -/// ``` -pub struct FixedSizeArrayField(std::marker::PhantomData); - -impl ArrowField for FixedSizeArrayField -where - T: ArrowField + ArrowEnableVecForType, -{ - type Type = [T; SIZE]; - - #[inline] - fn data_type() -> DataType { - arrow2::datatypes::DataType::FixedSizeList(Box::new(::field("item")), SIZE) - } -} - -impl ArrowSerialize for FixedSizeArrayField -where - T: ArrowSerialize + ArrowEnableVecForType + ArrowField + 'static, - ::MutableArrayType: Default, -{ - type MutableArrayType = MutableFixedSizeListArray<::MutableArrayType>; - #[inline] - fn new_array() -> Self::MutableArrayType { - Self::MutableArrayType::new_with_field( - ::new_array(), - "item", - ::is_nullable(), - SIZE, - ) - } - - fn arrow_serialize( - v: &::Type, - array: &mut Self::MutableArrayType, - ) -> arrow2::error::Result<()> { - let values = array.mut_values(); - for i in v { - ::arrow_serialize(i, values)?; - } - array.try_push_valid() - } -} - -pub struct FastFixedSizeArrayIter<'a, T, const SIZE: usize> -where - T: arrow2::types::NativeType, -{ - offset: usize, - end: usize, - array: &'a FixedSizeListArray, - values: &'a PrimitiveArray, -} - -impl<'a, T, const SIZE: usize> Iterator for FastFixedSizeArrayIter<'a, T, SIZE> -where - T: arrow2::types::NativeType, -{ - type Item = Option<[T; SIZE]>; - - fn next(&mut self) -> Option { - if self.offset < self.end { - if let Some(validity) = self.array.validity() { - if !validity.get_bit(self.offset) { - self.offset += 1; - return Some(None); - } - } - - let out: [T; SIZE] = - array_init::array_init(|i: usize| self.values.value(self.offset * SIZE + i)); - self.offset += 1; - Some(Some(out)) - } else { - None - } - } -} - -pub struct FastFixedSizeListArray(std::marker::PhantomData); - -#[cfg(not(target_os = "windows"))] -extern "C" { - fn do_not_call_into_iter(); // we never define this function, so the linker will fail -} - -impl<'a, T, const SIZE: usize> IntoIterator for &'a FastFixedSizeListArray -where - T: arrow2::types::NativeType, -{ - type Item = Option<[T; SIZE]>; - - type IntoIter = FastFixedSizeArrayIter<'a, T, SIZE>; - - #[cfg(not(target_os = "windows"))] - fn into_iter(self) -> Self::IntoIter { - #[allow(unsafe_code)] - // SAFETY: - // This exists so we get a link-error if some code tries to call into_iter - // Iteration should only happen via iter_from_array_ref. - // This is a quirk of the way the traits work in arrow2_convert. - unsafe { - do_not_call_into_iter(); - } - unreachable!(); - } - - // On windows the above linker trick doesn't work. - // We'll still catch the issue on build in Linux, but on windows just fall back to panic. - #[cfg(target_os = "windows")] - fn into_iter(self) -> Self::IntoIter { - panic!("Use iter_from_array_ref. This is a quirk of the way the traits work in arrow2_convert."); - } -} - -impl ArrowArray for FastFixedSizeListArray -where - T: arrow2::types::NativeType, -{ - type BaseArrayType = FixedSizeListArray; - - fn iter_from_array_ref(b: &dyn arrow2::array::Array) -> <&Self as IntoIterator>::IntoIter { - let array = b.as_any().downcast_ref::().unwrap(); - let values = array - .values() - .as_any() - .downcast_ref::>() - .unwrap(); - FastFixedSizeArrayIter:: { - offset: 0, - end: array.len(), - array, - values, - } - } -} - -impl ArrowDeserialize for FixedSizeArrayField -where - T: arrow2::types::NativeType - + ArrowDeserialize - + ArrowEnableVecForType - + ArrowField - + 'static, - ::ArrayType: 'static, - for<'b> &'b ::ArrayType: IntoIterator, -{ - type ArrayType = FastFixedSizeListArray; - - #[inline] - fn arrow_deserialize( - v: <&Self::ArrayType as IntoIterator>::Item, - ) -> Option<::Type> { - v - } -} diff --git a/crates/re_components/src/load_file.rs b/crates/re_components/src/load_file.rs deleted file mode 100644 index a71f1d60e532..000000000000 --- a/crates/re_components/src/load_file.rs +++ /dev/null @@ -1,159 +0,0 @@ -use re_log_types::DataCell; -use re_types::external::anyhow; - -/// Errors from [`data_cells_from_file_path`]. -#[derive(thiserror::Error, Debug)] -pub enum FromFileError { - #[cfg(not(target_arch = "wasm32"))] - #[error(transparent)] - FileRead(#[from] std::io::Error), - - #[error(transparent)] - DataCellError(#[from] re_log_types::DataCellError), - - #[cfg(feature = "image")] - #[error(transparent)] - TensorImageLoad(#[from] re_types::tensor_data::TensorImageLoadError), - - #[error("Unsupported file extension '{extension}' for file {path:?}. To load image files, make sure you compile with the 'image' feature")] - UnknownExtension { - extension: String, - path: std::path::PathBuf, - }, - - #[error(transparent)] - Other(#[from] anyhow::Error), -} - -/// Read the file at the given path. -/// -/// Supported file extensions are: -/// * `glb`, `gltf`, `obj`: encoded meshes, leaving it to the viewer to decode -/// * `jpg`, `jpeg`: encoded JPEG, leaving it to the viewer to decode. Requires the `image` feature. -/// * `png` and other image formats: decoded here. Requires the `image` feature. -/// -/// All other extensions will return an error. -#[cfg(not(target_arch = "wasm32"))] -pub fn data_cells_from_file_path( - file_path: &std::path::Path, -) -> Result, FromFileError> { - let extension = file_path - .extension() - .unwrap_or_default() - .to_ascii_lowercase() - .to_string_lossy() - .to_string(); - - match extension.as_str() { - "glb" | "gltf" | "obj" => { - use re_types::{archetypes::Asset3D, Archetype}; - let cells: Result, _> = Asset3D::from_file(file_path)? - // TODO(#3414): this should be a method of `Archetype` - .as_component_batches() - .into_iter() - .map(|comp_batch| { - let comp_batch = comp_batch.as_ref(); - Ok(DataCell::from_arrow( - comp_batch.name(), - comp_batch - .try_to_arrow() - .map_err(|err| anyhow::anyhow!("serialization failed: {err}"))?, - )) - }) - .collect(); - cells - } - - #[cfg(feature = "image")] - _ => { - use re_types::Archetype; - let indicator = ::Indicator::batch(1); - let indicator_cell = DataCell::from_arrow( - re_types::archetypes::Image::indicator_component(), - indicator.to_arrow(), - ); - - // Assume an image (there are so many image extensions): - let tensor = re_types::components::TensorData( - re_types::datatypes::TensorData::from_image_file(file_path)?, - ); - Ok(vec![ - indicator_cell, - DataCell::try_from_native(std::iter::once(&tensor))?, - ]) - } - - #[cfg(not(feature = "image"))] - _ => Err(FromFileError::UnknownExtension { - extension, - path: file_path.to_owned(), - }), - } -} - -pub fn data_cells_from_file_contents( - file_name: &str, - bytes: Vec, -) -> Result, FromFileError> { - re_tracing::profile_function!(file_name); - - let extension = std::path::Path::new(file_name) - .extension() - .unwrap_or_default() - .to_ascii_lowercase() - .to_string_lossy() - .to_string(); - - match extension.as_str() { - "glb" | "gltf" | "obj" => { - use re_types::{archetypes::Asset3D, components::MediaType, Archetype}; - let cells: Result, _> = - Asset3D::from_bytes(bytes, MediaType::guess_from_path(file_name)) - .as_component_batches() - .into_iter() - .map(|comp_batch| { - let comp_batch = comp_batch.as_ref(); - Ok(DataCell::from_arrow( - comp_batch.name(), - comp_batch - .try_to_arrow() - .map_err(|err| anyhow::anyhow!("serialization failed: {err}"))?, - )) - }) - .collect(); - cells - } - - #[cfg(feature = "image")] - _ => { - let format = if let Some(format) = image::ImageFormat::from_extension(extension) { - format - } else { - image::guess_format(&bytes) - .map_err(re_types::tensor_data::TensorImageLoadError::from)? - }; - - use re_types::Archetype; - let indicator = ::Indicator::batch(1); - let indicator_cell = DataCell::from_arrow( - re_types::archetypes::Image::indicator_component(), - indicator.to_arrow(), - ); - - // Assume an image (there are so many image extensions): - let tensor = re_types::components::TensorData( - re_types::datatypes::TensorData::from_image_bytes(bytes, format)?, - ); - Ok(vec![ - indicator_cell, - DataCell::try_from_native(std::iter::once(&tensor))?, - ]) - } - - #[cfg(not(feature = "image"))] - _ => Err(FromFileError::UnknownExtension { - extension, - path: file_name.to_owned().into(), - }), - } -} diff --git a/crates/re_components/src/mat.rs b/crates/re_components/src/mat.rs deleted file mode 100644 index bdb2eb966a55..000000000000 --- a/crates/re_components/src/mat.rs +++ /dev/null @@ -1,146 +0,0 @@ -use arrow2::{array::PrimitiveArray, datatypes::DataType}; -use arrow2_convert::{ - arrow_enable_vec_for_type, - deserialize::ArrowDeserialize, - field::{ArrowField, FixedSizeVec}, - serialize::ArrowSerialize, -}; - -use super::LegacyVec3D; - -/// A 3x3 column-major Matrix made up of 3 Vecs -/// -/// ``` -/// use re_components::LegacyMat3x3; -/// use arrow2_convert::field::ArrowField; -/// use arrow2::datatypes::{DataType, Field}; -/// -/// assert_eq!( -/// LegacyMat3x3::data_type(), -/// DataType::FixedSizeList( -/// Box::new(Field::new("item", DataType::Float32, false)), -/// 9 -/// ) -/// ); -/// ``` -#[derive(Copy, Clone, Debug, PartialEq)] -#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] -pub struct LegacyMat3x3([LegacyVec3D; 3]); - -impl LegacyMat3x3 { - pub const IDENTITY: LegacyMat3x3 = LegacyMat3x3([ - LegacyVec3D([1.0, 0.0, 0.0]), - LegacyVec3D([0.0, 1.0, 0.0]), - LegacyVec3D([0.0, 0.0, 1.0]), - ]); -} - -impl From for LegacyMat3x3 { - fn from(m: re_types::datatypes::Mat3x3) -> Self { - Self([ - LegacyVec3D([m[0], m[1], m[2]]), - LegacyVec3D([m[3], m[4], m[5]]), - LegacyVec3D([m[6], m[7], m[8]]), - ]) - } -} - -impl std::ops::Index for LegacyMat3x3 -where - Idx: std::slice::SliceIndex<[LegacyVec3D]>, -{ - type Output = Idx::Output; - - #[inline] - fn index(&self, index: Idx) -> &Self::Output { - &self.0[index] - } -} - -impl From<[[f32; 3]; 3]> for LegacyMat3x3 { - #[inline] - fn from(v: [[f32; 3]; 3]) -> Self { - Self([LegacyVec3D(v[0]), LegacyVec3D(v[1]), LegacyVec3D(v[2])]) - } -} - -#[cfg(feature = "glam")] -impl From for glam::Mat3 { - #[inline] - fn from(v: LegacyMat3x3) -> Self { - Self::from_cols(v[0].into(), v[1].into(), v[2].into()) - } -} - -#[cfg(feature = "glam")] -impl From for LegacyMat3x3 { - #[inline] - fn from(v: glam::Mat3) -> Self { - Self::from(v.to_cols_array_2d()) - } -} - -arrow_enable_vec_for_type!(LegacyMat3x3); - -impl ArrowField for LegacyMat3x3 { - type Type = Self; - - #[inline] - fn data_type() -> DataType { - as ArrowField>::data_type() - } -} - -impl ArrowSerialize for LegacyMat3x3 { - type MutableArrayType = as ArrowSerialize>::MutableArrayType; - - #[inline] - fn new_array() -> Self::MutableArrayType { - FixedSizeVec::::new_array() - } - - #[inline] - fn arrow_serialize(v: &Self, array: &mut Self::MutableArrayType) -> arrow2::error::Result<()> { - for col in v.0 { - array.mut_values().extend_from_slice(&col.0); - } - array.try_push_valid() - } -} - -impl ArrowDeserialize for LegacyMat3x3 { - type ArrayType = as ArrowDeserialize>::ArrayType; - - #[inline] - fn arrow_deserialize( - v: <&Self::ArrayType as IntoIterator>::Item, - ) -> Option<::Type> { - v.map(|v| { - let slice = v - .as_any() - .downcast_ref::>() - .unwrap() - .values() - .as_slice(); - LegacyMat3x3([ - LegacyVec3D(slice[0..3].try_into().unwrap()), - LegacyVec3D(slice[3..6].try_into().unwrap()), - LegacyVec3D(slice[6..9].try_into().unwrap()), - ]) - }) - } -} - -#[test] -fn test_mat3x3_roundtrip() { - use arrow2::array::Array; - use arrow2_convert::{deserialize::TryIntoCollection, serialize::TryIntoArrow}; - - let mats_in: Vec = vec![ - [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]].into(), - [[11.0, 12.0, 13.0], [14.0, 15.0, 16.0], [17.0, 18.0, 19.0]].into(), - ]; - let array: Box = mats_in.try_into_arrow().unwrap(); - let mats_out: Vec = TryIntoCollection::try_into_collection(array).unwrap(); - assert_eq!(mats_in, mats_out); -} diff --git a/crates/re_components/src/quaternion.rs b/crates/re_components/src/quaternion.rs deleted file mode 100644 index 855087a7f8f9..000000000000 --- a/crates/re_components/src/quaternion.rs +++ /dev/null @@ -1,133 +0,0 @@ -use arrow2::{array::PrimitiveArray, datatypes::DataType}; -use arrow2_convert::{ - arrow_enable_vec_for_type, - deserialize::ArrowDeserialize, - field::{ArrowField, FixedSizeVec}, - serialize::ArrowSerialize, -}; - -/// A Quaternion represented by 4 real numbers. -/// -/// ``` -/// use re_components::Quaternion; -/// use arrow2_convert::field::ArrowField; -/// use arrow2::datatypes::{DataType, Field}; -/// -/// assert_eq!( -/// Quaternion::data_type(), -/// DataType::FixedSizeList( -/// Box::new(Field::new("item", DataType::Float32, false)), -/// 4 -/// ) -/// ); -/// ``` -#[derive(Copy, Clone, Debug, PartialEq)] -#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] -pub struct Quaternion { - pub x: f32, - pub y: f32, - pub z: f32, - pub w: f32, -} - -impl Default for Quaternion { - #[inline] - fn default() -> Self { - Self::IDENTITY - } -} - -impl Quaternion { - pub const IDENTITY: Self = Self { - x: 0.0, - y: 0.0, - z: 0.0, - w: 1.0, - }; - - #[inline] - pub fn new(x: f32, y: f32, z: f32, w: f32) -> Self { - Self { x, y, z, w } - } - - #[inline] - pub fn from_xyzw([x, y, z, w]: [f32; 4]) -> Self { - Self { x, y, z, w } - } -} - -impl re_log_types::LegacyComponent for Quaternion { - #[inline] - fn legacy_name() -> re_log_types::ComponentName { - "rerun.quaternion".into() - } -} - -#[cfg(feature = "glam")] -impl From for glam::Quat { - #[inline] - fn from(q: Quaternion) -> Self { - Self::from_xyzw(q.x, q.y, q.z, q.w) - } -} - -#[cfg(feature = "glam")] -impl From for Quaternion { - #[inline] - fn from(q: glam::Quat) -> Self { - let (x, y, z, w) = q.into(); - Self { x, y, z, w } - } -} - -arrow_enable_vec_for_type!(Quaternion); - -impl ArrowField for Quaternion { - type Type = Self; - - #[inline] - fn data_type() -> DataType { - as ArrowField>::data_type() - } -} - -impl ArrowSerialize for Quaternion { - type MutableArrayType = as ArrowSerialize>::MutableArrayType; - - #[inline] - fn new_array() -> Self::MutableArrayType { - FixedSizeVec::::new_array() - } - - #[inline] - fn arrow_serialize(v: &Self, array: &mut Self::MutableArrayType) -> arrow2::error::Result<()> { - array.mut_values().extend_from_slice(&[v.x, v.y, v.z, v.w]); - array.try_push_valid() - } -} - -impl ArrowDeserialize for Quaternion { - type ArrayType = as ArrowDeserialize>::ArrayType; - - #[inline] - fn arrow_deserialize( - v: <&Self::ArrayType as IntoIterator>::Item, - ) -> Option<::Type> { - v.map(|v| { - let v = v - .as_any() - .downcast_ref::>() - .unwrap() - .values() - .as_slice(); - Quaternion { - x: v[0], - y: v[1], - z: v[2], - w: v[3], - } - }) - } -} - -re_log_types::component_legacy_shim!(Quaternion); diff --git a/crates/re_components/src/vec.rs b/crates/re_components/src/vec.rs deleted file mode 100644 index ab4e59ccb4be..000000000000 --- a/crates/re_components/src/vec.rs +++ /dev/null @@ -1,281 +0,0 @@ -use arrow2_convert::{ArrowDeserialize, ArrowField, ArrowSerialize}; - -use super::FixedSizeArrayField; - -/// Number of decimals shown for all vector display methods. -const DISPLAY_PRECISION: usize = 3; - -// --- Vec2D --- - -/// A vector in 2D space. -/// -/// ``` -/// # use re_components::LegacyVec2D; -/// # use arrow2_convert::field::ArrowField; -/// # use arrow2::datatypes::{DataType, Field}; -/// assert_eq!( -/// LegacyVec2D::data_type(), -/// DataType::FixedSizeList( -/// Box::new(Field::new("item", DataType::Float32, false)), -/// 2 -/// ) -/// ); -/// ``` -#[derive(Copy, Clone, Debug, Default, PartialEq, ArrowField, ArrowSerialize, ArrowDeserialize)] -#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] -#[arrow_field(transparent)] -pub struct LegacyVec2D(#[arrow_field(type = "FixedSizeArrayField")] pub [f32; 2]); - -impl LegacyVec2D { - #[inline] - pub fn x(&self) -> f32 { - self.0[0] - } - - #[inline] - pub fn y(&self) -> f32 { - self.0[1] - } -} - -impl From<[f32; 2]> for LegacyVec2D { - #[inline] - fn from(v: [f32; 2]) -> Self { - Self(v) - } -} - -impl std::ops::Index for LegacyVec2D -where - Idx: std::slice::SliceIndex<[f32]>, -{ - type Output = Idx::Output; - - #[inline] - fn index(&self, index: Idx) -> &Self::Output { - &self.0[index] - } -} - -impl re_log_types::LegacyComponent for LegacyVec2D { - fn legacy_name() -> re_log_types::ComponentName { - "rerun.vec2d".into() - } -} - -#[cfg(feature = "glam")] -impl From for glam::Vec2 { - fn from(v: LegacyVec2D) -> Self { - Self::from_slice(&v.0) - } -} - -#[cfg(feature = "glam")] -impl From for LegacyVec2D { - fn from(v: glam::Vec2) -> Self { - Self(v.to_array()) - } -} - -impl From for LegacyVec2D { - fn from(value: re_types::datatypes::Vec2D) -> Self { - Self(value.0) - } -} - -impl std::fmt::Display for LegacyVec2D { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "[{:.prec$}, {:.prec$}]", - self.x(), - self.y(), - prec = DISPLAY_PRECISION, - ) - } -} - -re_log_types::component_legacy_shim!(LegacyVec2D); - -// --- Vec3D --- - -/// A vector in 3D space. -/// -/// ``` -/// use re_components::LegacyVec3D; -/// use arrow2_convert::field::ArrowField; -/// use arrow2::datatypes::{DataType, Field}; -/// -/// assert_eq!( -/// LegacyVec3D::data_type(), -/// DataType::FixedSizeList( -/// Box::new(Field::new("item", DataType::Float32, false)), -/// 3 -/// ) -/// ); -/// ``` -#[derive(Copy, Clone, Debug, Default, PartialEq, ArrowField, ArrowSerialize, ArrowDeserialize)] -#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] -#[arrow_field(transparent)] -pub struct LegacyVec3D(#[arrow_field(type = "FixedSizeArrayField")] pub [f32; 3]); - -impl LegacyVec3D { - pub const ZERO: LegacyVec3D = LegacyVec3D([0.0; 3]); - - #[inline] - pub fn new(x: f32, y: f32, z: f32) -> Self { - Self::from([x, y, z]) - } - - #[inline] - pub fn x(&self) -> f32 { - self.0[0] - } - - #[inline] - pub fn y(&self) -> f32 { - self.0[1] - } - - #[inline] - pub fn z(&self) -> f32 { - self.0[2] - } -} - -impl From<[f32; 3]> for LegacyVec3D { - #[inline] - fn from(v: [f32; 3]) -> Self { - Self(v) - } -} - -impl std::ops::Index for LegacyVec3D -where - Idx: std::slice::SliceIndex<[f32]>, -{ - type Output = Idx::Output; - - #[inline] - fn index(&self, index: Idx) -> &Self::Output { - &self.0[index] - } -} - -impl re_log_types::LegacyComponent for LegacyVec3D { - fn legacy_name() -> re_log_types::ComponentName { - "rerun.vec3d".into() - } -} - -#[cfg(feature = "glam")] -impl From for glam::Vec3 { - #[inline] - fn from(v: LegacyVec3D) -> Self { - Self::from_slice(&v.0) - } -} - -#[cfg(feature = "glam")] -impl From for LegacyVec3D { - #[inline] - fn from(v: glam::Vec3) -> Self { - Self(v.to_array()) - } -} - -impl From for LegacyVec3D { - fn from(value: re_types::datatypes::Vec3D) -> Self { - Self(value.0) - } -} - -impl std::fmt::Display for LegacyVec3D { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "[{:.prec$}, {:.prec$}, {:.prec$}]", - self.x(), - self.y(), - self.z(), - prec = DISPLAY_PRECISION, - ) - } -} - -re_log_types::component_legacy_shim!(LegacyVec3D); - -// --- Vec4D --- - -/// A vector in 4D space. -/// -/// ``` -/// # use re_components::LegacyVec4D; -/// # use arrow2_convert::field::ArrowField; -/// # use arrow2::datatypes::{DataType, Field}; -/// assert_eq!( -/// LegacyVec4D::data_type(), -/// DataType::FixedSizeList( -/// Box::new(Field::new("item", DataType::Float32, false)), -/// 4 -/// ) -/// ); -/// ``` -#[derive(Copy, Clone, Debug, Default, PartialEq, ArrowField, ArrowSerialize, ArrowDeserialize)] -#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] -#[arrow_field(transparent)] -pub struct LegacyVec4D(#[arrow_field(type = "FixedSizeArrayField")] pub [f32; 4]); - -impl LegacyVec4D { - #[inline] - pub fn x(&self) -> f32 { - self.0[0] - } - - #[inline] - pub fn y(&self) -> f32 { - self.0[1] - } - - #[inline] - pub fn z(&self) -> f32 { - self.0[2] - } - - #[inline] - pub fn w(&self) -> f32 { - self.0[3] - } -} - -impl std::fmt::Display for LegacyVec4D { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "[{:.prec$}, {:.prec$}, {:.prec$}, {:.prec$}]", - self.x(), - self.y(), - self.z(), - self.w(), - prec = DISPLAY_PRECISION - ) - } -} - -impl From for LegacyVec4D { - fn from(value: re_types::datatypes::Vec4D) -> Self { - Self(value.0) - } -} - -#[test] -fn test_vec4d() { - use arrow2_convert::{deserialize::TryIntoCollection, serialize::TryIntoArrow}; - let data = [ - LegacyVec4D([0.0, 1.0, 2.0, 3.0]), - LegacyVec4D([0.1, 1.1, 2.1, 3.1]), - ]; - let array: Box = data.try_into_arrow().unwrap(); - let ret: Vec = array.try_into_collection().unwrap(); - assert_eq!(&data, ret.as_slice()); -} diff --git a/crates/re_components/tests/arrow_msg.rs b/crates/re_components/tests/arrow_msg.rs deleted file mode 100644 index 47b0b1e28c12..000000000000 --- a/crates/re_components/tests/arrow_msg.rs +++ /dev/null @@ -1,31 +0,0 @@ -use re_components::datagen::{build_frame_nr, build_some_colors, build_some_positions2d}; -use re_log_types::{ArrowMsg, DataRow, DataTable, RowId}; - -#[test] -fn arrow_msg_roundtrip() { - let row = DataRow::from_cells2( - RowId::random(), - "world/rects", - [build_frame_nr(0.into())], - 1, - (build_some_positions2d(1), build_some_colors(1)), - ) - .unwrap(); - - let table_in = { - let mut table = row.into_table(); - table.compute_all_size_bytes(); - table - }; - let msg_in = table_in.to_arrow_msg().unwrap(); - let buf = rmp_serde::to_vec(&msg_in).unwrap(); - let msg_out: ArrowMsg = rmp_serde::from_slice(&buf).unwrap(); - let table_out = { - let mut table = DataTable::from_arrow_msg(&msg_out).unwrap(); - table.compute_all_size_bytes(); - table - }; - - assert_eq!(msg_in, msg_out); - assert_eq!(table_in, table_out); -} diff --git a/crates/re_components/tests/data_row.rs b/crates/re_components/tests/data_row.rs deleted file mode 100644 index 8738b9b2d75e..000000000000 --- a/crates/re_components/tests/data_row.rs +++ /dev/null @@ -1,71 +0,0 @@ -use re_log_types::{DataReadError, DataRow, DataRowError, EntityPath, RowId, TimePoint}; -use re_types::{ - components::{Color, Position2D, Text}, - Loggable as _, -}; - -#[test] -fn data_row_error_num_instances() { - let row_id = RowId::ZERO; - let timepoint = TimePoint::timeless(); - - let num_instances = 2; - let positions: &[Position2D] = &[[10.0, 10.0].into(), [20.0, 20.0].into()]; - let colors: &[_] = &[Color::from_rgb(128, 128, 128)]; - let labels: &[Text] = &[]; - - // 0 = clear: legal - DataRow::from_cells1(row_id, "a/b/c", timepoint.clone(), num_instances, labels).unwrap(); - - // 1 = splat: legal - DataRow::from_cells1(row_id, "a/b/c", timepoint.clone(), num_instances, colors).unwrap(); - - // num_instances = standard: legal - DataRow::from_cells1(row_id, "a/b/c", timepoint.clone(), num_instances, positions).unwrap(); - - // anything else is illegal - let positions: &[Position2D] = &[ - [10.0, 10.0].into(), - [20.0, 20.0].into(), - [30.0, 30.0].into(), - ]; - let err = - DataRow::from_cells1(row_id, "a/b/c", timepoint, num_instances, positions).unwrap_err(); - - match err { - DataRowError::DataRead(DataReadError::WrongNumberOfInstances { - entity_path, - component, - expected_num_instances, - num_instances, - }) => { - assert_eq!(EntityPath::from("a/b/c"), entity_path); - assert_eq!(Position2D::name(), component); - assert_eq!(2, expected_num_instances); - assert_eq!(3, num_instances); - } - _ => unreachable!(), - } -} - -#[test] -fn data_row_error_duped_components() { - let row_id = RowId::ZERO; - let timepoint = TimePoint::timeless(); - - let positions: &[Position2D] = &[[10.0, 10.0].into(), [20.0, 20.0].into()]; - - let err = - DataRow::from_cells2(row_id, "a/b/c", timepoint, 2, (positions, positions)).unwrap_err(); - - match err { - DataRowError::DataRead(DataReadError::DupedComponent { - entity_path, - component, - }) => { - assert_eq!(EntityPath::from("a/b/c"), entity_path); - assert_eq!(Position2D::name(), component); - } - _ => unreachable!(), - } -} diff --git a/crates/re_components/tests/data_table.rs b/crates/re_components/tests/data_table.rs deleted file mode 100644 index 19968fdcb929..000000000000 --- a/crates/re_components/tests/data_table.rs +++ /dev/null @@ -1,149 +0,0 @@ -use re_log_types::{DataCell, DataRow, DataTable, RowId, SizeBytes as _, TableId, TimePoint}; - -#[test] -fn data_table_sizes_basics() { - use arrow2::array::{BooleanArray, UInt64Array}; - use re_types::Loggable as _; - - fn expect(mut cell: DataCell, num_rows: usize, num_bytes: u64) { - cell.compute_size_bytes(); - - let row = DataRow::from_cells1( - RowId::random(), - "a/b/c", - TimePoint::default(), - cell.num_instances(), - cell, - ) - .unwrap(); - - let table = DataTable::from_rows( - TableId::random(), - std::iter::repeat_with(|| row.clone()).take(num_rows), - ); - assert_eq!(num_bytes, table.heap_size_bytes()); - - let mut table = DataTable::from_arrow_msg(&table.to_arrow_msg().unwrap()).unwrap(); - table.compute_all_size_bytes(); - let num_bytes = table.heap_size_bytes(); - assert_eq!(num_bytes, table.heap_size_bytes()); - } - - // boolean - let mut cell = DataCell::from_arrow( - "some_bools".into(), - BooleanArray::from(vec![Some(true), Some(false), Some(true)]).boxed(), - ); - cell.compute_size_bytes(); - expect( - cell.clone(), // - 10_000, // num_rows - 2_690_064, // expected_num_bytes - ); - expect( - DataCell::from_arrow("some_bools".into(), cell.to_arrow().sliced(1, 1)), - 10_000, // num_rows - 2_690_064, // expected_num_bytes - ); - - // primitive - let mut cell = DataCell::from_arrow( - "some_u64s".into(), - UInt64Array::from_vec(vec![1, 2, 3]).boxed(), - ); - cell.compute_size_bytes(); - expect( - cell.clone(), // - 10_000, // num_rows - 2_840_064, // expected_num_bytes - ); - expect( - DataCell::from_arrow("some_u64s".into(), cell.to_arrow().sliced(1, 1)), - 10_000, // num_rows - 2_680_064, // expected_num_bytes - ); - - // utf8 (and more generally: dyn_binary) - let mut cell = DataCell::from_native( - [ - re_types::components::Text("hey".into()), - re_types::components::Text("hey".into()), - re_types::components::Text("hey".into()), - ] - .as_slice(), - ); - cell.compute_size_bytes(); - expect( - cell.clone(), // - 10_000, // num_rows - 3_090_064, // expected_num_bytes - ); - expect( - DataCell::from_arrow( - re_types::components::Text::name(), - cell.to_arrow().sliced(1, 1), - ), - 10_000, // num_rows - 2_950_064, // expected_num_bytes - ); - - // fixedsizelist - let mut cell = DataCell::from_native( - [ - re_types::components::Position2D::from([42.0, 666.0]), - re_types::components::Position2D::from([42.0, 666.0]), - re_types::components::Position2D::from([42.0, 666.0]), - ] - .as_slice(), - ); - cell.compute_size_bytes(); - expect( - cell.clone(), // - 10_000, // num_rows - 4_080_064, - ); - expect( - DataCell::from_arrow( - re_types::components::Position2D::name(), - cell.to_arrow().sliced(1, 1), - ), - 10_000, // num_rows - 3_920_064, - ); - - // variable list - let mut cell = DataCell::from_native( - [ - re_types::components::LineStrip2D::from(vec![ - [42.0, 666.0], - [42.0, 666.0], - [42.0, 666.0], - ]), - re_types::components::LineStrip2D::from(vec![ - [42.0, 666.0], - [42.0, 666.0], - [42.0, 666.0], - ]), - re_types::components::LineStrip2D::from(vec![ - [42.0, 666.0], - [42.0, 666.0], - [42.0, 666.0], - ]), - ] - .as_slice(), - ); - cell.compute_size_bytes(); - expect( - cell.clone(), // - 10_000, // num_rows - 6_120_064, // expected_num_bytes - ); - expect( - DataCell::from_arrow( - re_types::components::Position2D::name(), - cell.to_arrow().sliced(1, 1), - ), - 10_000, // num_rows - 5_560_064, // expected_num_bytes - ); -} diff --git a/crates/re_components/tests/data_table_batcher.rs b/crates/re_components/tests/data_table_batcher.rs deleted file mode 100644 index 3d47303164e7..000000000000 --- a/crates/re_components/tests/data_table_batcher.rs +++ /dev/null @@ -1,323 +0,0 @@ -use crossbeam::{channel::TryRecvError, select}; -use itertools::Itertools as _; - -use re_log_types::{ - DataRow, DataTableBatcher, DataTableBatcherConfig, RowId, SizeBytes, TimePoint, Timeline, -}; -use re_log_types::{DataTable, TableId, Time}; -use re_types::components::{Color, Position2D, Text}; - -#[test] -fn manual_trigger() { - let batcher = DataTableBatcher::new(DataTableBatcherConfig::NEVER).unwrap(); - let tables = batcher.tables(); - - let mut expected = create_table(); - expected.compute_all_size_bytes(); - - for _ in 0..3 { - assert_eq!(Err(TryRecvError::Empty), tables.try_recv()); - - for row in expected.to_rows() { - batcher.push_row(row.unwrap()); - } - - assert_eq!(Err(TryRecvError::Empty), tables.try_recv()); - - batcher.flush_blocking(); - - { - let mut table = tables.recv().unwrap(); - // NOTE: Override the resulting table's ID so they can be compared. - table.table_id = expected.table_id; - - similar_asserts::assert_eq!(expected, table); - } - - assert_eq!(Err(TryRecvError::Empty), tables.try_recv()); - } - - drop(batcher); - - assert_eq!(Err(TryRecvError::Disconnected), tables.try_recv()); -} - -#[test] -fn shutdown_trigger() { - let batcher = DataTableBatcher::new(DataTableBatcherConfig::NEVER).unwrap(); - let tables = batcher.tables(); - - let table = create_table(); - let rows: Vec<_> = table.to_rows().try_collect().unwrap(); - - for _ in 0..3 { - assert_eq!(Err(TryRecvError::Empty), tables.try_recv()); - - for row in rows.clone() { - batcher.push_row(row); - } - - assert_eq!(Err(TryRecvError::Empty), tables.try_recv()); - } - - drop(batcher); - - let expected = DataTable::from_rows( - TableId::ZERO, - std::iter::repeat_with(|| rows.clone()).take(3).flatten(), - ); - - select! { - recv(tables) -> batch => { - let mut table = batch.unwrap(); - // NOTE: Override the resulting table's ID so they can be compared. - table.table_id = expected.table_id; - - similar_asserts::assert_eq!(expected, table); - } - default(std::time::Duration::from_millis(50)) => { - panic!("output channel never yielded any table"); - } - } - - assert_eq!(Err(TryRecvError::Disconnected), tables.try_recv()); -} - -#[test] -fn num_bytes_trigger() { - let table = create_table(); - let rows: Vec<_> = table.to_rows().try_collect().unwrap(); - let flush_duration = std::time::Duration::from_millis(50); - let flush_num_bytes = rows - .iter() - .take(rows.len() - 1) - .map(|row| row.total_size_bytes()) - .sum::(); - - let batcher = DataTableBatcher::new(DataTableBatcherConfig { - flush_num_bytes, - flush_tick: flush_duration, - ..DataTableBatcherConfig::NEVER - }) - .unwrap(); - let tables = batcher.tables(); - - assert_eq!(Err(TryRecvError::Empty), tables.try_recv()); - - for row in table.to_rows() { - batcher.push_row(row.unwrap()); - } - - // Expect all rows except for the last one (num_bytes trigger). - select! { - recv(tables) -> batch => { - let table = batch.unwrap(); - let expected = DataTable::from_rows( - table.table_id, - rows.clone().into_iter().take(rows.len() - 1), - ); - similar_asserts::assert_eq!(expected, table); - } - default(flush_duration) => { - panic!("output channel never yielded any table"); - } - } - - // Expect just the last row (duration trigger). - select! { - recv(tables) -> batch => { - let table = batch.unwrap(); - let expected = DataTable::from_rows( - table.table_id, - rows.last().cloned(), - ); - similar_asserts::assert_eq!(expected, table); - } - default(flush_duration * 2) => { - panic!("output channel never yielded any table"); - } - } - - assert_eq!(Err(TryRecvError::Empty), tables.try_recv()); - - drop(batcher); - - assert_eq!(Err(TryRecvError::Disconnected), tables.try_recv()); -} - -#[test] -fn num_rows_trigger() { - let table = create_table(); - let rows: Vec<_> = table.to_rows().try_collect().unwrap(); - let flush_duration = std::time::Duration::from_millis(50); - let flush_num_rows = rows.len() as u64 - 1; - - let batcher = DataTableBatcher::new(DataTableBatcherConfig { - flush_num_rows, - flush_tick: flush_duration, - ..DataTableBatcherConfig::NEVER - }) - .unwrap(); - let tables = batcher.tables(); - - assert_eq!(Err(TryRecvError::Empty), tables.try_recv()); - - for row in table.to_rows() { - batcher.push_row(row.unwrap()); - } - - // Expect all rows except for the last one. - select! { - recv(tables) -> batch => { - let table = batch.unwrap(); - let expected = DataTable::from_rows( - table.table_id, - rows.clone().into_iter().take(rows.len() - 1), - ); - similar_asserts::assert_eq!(expected, table); - } - default(flush_duration) => { - panic!("output channel never yielded any table"); - } - } - - // Expect just the last row. - select! { - recv(tables) -> batch => { - let table = batch.unwrap(); - let expected = DataTable::from_rows( - table.table_id, - rows.last().cloned(), - ); - similar_asserts::assert_eq!(expected, table); - } - default(flush_duration * 2) => { - panic!("output channel never yielded any table"); - } - } - - assert_eq!(Err(TryRecvError::Empty), tables.try_recv()); - - drop(batcher); - - assert_eq!(Err(TryRecvError::Disconnected), tables.try_recv()); -} - -#[test] -fn duration_trigger() { - let table = create_table(); - let rows: Vec<_> = table.to_rows().try_collect().unwrap(); - - let flush_duration = std::time::Duration::from_millis(50); - - let batcher = DataTableBatcher::new(DataTableBatcherConfig { - flush_tick: flush_duration, - ..DataTableBatcherConfig::NEVER - }) - .unwrap(); - let tables = batcher.tables(); - - assert_eq!(Err(TryRecvError::Empty), tables.try_recv()); - - _ = std::thread::Builder::new().spawn({ - let mut rows = rows.clone(); - let batcher = batcher.clone(); - move || { - for row in rows.drain(..rows.len() - 1) { - batcher.push_row(row); - } - - std::thread::sleep(flush_duration * 2); - - let row = rows.last().cloned().unwrap(); - batcher.push_row(row); - } - }); - - // Expect all rows except for the last one. - select! { - recv(tables) -> batch => { - let table = batch.unwrap(); - let expected = DataTable::from_rows( - table.table_id, - rows.clone().into_iter().take(rows.len() - 1), - ); - similar_asserts::assert_eq!(expected, table); - } - default(flush_duration * 2) => { - panic!("output channel never yielded any table"); - } - } - - // Expect just the last row. - select! { - recv(tables) -> batch => { - let table = batch.unwrap(); - let expected = DataTable::from_rows( - table.table_id, - rows.last().cloned(), - ); - similar_asserts::assert_eq!(expected, table); - } - default(flush_duration * 4) => { - panic!("output channel never yielded any table"); - } - } - - assert_eq!(Err(TryRecvError::Empty), tables.try_recv()); - - drop(batcher); - - assert_eq!(Err(TryRecvError::Disconnected), tables.try_recv()); -} - -fn create_table() -> DataTable { - let timepoint = |frame_nr: i64| { - TimePoint::from([ - (Timeline::log_time(), Time::now().into()), - (Timeline::new_sequence("frame_nr"), frame_nr.into()), - ]) - }; - - let row0 = { - let num_instances = 2; - let positions: &[Position2D] = &[[10.0, 10.0].into(), [20.0, 20.0].into()]; - let colors: &[_] = &[Color::from_rgb(128, 128, 128)]; - let labels: &[Text] = &[]; - - DataRow::from_cells3( - RowId::random(), - "a", - timepoint(1), - num_instances, - (positions, colors, labels), - ) - .unwrap() - }; - - let row1 = { - let num_instances = 0; - let colors: &[Color] = &[]; - - DataRow::from_cells1(RowId::random(), "b", timepoint(1), num_instances, colors).unwrap() - }; - - let row2 = { - let num_instances = 1; - let colors: &[_] = &[Color::from_rgb(255, 255, 255)]; - let labels: &[_] = &[Text("hey".into())]; - - DataRow::from_cells2( - RowId::random(), - "c", - timepoint(2), - num_instances, - (colors, labels), - ) - .unwrap() - }; - - let mut table = DataTable::from_rows(TableId::ZERO, [row0, row1, row2]); - table.compute_all_size_bytes(); - table -} From e9596e4fd2bb5b92c2bd6b6f464ebba664218e5a Mon Sep 17 00:00:00 2001 From: Clement Rey Date: Mon, 25 Sep 2023 11:17:08 +0200 Subject: [PATCH 2/3] adapt everything to the brand new world --- Cargo.lock | 49 +-- crates/re_arrow_store/Cargo.toml | 2 +- crates/re_arrow_store/benches/arrow2.rs | 3 +- crates/re_arrow_store/benches/data_store.rs | 6 +- .../re_arrow_store/examples/dump_dataframe.rs | 7 +- .../examples/latest_component.rs | 4 +- .../examples/latest_components.rs | 4 +- .../examples/range_components.rs | 4 +- crates/re_arrow_store/src/arrow_util.rs | 89 ++--- crates/re_arrow_store/src/store.rs | 6 +- crates/re_arrow_store/tests/correctness.rs | 6 +- crates/re_arrow_store/tests/data_store.rs | 9 +- crates/re_arrow_store/tests/dump.rs | 6 +- crates/re_arrow_store/tests/internals.rs | 5 +- crates/re_data_source/Cargo.toml | 1 - .../re_data_source/src/load_file_contents.rs | 2 +- crates/re_data_source/src/load_file_path.rs | 2 +- crates/re_data_store/Cargo.toml | 3 +- crates/re_data_store/examples/memory_usage.rs | 4 +- crates/re_data_store/src/store_db.rs | 47 --- crates/re_data_ui/Cargo.toml | 1 - crates/re_data_ui/src/data.rs | 54 ---- crates/re_log_encoding/Cargo.toml | 3 +- .../benches/msg_encode_benchmark.rs | 11 +- crates/re_log_types/src/data_table.rs | 71 ++++ crates/re_log_types/src/lib.rs | 19 ++ crates/re_log_types/src/load_file.rs | 158 +++++++++ crates/re_query/Cargo.toml | 7 +- crates/re_query/benches/query_benchmark.rs | 189 +++++++---- crates/re_query/benches/query_benchmark2.rs | 304 ------------------ crates/re_query/examples/range.rs | 4 +- crates/re_query/src/query.rs | 2 +- .../re_query/tests/archetype_query_tests.rs | 3 +- .../re_query/tests/archetype_range_tests.rs | 3 +- crates/re_query/tests/query_tests.rs | 3 +- crates/re_query/tests/range_tests.rs | 3 +- crates/re_sdk/Cargo.toml | 7 +- crates/re_sdk/src/lib.rs | 7 - crates/re_sdk/src/recording_stream.rs | 11 +- crates/re_space_view_bar_chart/Cargo.toml | 1 - crates/re_space_view_spatial/Cargo.toml | 1 - crates/re_space_view_tensor/Cargo.toml | 1 - crates/re_space_view_text_log/Cargo.toml | 1 - crates/re_space_view_time_series/Cargo.toml | 1 - crates/re_types/Cargo.toml | 5 + crates/re_types/src/datagen.rs | 63 ++++ crates/re_types/src/lib.rs | 9 + crates/re_viewer/Cargo.toml | 1 - crates/re_viewer/src/lib.rs | 6 +- crates/re_viewer_context/Cargo.toml | 3 +- crates/re_viewport/Cargo.toml | 1 - rerun_py/Cargo.toml | 1 - rerun_py/src/arrow.rs | 27 +- rerun_py/src/python_bridge.rs | 5 +- 54 files changed, 549 insertions(+), 696 deletions(-) create mode 100644 crates/re_log_types/src/load_file.rs delete mode 100644 crates/re_query/benches/query_benchmark2.rs create mode 100644 crates/re_types/src/datagen.rs diff --git a/Cargo.lock b/Cargo.lock index a277bb3dc488..823001a5c3d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2189,7 +2189,6 @@ dependencies = [ "bytemuck", "cfg-if", "crunchy", - "serde", ] [[package]] @@ -4041,7 +4040,6 @@ dependencies = [ "polars-core", "polars-ops", "rand", - "re_components", "re_error", "re_format", "re_log", @@ -4080,36 +4078,6 @@ dependencies = [ "wasm-bindgen-cli-support", ] -[[package]] -name = "re_components" -version = "0.9.0-alpha.4" -dependencies = [ - "array-init", - "arrow2", - "arrow2_convert", - "bytemuck", - "crossbeam", - "document-features", - "ecolor", - "glam", - "half 2.3.1", - "image", - "itertools 0.11.0", - "lazy_static", - "ndarray", - "rand", - "re_log", - "re_log_types", - "re_tracing", - "re_types", - "rmp-serde", - "serde", - "similar-asserts", - "thiserror", - "zune-core", - "zune-jpeg", -] - [[package]] name = "re_crash_handler" version = "0.9.0-alpha.4" @@ -4130,7 +4098,6 @@ dependencies = [ "itertools 0.11.0", "rayon", "re_build_tools", - "re_components", "re_log", "re_log_encoding", "re_log_types", @@ -4152,7 +4119,6 @@ dependencies = [ "nohash-hasher", "rand", "re_arrow_store", - "re_components", "re_format", "re_int_histogram", "re_log", @@ -4178,7 +4144,6 @@ dependencies = [ "image", "itertools 0.11.0", "re_arrow_store", - "re_components", "re_data_store", "re_error", "re_format", @@ -4246,12 +4211,12 @@ dependencies = [ "mimalloc", "parking_lot 0.12.1", "re_build_info", - "re_components", "re_error", "re_log", "re_log_types", "re_smart_channel", "re_tracing", + "re_types", "rmp-serde", "serde_test", "thiserror", @@ -4330,7 +4295,6 @@ dependencies = [ "polars-core", "rand", "re_arrow_store", - "re_components", "re_data_store", "re_format", "re_log", @@ -4410,7 +4374,6 @@ dependencies = [ "rand", "re_build_info", "re_build_tools", - "re_components", "re_log", "re_log_encoding", "re_log_types", @@ -4472,7 +4435,6 @@ dependencies = [ "egui", "egui_plot", "re_arrow_store", - "re_components", "re_data_store", "re_log", "re_log_types", @@ -4502,7 +4464,6 @@ dependencies = [ "parking_lot 0.12.1", "rayon", "re_arrow_store", - "re_components", "re_data_store", "re_data_ui", "re_error", @@ -4533,7 +4494,6 @@ dependencies = [ "half 2.3.1", "ndarray", "re_arrow_store", - "re_components", "re_data_store", "re_data_ui", "re_log", @@ -4574,7 +4534,6 @@ dependencies = [ "egui", "egui_extras", "re_arrow_store", - "re_components", "re_data_store", "re_data_ui", "re_log", @@ -4596,7 +4555,6 @@ dependencies = [ "egui_plot", "itertools 0.11.0", "re_arrow_store", - "re_components", "re_format", "re_log", "re_log_types", @@ -4694,6 +4652,7 @@ dependencies = [ "mime_guess", "ndarray", "once_cell", + "rand", "rayon", "re_build_tools", "re_error", @@ -4779,7 +4738,6 @@ dependencies = [ "re_arrow_store", "re_build_info", "re_build_tools", - "re_components", "re_data_source", "re_data_store", "re_data_ui", @@ -4838,7 +4796,6 @@ dependencies = [ "nohash-hasher", "parking_lot 0.12.1", "re_arrow_store", - "re_components", "re_data_source", "re_data_store", "re_log", @@ -4871,7 +4828,6 @@ dependencies = [ "itertools 0.11.0", "nohash-hasher", "re_arrow_store", - "re_components", "re_data_store", "re_data_ui", "re_log", @@ -5059,7 +5015,6 @@ dependencies = [ "rand", "re_build_info", "re_build_tools", - "re_components", "re_error", "re_log", "re_log_types", diff --git a/crates/re_arrow_store/Cargo.toml b/crates/re_arrow_store/Cargo.toml index aae1ff2b9957..f95ee9974453 100644 --- a/crates/re_arrow_store/Cargo.toml +++ b/crates/re_arrow_store/Cargo.toml @@ -72,7 +72,7 @@ polars-ops = { workspace = true, optional = true, features = [ [dev-dependencies] -re_components = { workspace = true, features = ["arrow_datagen"] } +re_types = { workspace = true, features = ["datagen"] } anyhow.workspace = true criterion = "0.5" diff --git a/crates/re_arrow_store/benches/arrow2.rs b/crates/re_arrow_store/benches/arrow2.rs index 2e542d540235..8b77c84fd2ba 100644 --- a/crates/re_arrow_store/benches/arrow2.rs +++ b/crates/re_arrow_store/benches/arrow2.rs @@ -8,8 +8,9 @@ use std::sync::Arc; use arrow2::array::{Array, PrimitiveArray, StructArray, UnionArray}; use criterion::{criterion_group, Criterion}; use itertools::Itertools; -use re_components::datagen::{build_some_instances, build_some_positions2d}; + use re_log_types::{DataCell, SizeBytes as _}; +use re_types::datagen::{build_some_instances, build_some_positions2d}; use re_types::{ components::{InstanceKey, Position2D}, testing::{build_some_large_structs, LargeStruct}, diff --git a/crates/re_arrow_store/benches/data_store.rs b/crates/re_arrow_store/benches/data_store.rs index bf72f12d438a..1f8185e5b61e 100644 --- a/crates/re_arrow_store/benches/data_store.rs +++ b/crates/re_arrow_store/benches/data_store.rs @@ -8,8 +8,10 @@ use re_arrow_store::{ DataStore, DataStoreConfig, GarbageCollectionOptions, GarbageCollectionTarget, LatestAtQuery, RangeQuery, TimeInt, TimeRange, }; -use re_components::datagen::{build_frame_nr, build_some_instances}; -use re_log_types::{DataCell, DataRow, DataTable, EntityPath, RowId, TableId, TimeType, Timeline}; +use re_log_types::{ + build_frame_nr, DataCell, DataRow, DataTable, EntityPath, RowId, TableId, TimeType, Timeline, +}; +use re_types::datagen::build_some_instances; use re_types::{ components::InstanceKey, testing::{build_some_large_structs, LargeStruct}, diff --git a/crates/re_arrow_store/examples/dump_dataframe.rs b/crates/re_arrow_store/examples/dump_dataframe.rs index fb75ff693f34..0728c018788a 100644 --- a/crates/re_arrow_store/examples/dump_dataframe.rs +++ b/crates/re_arrow_store/examples/dump_dataframe.rs @@ -5,11 +5,8 @@ //! ``` use re_arrow_store::{test_row, DataStore}; -use re_components::datagen::{ - build_frame_nr, build_log_time, build_some_instances, build_some_instances_from, - build_some_positions2d, -}; -use re_log_types::{EntityPath, Time}; +use re_log_types::{build_frame_nr, build_log_time, EntityPath, Time}; +use re_types::datagen::{build_some_instances, build_some_instances_from, build_some_positions2d}; use re_types::{components::InstanceKey, testing::build_some_large_structs, Loggable}; // --- diff --git a/crates/re_arrow_store/examples/latest_component.rs b/crates/re_arrow_store/examples/latest_component.rs index e3990be54e1c..35d6248e0985 100644 --- a/crates/re_arrow_store/examples/latest_component.rs +++ b/crates/re_arrow_store/examples/latest_component.rs @@ -6,8 +6,8 @@ use re_arrow_store::polars_util::latest_component; use re_arrow_store::{test_row, DataStore, LatestAtQuery, TimeType, Timeline}; -use re_components::datagen::{build_frame_nr, build_some_positions2d}; -use re_log_types::EntityPath; +use re_log_types::{build_frame_nr, EntityPath}; +use re_types::datagen::build_some_positions2d; use re_types::{ components::{InstanceKey, Position2D}, testing::{build_some_large_structs, LargeStruct}, diff --git a/crates/re_arrow_store/examples/latest_components.rs b/crates/re_arrow_store/examples/latest_components.rs index ddb29d445b5b..5203c312cc46 100644 --- a/crates/re_arrow_store/examples/latest_components.rs +++ b/crates/re_arrow_store/examples/latest_components.rs @@ -8,8 +8,8 @@ use polars_core::prelude::*; use re_arrow_store::polars_util::latest_components; use re_arrow_store::{test_row, DataStore, LatestAtQuery, TimeType, Timeline}; -use re_components::datagen::{build_frame_nr, build_some_positions2d}; -use re_log_types::EntityPath; +use re_log_types::{build_frame_nr, EntityPath}; +use re_types::datagen::build_some_positions2d; use re_types::{ components::{InstanceKey, Position2D}, testing::{build_some_large_structs, LargeStruct}, diff --git a/crates/re_arrow_store/examples/range_components.rs b/crates/re_arrow_store/examples/range_components.rs index f83fefab3159..125a6d37b163 100644 --- a/crates/re_arrow_store/examples/range_components.rs +++ b/crates/re_arrow_store/examples/range_components.rs @@ -6,8 +6,8 @@ use polars_core::prelude::JoinType; use re_arrow_store::{polars_util, test_row, DataStore, RangeQuery, TimeRange}; -use re_components::datagen::{build_frame_nr, build_some_positions2d}; -use re_log_types::{EntityPath, TimeType, Timeline}; +use re_log_types::{build_frame_nr, EntityPath, TimeType, Timeline}; +use re_types::datagen::build_some_positions2d; use re_types::{ components::{InstanceKey, Position2D}, testing::{build_some_large_structs, LargeStruct}, diff --git a/crates/re_arrow_store/src/arrow_util.rs b/crates/re_arrow_store/src/arrow_util.rs index d4dec94120b9..b0cb2a7de78f 100644 --- a/crates/re_arrow_store/src/arrow_util.rs +++ b/crates/re_arrow_store/src/arrow_util.rs @@ -224,82 +224,39 @@ impl ArrayExt for dyn Array { #[test] fn test_clean_for_polars_nomodify() { - use re_components::datagen::build_some_colors; use re_log_types::DataCell; + use re_types::datagen::build_some_colors; // Colors don't need polars cleaning let cell: DataCell = build_some_colors(5).try_into().unwrap(); let cleaned = cell.as_arrow_ref().clean_for_polars(); assert_eq!(cell.as_arrow_ref(), &*cleaned); -} - -#[cfg(test)] -mod tests { - use arrow2::datatypes::{DataType, Field, UnionMode}; - use arrow2_convert::{ArrowDeserialize, ArrowField, ArrowSerialize}; - use re_components::FixedSizeArrayField; - use re_log_types::DataCell; - - use crate::ArrayExt; - #[derive( - Copy, Clone, Debug, Default, PartialEq, ArrowField, ArrowSerialize, ArrowDeserialize, - )] - #[arrow_field(transparent)] - pub struct Vec3D(#[arrow_field(type = "FixedSizeArrayField")] pub [f32; 3]); - - #[derive(Clone, Copy, Debug, PartialEq, ArrowField, ArrowSerialize, ArrowDeserialize)] - #[arrow_field(type = "dense")] - enum TestComponentWithUnionAndFixedSizeList { - Bool(bool), - Vec3D(Vec3D), - } - - re_log_types::component_legacy_shim!(TestComponentWithUnionAndFixedSizeList); + crate::polars_util::dataframe_from_cells(&[Some(cell)]).unwrap(); +} - impl re_log_types::LegacyComponent for TestComponentWithUnionAndFixedSizeList { - fn legacy_name() -> re_log_types::ComponentName { - "test_component_with_union_and_fixed_size_list".into() - } - } +#[test] +fn test_clean_for_polars_modify() { + use std::f32::consts::PI; - #[test] - fn test_clean_for_polars_modify() { - // Pick a type with both Unions and FixedSizeLists - let elements = vec![TestComponentWithUnionAndFixedSizeList::Bool(false)]; + use re_log_types::DataCell; + use re_types::components::Transform3D; + use re_types::datatypes::{ + Angle, RotationAxisAngle, Scale3D, TranslationAndMat3x3, TranslationRotationScale3D, + }; - let cell: DataCell = elements.try_into().unwrap(); - assert_eq!( - *cell.datatype(), - DataType::Union( - vec![ - Field::new("Bool", DataType::Boolean, false), - Field::new( - "Vec3D", - DataType::FixedSizeList( - Box::new(Field::new("item", DataType::Float32, false)), - 3 - ), - false - ) - ], - None, - UnionMode::Dense - ) - ); + let cell = DataCell::try_from_native([ + Transform3D::new(TranslationAndMat3x3::translation([1.0, 0.0, 0.0])), // + Transform3D::new(TranslationRotationScale3D { + rotation: Some(RotationAxisAngle::new([0.0, 0.0, 1.0], Angle::Radians(PI / 4.)).into()), + scale: Some(Scale3D::from(2.0)), + ..Default::default() + }), + ]) + .unwrap(); - let cleaned = cell.as_arrow_ref().clean_for_polars(); + let cleaned = cell.as_arrow_ref().clean_for_polars(); + assert_ne!(cell.as_arrow_ref(), &*cleaned); - assert_eq!( - *cleaned.data_type(), - DataType::Struct(vec![ - Field::new("Bool", DataType::Boolean, false), - Field::new( - "Vec3D", - DataType::List(Box::new(Field::new("item", DataType::Float32, false))), - false - ) - ],) - ); - } + crate::polars_util::dataframe_from_cells(&[Some(cell)]).unwrap(); } diff --git a/crates/re_arrow_store/src/store.rs b/crates/re_arrow_store/src/store.rs index baf55f9bbcf5..083883e84a99 100644 --- a/crates/re_arrow_store/src/store.rs +++ b/crates/re_arrow_store/src/store.rs @@ -335,7 +335,7 @@ impl DataStore { /// ``` #[test] fn datastore_internal_repr() { - use re_components::datagen::data_table_example; + use re_log_types::DataTable; use re_types::{components::InstanceKey, Loggable as _}; let mut store = DataStore::new( @@ -347,11 +347,11 @@ fn datastore_internal_repr() { }, ); - let timeless = data_table_example(true); + let timeless = DataTable::example(true); eprintln!("{timeless}"); store.insert_table(&timeless).unwrap(); - let temporal = data_table_example(false); + let temporal = DataTable::example(false); eprintln!("{temporal}"); store.insert_table(&temporal).unwrap(); diff --git a/crates/re_arrow_store/tests/correctness.rs b/crates/re_arrow_store/tests/correctness.rs index c3a2f8f65022..6b54dac27c62 100644 --- a/crates/re_arrow_store/tests/correctness.rs +++ b/crates/re_arrow_store/tests/correctness.rs @@ -10,10 +10,10 @@ use re_arrow_store::{ test_row, test_util::sanity_unwrap, DataStore, DataStoreConfig, DataStoreStats, GarbageCollectionOptions, LatestAtQuery, WriteError, }; -use re_components::datagen::{ - build_frame_nr, build_log_time, build_some_colors, build_some_instances, build_some_positions2d, +use re_log_types::{ + build_frame_nr, build_log_time, DataCell, Duration, EntityPath, Time, TimeType, Timeline, }; -use re_log_types::{DataCell, Duration, EntityPath, Time, TimeType, Timeline}; +use re_types::datagen::{build_some_colors, build_some_instances, build_some_positions2d}; use re_types::{components::InstanceKey, Loggable as _}; // --- diff --git a/crates/re_arrow_store/tests/data_store.rs b/crates/re_arrow_store/tests/data_store.rs index 58ee3950448b..b3471dc2b2c7 100644 --- a/crates/re_arrow_store/tests/data_store.rs +++ b/crates/re_arrow_store/tests/data_store.rs @@ -15,11 +15,12 @@ use re_arrow_store::{ DataStoreStats, GarbageCollectionOptions, GarbageCollectionTarget, LatestAtQuery, RangeQuery, TimeInt, TimeRange, }; -use re_components::datagen::{ - build_frame_nr, build_some_colors, build_some_instances, build_some_instances_from, - build_some_positions2d, +use re_log_types::{ + build_frame_nr, DataCell, DataRow, DataTable, EntityPath, TableId, TimeType, Timeline, +}; +use re_types::datagen::{ + build_some_colors, build_some_instances, build_some_instances_from, build_some_positions2d, }; -use re_log_types::{DataCell, DataRow, DataTable, EntityPath, TableId, TimeType, Timeline}; use re_types::{ components::{Color, InstanceKey, Position2D}, testing::{build_some_large_structs, LargeStruct}, diff --git a/crates/re_arrow_store/tests/dump.rs b/crates/re_arrow_store/tests/dump.rs index 9047d7e17269..f44e849a69ad 100644 --- a/crates/re_arrow_store/tests/dump.rs +++ b/crates/re_arrow_store/tests/dump.rs @@ -7,10 +7,8 @@ use re_arrow_store::{ test_row, test_util::sanity_unwrap, DataStore, DataStoreStats, GarbageCollectionOptions, TimeInt, TimeRange, Timeline, }; -use re_components::datagen::{ - build_frame_nr, build_log_time, build_some_colors, build_some_instances, build_some_positions2d, -}; -use re_log_types::{DataTable, EntityPath, TableId}; +use re_log_types::{build_frame_nr, build_log_time, DataTable, EntityPath, TableId}; +use re_types::datagen::{build_some_colors, build_some_instances, build_some_positions2d}; use re_types::{components::InstanceKey, Loggable as _}; // --- Dump --- diff --git a/crates/re_arrow_store/tests/internals.rs b/crates/re_arrow_store/tests/internals.rs index d60c819fd0ea..2dc3fa2a5845 100644 --- a/crates/re_arrow_store/tests/internals.rs +++ b/crates/re_arrow_store/tests/internals.rs @@ -5,9 +5,8 @@ use std::sync::atomic::{AtomicBool, Ordering::SeqCst}; use re_arrow_store::{DataStore, DataStoreConfig}; -use re_components::datagen::{build_frame_nr, build_some_instances}; -use re_log_types::{DataRow, EntityPath, RowId, TimePoint}; -use re_types::{components::InstanceKey, Loggable as _}; +use re_log_types::{build_frame_nr, DataRow, EntityPath, RowId, TimePoint}; +use re_types::{components::InstanceKey, datagen::build_some_instances, Loggable as _}; // --- Internals --- diff --git a/crates/re_data_source/Cargo.toml b/crates/re_data_source/Cargo.toml index 980f0a2b9547..2c6f7a4e063b 100644 --- a/crates/re_data_source/Cargo.toml +++ b/crates/re_data_source/Cargo.toml @@ -21,7 +21,6 @@ default = [] [dependencies] -re_components.workspace = true re_log_encoding = { workspace = true, features = ["decoder"] } re_log_types.workspace = true re_log.workspace = true diff --git a/crates/re_data_source/src/load_file_contents.rs b/crates/re_data_source/src/load_file_contents.rs index c727ec9e5549..4d98b52c3525 100644 --- a/crates/re_data_source/src/load_file_contents.rs +++ b/crates/re_data_source/src/load_file_contents.rs @@ -82,7 +82,7 @@ fn log_msg_from_file_contents( let FileContents { name, bytes } = file_contents; let entity_path = re_log_types::EntityPath::from_single_string(name.clone()); - let cells = re_components::data_cells_from_file_contents(&name, bytes.to_vec())?; + let cells = re_log_types::data_cells_from_file_contents(&name, bytes.to_vec())?; let num_instances = cells.first().map_or(0, |cell| cell.num_instances()); diff --git a/crates/re_data_source/src/load_file_path.rs b/crates/re_data_source/src/load_file_path.rs index b29aa16e52ad..d5466d995488 100644 --- a/crates/re_data_source/src/load_file_path.rs +++ b/crates/re_data_source/src/load_file_path.rs @@ -76,7 +76,7 @@ fn log_msg_from_file_path( file_path: &std::path::Path, ) -> anyhow::Result { let entity_path = re_log_types::EntityPath::from_file_path_as_single_string(file_path); - let cells = re_components::data_cells_from_file_path(file_path)?; + let cells = re_log_types::data_cells_from_file_path(file_path)?; let num_instances = cells.first().map_or(0, |cell| cell.num_instances()); diff --git a/crates/re_data_store/Cargo.toml b/crates/re_data_store/Cargo.toml index 5967566f33be..e2ba2b26f3e2 100644 --- a/crates/re_data_store/Cargo.toml +++ b/crates/re_data_store/Cargo.toml @@ -25,7 +25,6 @@ serde = ["dep:serde", "re_log_types/serde"] [dependencies] re_arrow_store.workspace = true -re_components.workspace = true re_format.workspace = true re_int_histogram.workspace = true re_log_encoding = { workspace = true, optional = true } @@ -44,7 +43,7 @@ serde = { version = "1", features = ["derive", "rc"], optional = true } thiserror = "1.0" [dev-dependencies] -re_components = { workspace = true, features = ["arrow_datagen"] } +re_types = { workspace = true, features = ["datagen"] } re_log_encoding = { workspace = true, features = ["decoder", "encoder"] } criterion = "0.5" mimalloc.workspace = true diff --git a/crates/re_data_store/examples/memory_usage.rs b/crates/re_data_store/examples/memory_usage.rs index c0dbd9140691..b04d279b6e97 100644 --- a/crates/re_data_store/examples/memory_usage.rs +++ b/crates/re_data_store/examples/memory_usage.rs @@ -55,8 +55,8 @@ fn main() { } fn log_messages() { - use re_components::datagen::{build_frame_nr, build_some_positions2d}; - use re_log_types::{LogMsg, TimeInt, TimePoint, Timeline}; + use re_log_types::{build_frame_nr, LogMsg, TimeInt, TimePoint, Timeline}; + use re_types::datagen::build_some_positions2d; // Note: we use Box in this function so that we also count the "static" // part of all the data, i.e. its `std::mem::size_of`. diff --git a/crates/re_data_store/src/store_db.rs b/crates/re_data_store/src/store_db.rs index d0b36f8f7ea9..8c1205fb9f67 100644 --- a/crates/re_data_store/src/store_db.rs +++ b/crates/re_data_store/src/store_db.rs @@ -69,9 +69,6 @@ impl EntityDb { fn try_add_arrow_msg(&mut self, msg: &ArrowMsg) -> Result<(), Error> { re_tracing::profile_function!(); - #[cfg(debug_assertions)] - check_known_component_schemas(msg); - // TODO(#1760): Compute the size of the datacells in the batching threads on the clients. let mut table = DataTable::from_arrow_msg(msg)?; table.compute_all_size_bytes(); @@ -218,50 +215,6 @@ impl EntityDb { } } -/// Check that known (`rerun.`) components have the expected schemas. -#[cfg(debug_assertions)] -fn check_known_component_schemas(msg: &ArrowMsg) { - // Check that we have the expected schemas - let known_fields: ahash::HashMap<&str, &arrow2::datatypes::Field> = - re_components::iter_registered_field_types() - .map(|field| (field.name.as_str(), field)) - .collect(); - - for actual in &msg.schema.fields { - if let Some(expected) = known_fields.get(actual.name.as_str()) { - if let arrow2::datatypes::DataType::List(actual_field) = &actual.data_type { - // NOTE: Don't care about extensions until the migration is over (arrow2-convert - // issues). - let actual_datatype = actual_field.data_type.to_logical_type(); - let expected_datatype = expected.data_type.to_logical_type(); - if actual_datatype != expected_datatype { - re_log::warn_once!( - "The incoming component {:?} had the type:\n{:#?}\nExpected type:\n{:#?}", - actual.name, - actual_field.data_type, - expected.data_type, - ); - } - if actual.is_nullable != expected.is_nullable { - re_log::warn_once!( - "The incoming component {:?} has is_nullable={}, expected is_nullable={}", - actual.name, - actual.is_nullable, - expected.is_nullable, - ); - } - } else { - re_log::warn_once!( - "The incoming component {:?} was:\n{:#?}\nExpected:\n{:#?}", - actual.name, - actual.data_type, - expected.data_type, - ); - } - } - } -} - // ---------------------------------------------------------------------------- /// A in-memory database built from a stream of [`LogMsg`]es. diff --git a/crates/re_data_ui/Cargo.toml b/crates/re_data_ui/Cargo.toml index f550fd97b8d7..56c84b6e827f 100644 --- a/crates/re_data_ui/Cargo.toml +++ b/crates/re_data_ui/Cargo.toml @@ -17,7 +17,6 @@ all-features = true [dependencies] re_arrow_store.workspace = true -re_components.workspace = true re_data_store.workspace = true re_error.workspace = true re_format.workspace = true diff --git a/crates/re_data_ui/src/data.rs b/crates/re_data_ui/src/data.rs index f1de86ebafbd..4c0b793a8eac 100644 --- a/crates/re_data_ui/src/data.rs +++ b/crates/re_data_ui/src/data.rs @@ -1,6 +1,5 @@ use egui::Vec2; -use re_components::{LegacyMat3x3, LegacyVec2D, LegacyVec3D}; use re_format::format_f32; use re_types::components::{ Color, LineStrip2D, LineStrip3D, Material, MeshProperties, ViewCoordinates, @@ -71,33 +70,6 @@ impl DataUi for ViewCoordinates { } } -impl DataUi for LegacyMat3x3 { - fn data_ui( - &self, - _ctx: &mut ViewerContext<'_>, - ui: &mut egui::Ui, - _verbosity: UiVerbosity, - _query: &re_arrow_store::LatestAtQuery, - ) { - egui::Grid::new("mat3").num_columns(3).show(ui, |ui| { - ui.monospace(self[0][0].to_string()); - ui.monospace(self[1][0].to_string()); - ui.monospace(self[2][0].to_string()); - ui.end_row(); - - ui.monospace(self[0][1].to_string()); - ui.monospace(self[1][1].to_string()); - ui.monospace(self[2][1].to_string()); - ui.end_row(); - - ui.monospace(self[0][2].to_string()); - ui.monospace(self[1][2].to_string()); - ui.monospace(self[2][2].to_string()); - ui.end_row(); - }); - } -} - impl DataUi for re_types::datatypes::Mat3x3 { fn data_ui( &self, @@ -125,19 +97,6 @@ impl DataUi for re_types::datatypes::Mat3x3 { } } -impl DataUi for LegacyVec2D { - fn data_ui( - &self, - _ctx: &mut ViewerContext<'_>, - ui: &mut egui::Ui, - _verbosity: UiVerbosity, - _query: &re_arrow_store::LatestAtQuery, - ) { - ui.label(self.to_string()); - } -} - -// TODO(cmc): annihilate legacy impl DataUi for re_types::datatypes::Vec2D { fn data_ui( &self, @@ -150,19 +109,6 @@ impl DataUi for re_types::datatypes::Vec2D { } } -impl DataUi for LegacyVec3D { - fn data_ui( - &self, - _ctx: &mut ViewerContext<'_>, - ui: &mut egui::Ui, - _verbosity: UiVerbosity, - _query: &re_arrow_store::LatestAtQuery, - ) { - ui.label(self.to_string()); - } -} - -// TODO(cmc): annihilate legacy impl DataUi for re_types::datatypes::Vec3D { fn data_ui( &self, diff --git a/crates/re_log_encoding/Cargo.toml b/crates/re_log_encoding/Cargo.toml index 1c97b1bb7074..4bcbe7064ce7 100644 --- a/crates/re_log_encoding/Cargo.toml +++ b/crates/re_log_encoding/Cargo.toml @@ -54,10 +54,11 @@ wasm-bindgen-futures = "0.4" web-sys = { version = "0.3.52", features = ["Window"] } [dev-dependencies] +re_types = { workspace = true, features = ["datagen"] } + criterion = "0.5" mimalloc.workspace = true serde_test = { version = "1" } -re_components = { workspace = true, features = ["arrow_datagen"] } [lib] bench = false diff --git a/crates/re_log_encoding/benches/msg_encode_benchmark.rs b/crates/re_log_encoding/benches/msg_encode_benchmark.rs index 5d8e26891e69..e2e093ecfb8f 100644 --- a/crates/re_log_encoding/benches/msg_encode_benchmark.rs +++ b/crates/re_log_encoding/benches/msg_encode_benchmark.rs @@ -4,11 +4,11 @@ compile_error!("msg_encode_benchmark requires 'decoder' and 'encoder' features." #[global_allocator] static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; -use re_components::datagen::{build_frame_nr, build_some_colors, build_some_positions2d}; - use re_log_types::{ - entity_path, DataRow, DataTable, Index, LogMsg, RowId, StoreId, StoreKind, TableId, + entity_path, DataRow, DataTable, Index, LogMsg, RowId, StoreId, StoreKind, TableId, TimeInt, + TimeType, Timeline, }; +use re_types::datagen::{build_some_colors, build_some_positions2d}; use criterion::{criterion_group, criterion_main, Criterion}; @@ -238,3 +238,8 @@ fn batch_points_arrow(c: &mut Criterion) { }); } } + +/// Build a ([`Timeline`], [`TimeInt`]) tuple from `frame_nr` suitable for inserting in a [`re_log_types::TimePoint`]. +fn build_frame_nr(frame_nr: TimeInt) -> (Timeline, TimeInt) { + (Timeline::new("frame_nr", TimeType::Sequence), frame_nr) +} diff --git a/crates/re_log_types/src/data_table.rs b/crates/re_log_types/src/data_table.rs index e2401c5c1b46..b86d80b8542b 100644 --- a/crates/re_log_types/src/data_table.rs +++ b/crates/re_log_types/src/data_table.rs @@ -1271,3 +1271,74 @@ impl DataTable { Ok(()) } } + +// --- + +#[cfg(not(target_arch = "wasm32"))] +impl DataTable { + /// Crafts a simple but interesting [`DataTable`]. + pub fn example(timeless: bool) -> DataTable { + use crate::Time; + use re_types::components::{Color, Position2D, Text}; + + let table_id = TableId::random(); + + let mut tick = 0i64; + let mut timepoint = |frame_nr: i64| { + let tp = if timeless { + TimePoint::timeless() + } else { + TimePoint::from([ + (Timeline::log_time(), Time::now().into()), + (Timeline::log_tick(), tick.into()), + (Timeline::new_sequence("frame_nr"), frame_nr.into()), + ]) + }; + tick += 1; + tp + }; + + let row0 = { + let num_instances = 2; + let positions: &[Position2D] = &[[10.0, 10.0].into(), [20.0, 20.0].into()]; + let colors: &[_] = &[Color::from_rgb(128, 128, 128)]; + let labels: &[Text] = &[]; + + DataRow::from_cells3( + RowId::random(), + "a", + timepoint(1), + num_instances, + (positions, colors, labels), + ) + .unwrap() + }; + + let row1 = { + let num_instances = 0; + let colors: &[Color] = &[]; + + DataRow::from_cells1(RowId::random(), "b", timepoint(1), num_instances, colors).unwrap() + }; + + let row2 = { + let num_instances = 1; + let colors: &[_] = &[Color::from_rgb(255, 255, 255)]; + let labels: &[_] = &[Text("hey".into())]; + + DataRow::from_cells2( + RowId::random(), + "c", + timepoint(2), + num_instances, + (colors, labels), + ) + .unwrap() + }; + + let mut table = DataTable::from_rows(table_id, [row0, row1, row2]); + table.compute_all_size_bytes(); + + table + } +} diff --git a/crates/re_log_types/src/lib.rs b/crates/re_log_types/src/lib.rs index 19690af81a5e..4ed9ebb6a641 100644 --- a/crates/re_log_types/src/lib.rs +++ b/crates/re_log_types/src/lib.rs @@ -68,6 +68,13 @@ pub use self::data_table_batcher::{ DataTableBatcher, DataTableBatcherConfig, DataTableBatcherError, }; +mod load_file; + +#[cfg(not(target_arch = "wasm32"))] +pub use self::load_file::data_cells_from_file_path; + +pub use self::load_file::{data_cells_from_file_contents, FromFileError}; + pub mod external { pub use arrow2; pub use arrow2_convert; @@ -487,6 +494,18 @@ macro_rules! component_legacy_shim { } } }; +} + +// --- +/// Build a ([`Timeline`], [`TimeInt`]) tuple from `log_time` suitable for inserting in a [`re_log_types::TimePoint`]. +#[inline] +pub fn build_log_time(log_time: Time) -> (Timeline, TimeInt) { + (Timeline::log_time(), log_time.into()) +} +/// Build a ([`Timeline`], [`TimeInt`]) tuple from `frame_nr` suitable for inserting in a [`re_log_types::TimePoint`]. +#[inline] +pub fn build_frame_nr(frame_nr: TimeInt) -> (Timeline, TimeInt) { + (Timeline::new("frame_nr", TimeType::Sequence), frame_nr) } diff --git a/crates/re_log_types/src/load_file.rs b/crates/re_log_types/src/load_file.rs new file mode 100644 index 000000000000..1008c34348d8 --- /dev/null +++ b/crates/re_log_types/src/load_file.rs @@ -0,0 +1,158 @@ +use crate::DataCell; + +/// Errors from [`data_cells_from_file_path`]. +#[derive(thiserror::Error, Debug)] +pub enum FromFileError { + #[cfg(not(target_arch = "wasm32"))] + #[error(transparent)] + FileRead(#[from] std::io::Error), + + #[error(transparent)] + DataCellError(#[from] crate::DataCellError), + + #[cfg(feature = "image")] + #[error(transparent)] + TensorImageLoad(#[from] re_types::tensor_data::TensorImageLoadError), + + #[error("Unsupported file extension '{extension}' for file {path:?}. To load image files, make sure you compile with the 'image' feature")] + UnknownExtension { + extension: String, + path: std::path::PathBuf, + }, + + #[error(transparent)] + Other(#[from] anyhow::Error), +} + +/// Read the file at the given path. +/// +/// Supported file extensions are: +/// * `glb`, `gltf`, `obj`: encoded meshes, leaving it to the viewer to decode +/// * `jpg`, `jpeg`: encoded JPEG, leaving it to the viewer to decode. Requires the `image` feature. +/// * `png` and other image formats: decoded here. Requires the `image` feature. +/// +/// All other extensions will return an error. +#[cfg(not(target_arch = "wasm32"))] +pub fn data_cells_from_file_path( + file_path: &std::path::Path, +) -> Result, FromFileError> { + let extension = file_path + .extension() + .unwrap_or_default() + .to_ascii_lowercase() + .to_string_lossy() + .to_string(); + + match extension.as_str() { + "glb" | "gltf" | "obj" => { + use re_types::{archetypes::Asset3D, Archetype}; + let cells: Result, _> = Asset3D::from_file(file_path)? + // TODO(#3414): this should be a method of `Archetype` + .as_component_batches() + .into_iter() + .map(|comp_batch| { + let comp_batch = comp_batch.as_ref(); + Ok(DataCell::from_arrow( + comp_batch.name(), + comp_batch + .try_to_arrow() + .map_err(|err| anyhow::anyhow!("serialization failed: {err}"))?, + )) + }) + .collect(); + cells + } + + #[cfg(feature = "image")] + _ => { + use re_types::Archetype; + let indicator = ::Indicator::batch(1); + let indicator_cell = DataCell::from_arrow( + re_types::archetypes::Image::indicator_component(), + indicator.to_arrow(), + ); + + // Assume an image (there are so many image extensions): + let tensor = re_types::components::TensorData( + re_types::datatypes::TensorData::from_image_file(file_path)?, + ); + Ok(vec![ + indicator_cell, + DataCell::try_from_native(std::iter::once(&tensor))?, + ]) + } + + #[cfg(not(feature = "image"))] + _ => Err(FromFileError::UnknownExtension { + extension, + path: file_path.to_owned(), + }), + } +} + +pub fn data_cells_from_file_contents( + file_name: &str, + bytes: Vec, +) -> Result, FromFileError> { + re_tracing::profile_function!(file_name); + + let extension = std::path::Path::new(file_name) + .extension() + .unwrap_or_default() + .to_ascii_lowercase() + .to_string_lossy() + .to_string(); + + match extension.as_str() { + "glb" | "gltf" | "obj" => { + use re_types::{archetypes::Asset3D, components::MediaType, Archetype}; + let cells: Result, _> = + Asset3D::from_bytes(bytes, MediaType::guess_from_path(file_name)) + .as_component_batches() + .into_iter() + .map(|comp_batch| { + let comp_batch = comp_batch.as_ref(); + Ok(DataCell::from_arrow( + comp_batch.name(), + comp_batch + .try_to_arrow() + .map_err(|err| anyhow::anyhow!("serialization failed: {err}"))?, + )) + }) + .collect(); + cells + } + + #[cfg(feature = "image")] + _ => { + let format = if let Some(format) = image::ImageFormat::from_extension(extension) { + format + } else { + image::guess_format(&bytes) + .map_err(re_types::tensor_data::TensorImageLoadError::from)? + }; + + use re_types::Archetype; + let indicator = ::Indicator::batch(1); + let indicator_cell = DataCell::from_arrow( + re_types::archetypes::Image::indicator_component(), + indicator.to_arrow(), + ); + + // Assume an image (there are so many image extensions): + let tensor = re_types::components::TensorData( + re_types::datatypes::TensorData::from_image_bytes(bytes, format)?, + ); + Ok(vec![ + indicator_cell, + DataCell::try_from_native(std::iter::once(&tensor))?, + ]) + } + + #[cfg(not(feature = "image"))] + _ => Err(FromFileError::UnknownExtension { + extension, + path: file_name.to_owned().into(), + }), + } +} diff --git a/crates/re_query/Cargo.toml b/crates/re_query/Cargo.toml index 4f8678ad15e7..1c130c98824b 100644 --- a/crates/re_query/Cargo.toml +++ b/crates/re_query/Cargo.toml @@ -26,11 +26,10 @@ polars = ["dep:polars-core", "re_arrow_store/polars"] [dependencies] # Rerun dependencies: re_arrow_store.workspace = true -re_components = { workspace = true, features = ["arrow_datagen"] } re_data_store.workspace = true re_format.workspace = true re_log_types.workspace = true -re_types.workspace = true +re_types = { workspace = true, features = ["datagen"] } re_log.workspace = true re_tracing.workspace = true @@ -75,7 +74,3 @@ required-features = ["polars"] [[bench]] name = "query_benchmark" harness = false - -[[bench]] -name = "query_benchmark2" -harness = false diff --git a/crates/re_query/benches/query_benchmark.rs b/crates/re_query/benches/query_benchmark.rs index 57e6917e7260..826b966e35c6 100644 --- a/crates/re_query/benches/query_benchmark.rs +++ b/crates/re_query/benches/query_benchmark.rs @@ -5,14 +5,11 @@ use criterion::{criterion_group, criterion_main, Criterion}; use itertools::Itertools; use re_arrow_store::{DataStore, LatestAtQuery}; -use re_components::{ - datagen::{build_frame_nr, build_some_colors, build_some_positions2d, build_some_vec3d}, - LegacyVec3D, -}; -use re_log_types::{entity_path, DataRow, EntityPath, Index, RowId, TimeType, Timeline}; -use re_query::query_entity_with_primary; +use re_log_types::{entity_path, DataRow, EntityPath, Index, RowId, TimeInt, TimeType, Timeline}; +use re_query::query_archetype; use re_types::{ - components::{Color, InstanceKey, Position2D}, + archetypes::Points2D, + components::{Color, InstanceKey, Position2D, Text}, Loggable as _, }; @@ -23,9 +20,9 @@ const NUM_FRAMES_POINTS: u32 = 1_000; #[cfg(not(debug_assertions))] const NUM_POINTS: u32 = 1_000; #[cfg(not(debug_assertions))] -const NUM_FRAMES_VECS: u32 = 10; +const NUM_FRAMES_STRINGS: u32 = 1_000; #[cfg(not(debug_assertions))] -const NUM_VECS: u32 = 100_000; +const NUM_STRINGS: u32 = 1_000; // `cargo test` also runs the benchmark setup code, so make sure they run quickly: #[cfg(debug_assertions)] @@ -33,15 +30,57 @@ const NUM_FRAMES_POINTS: u32 = 1; #[cfg(debug_assertions)] const NUM_POINTS: u32 = 1; #[cfg(debug_assertions)] -const NUM_FRAMES_VECS: u32 = 1; +const NUM_FRAMES_STRINGS: u32 = 1; #[cfg(debug_assertions)] -const NUM_VECS: u32 = 1; +const NUM_STRINGS: u32 = 1; -criterion_group!(benches, mono_points, batch_points, batch_vecs); +criterion_group!( + benches, + mono_points, + mono_strings, + batch_points, + batch_strings +); criterion_main!(benches); // --- Benchmarks --- +pub fn build_some_point2d(len: usize) -> Vec { + use rand::Rng as _; + let mut rng = rand::thread_rng(); + + (0..len) + .map(|_| Position2D::new(rng.gen_range(0.0..10.0), rng.gen_range(0.0..10.0))) + .collect() +} + +/// Create `len` dummy colors +pub fn build_some_colors(len: usize) -> Vec { + (0..len).map(|i| Color::from(i as u32)).collect() +} + +/// Build a ([`Timeline`], [`TimeInt`]) tuple from `frame_nr` suitable for inserting in a [`re_log_types::TimePoint`]. +pub fn build_frame_nr(frame_nr: TimeInt) -> (Timeline, TimeInt) { + (Timeline::new("frame_nr", TimeType::Sequence), frame_nr) +} + +pub fn build_some_strings(len: usize) -> Vec { + use rand::Rng as _; + let mut rng = rand::thread_rng(); + + (0..len) + .map(|_| { + let ilen: usize = rng.gen_range(0..10000); + let s: String = rand::thread_rng() + .sample_iter(&rand::distributions::Alphanumeric) + .take(ilen) + .map(char::from) + .collect(); + Text::from(s) + }) + .collect() +} + fn mono_points(c: &mut Criterion) { // Each mono point gets logged at a different path let paths = (0..NUM_POINTS) @@ -50,7 +89,7 @@ fn mono_points(c: &mut Criterion) { let msgs = build_points_rows(&paths, 1); { - let mut group = c.benchmark_group("arrow_mono_points"); + let mut group = c.benchmark_group("arrow_mono_points2"); // Mono-insert is slow -- decrease the sample size group.sample_size(10); group.throughput(criterion::Throughput::Elements( @@ -62,7 +101,7 @@ fn mono_points(c: &mut Criterion) { } { - let mut group = c.benchmark_group("arrow_mono_points"); + let mut group = c.benchmark_group("arrow_mono_points2"); group.throughput(criterion::Throughput::Elements(NUM_POINTS as _)); let mut store = insert_rows(msgs.iter()); group.bench_function("query", |b| { @@ -71,13 +110,41 @@ fn mono_points(c: &mut Criterion) { } } +fn mono_strings(c: &mut Criterion) { + // Each mono string gets logged at a different path + let paths = (0..NUM_STRINGS) + .map(move |string_idx| entity_path!("strings", Index::Sequence(string_idx as _))) + .collect_vec(); + let msgs = build_strings_rows(&paths, 1); + + { + let mut group = c.benchmark_group("arrow_mono_strings2"); + group.sample_size(10); + group.throughput(criterion::Throughput::Elements( + (NUM_STRINGS * NUM_FRAMES_STRINGS) as _, + )); + group.bench_function("insert", |b| { + b.iter(|| insert_rows(msgs.iter())); + }); + } + + { + let mut group = c.benchmark_group("arrow_mono_strings2"); + group.throughput(criterion::Throughput::Elements(NUM_POINTS as _)); + let mut store = insert_rows(msgs.iter()); + group.bench_function("query", |b| { + b.iter(|| query_and_visit_strings(&mut store, &paths)); + }); + } +} + fn batch_points(c: &mut Criterion) { // Batch points are logged together at a single path let paths = [EntityPath::from("points")]; let msgs = build_points_rows(&paths, NUM_POINTS as _); { - let mut group = c.benchmark_group("arrow_batch_points"); + let mut group = c.benchmark_group("arrow_batch_points2"); group.throughput(criterion::Throughput::Elements( (NUM_POINTS * NUM_FRAMES_POINTS) as _, )); @@ -87,7 +154,7 @@ fn batch_points(c: &mut Criterion) { } { - let mut group = c.benchmark_group("arrow_batch_points"); + let mut group = c.benchmark_group("arrow_batch_points2"); group.throughput(criterion::Throughput::Elements(NUM_POINTS as _)); let mut store = insert_rows(msgs.iter()); group.bench_function("query", |b| { @@ -96,15 +163,15 @@ fn batch_points(c: &mut Criterion) { } } -fn batch_vecs(c: &mut Criterion) { - // Batch points are logged together at a single path - let paths = [EntityPath::from("vec")]; - let msgs = build_vecs_rows(&paths, NUM_VECS as _); +fn batch_strings(c: &mut Criterion) { + // Batch strings are logged together at a single path + let paths = [EntityPath::from("points")]; + let msgs = build_strings_rows(&paths, NUM_STRINGS as _); { - let mut group = c.benchmark_group("arrow_batch_vecs"); + let mut group = c.benchmark_group("arrow_batch_strings2"); group.throughput(criterion::Throughput::Elements( - (NUM_VECS * NUM_FRAMES_VECS) as _, + (NUM_STRINGS * NUM_FRAMES_STRINGS) as _, )); group.bench_function("insert", |b| { b.iter(|| insert_rows(msgs.iter())); @@ -112,11 +179,11 @@ fn batch_vecs(c: &mut Criterion) { } { - let mut group = c.benchmark_group("arrow_batch_vecs"); - group.throughput(criterion::Throughput::Elements(NUM_VECS as _)); + let mut group = c.benchmark_group("arrow_batch_strings2"); + group.throughput(criterion::Throughput::Elements(NUM_POINTS as _)); let mut store = insert_rows(msgs.iter()); group.bench_function("query", |b| { - b.iter(|| query_and_visit_vecs(&mut store, &paths)); + b.iter(|| query_and_visit_strings(&mut store, &paths)); }); } } @@ -132,7 +199,7 @@ fn build_points_rows(paths: &[EntityPath], pts: usize) -> Vec { path.clone(), [build_frame_nr((frame_idx as i64).into())], pts as _, - (build_some_positions2d(pts), build_some_colors(pts)), + (build_some_point2d(pts), build_some_colors(pts)), ) .unwrap(); // NOTE: Using unsized cells will crash in debug mode, and benchmarks are run for 1 iteration, @@ -146,16 +213,24 @@ fn build_points_rows(paths: &[EntityPath], pts: usize) -> Vec { .collect() } -fn build_vecs_rows(paths: &[EntityPath], pts: usize) -> Vec { - (0..NUM_FRAMES_VECS) +fn build_strings_rows(paths: &[EntityPath], num_strings: usize) -> Vec { + (0..NUM_FRAMES_STRINGS) .flat_map(move |frame_idx| { paths.iter().map(move |path| { - let mut row = DataRow::from_cells1( + let mut row = DataRow::from_cells2( RowId::ZERO, path.clone(), [build_frame_nr((frame_idx as i64).into())], - pts as _, - build_some_vec3d(pts), + num_strings as _, + // We still need to create points because they are the primary for the + // archetype query we want to do. We won't actually deserialize the points + // during the query -- we just need it for the primary keys. + // TODO(jleibs): switch this to use `TextEntry` once the new type has + // landed. + ( + build_some_point2d(num_strings), + build_some_strings(num_strings), + ), ) .unwrap(); // NOTE: Using unsized cells will crash in debug mode, and benchmarks are run for 1 iteration, @@ -188,42 +263,42 @@ fn query_and_visit_points(store: &mut DataStore, paths: &[EntityPath]) -> Vec(store, &query, path, &[Color::name()]) - .and_then(|entity_view| { - entity_view.visit2(|_: InstanceKey, pos: Position2D, color: Option| { - points.push(SavePoint { - _pos: pos, - _color: color, - }); - }) - }) - .ok() - .unwrap(); + let arch_view = query_archetype::(store, &query, path).unwrap(); + itertools::izip!( + arch_view.iter_required_component::().unwrap(), + arch_view.iter_optional_component::().unwrap() + ) + .for_each(|(pos, color)| { + points.push(SavePoint { + _pos: pos, + _color: color, + }); + }); } assert_eq!(NUM_POINTS as usize, points.len()); points } -struct SaveVec { - _vec: LegacyVec3D, +struct SaveString { + _label: Option, } -fn query_and_visit_vecs(store: &mut DataStore, paths: &[EntityPath]) -> Vec { +fn query_and_visit_strings(store: &mut DataStore, paths: &[EntityPath]) -> Vec { let timeline_frame_nr = Timeline::new("frame_nr", TimeType::Sequence); - let query = LatestAtQuery::new(timeline_frame_nr, (NUM_FRAMES_POINTS as i64 / 2).into()); + let query = LatestAtQuery::new(timeline_frame_nr, (NUM_FRAMES_STRINGS as i64 / 2).into()); - let mut rects = Vec::with_capacity(NUM_VECS as _); + let mut strings = Vec::with_capacity(NUM_STRINGS as _); for path in paths { - query_entity_with_primary::(store, &query, path, &[]) - .and_then(|entity_view| { - entity_view.visit1(|_: InstanceKey, vec: LegacyVec3D| { - rects.push(SaveVec { _vec: vec }); - }) - }) - .ok() - .unwrap(); + let arch_view = query_archetype::(store, &query, path).unwrap(); + arch_view + .iter_optional_component::() + .unwrap() + .for_each(|label| { + strings.push(SaveString { _label: label }); + }); } - assert_eq!(NUM_VECS as usize, rects.len()); - rects + assert_eq!(NUM_STRINGS as usize, strings.len()); + + criterion::black_box(strings) } diff --git a/crates/re_query/benches/query_benchmark2.rs b/crates/re_query/benches/query_benchmark2.rs deleted file mode 100644 index 826b966e35c6..000000000000 --- a/crates/re_query/benches/query_benchmark2.rs +++ /dev/null @@ -1,304 +0,0 @@ -#[global_allocator] -static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; - -use criterion::{criterion_group, criterion_main, Criterion}; - -use itertools::Itertools; -use re_arrow_store::{DataStore, LatestAtQuery}; -use re_log_types::{entity_path, DataRow, EntityPath, Index, RowId, TimeInt, TimeType, Timeline}; -use re_query::query_archetype; -use re_types::{ - archetypes::Points2D, - components::{Color, InstanceKey, Position2D, Text}, - Loggable as _, -}; - -// --- - -#[cfg(not(debug_assertions))] -const NUM_FRAMES_POINTS: u32 = 1_000; -#[cfg(not(debug_assertions))] -const NUM_POINTS: u32 = 1_000; -#[cfg(not(debug_assertions))] -const NUM_FRAMES_STRINGS: u32 = 1_000; -#[cfg(not(debug_assertions))] -const NUM_STRINGS: u32 = 1_000; - -// `cargo test` also runs the benchmark setup code, so make sure they run quickly: -#[cfg(debug_assertions)] -const NUM_FRAMES_POINTS: u32 = 1; -#[cfg(debug_assertions)] -const NUM_POINTS: u32 = 1; -#[cfg(debug_assertions)] -const NUM_FRAMES_STRINGS: u32 = 1; -#[cfg(debug_assertions)] -const NUM_STRINGS: u32 = 1; - -criterion_group!( - benches, - mono_points, - mono_strings, - batch_points, - batch_strings -); -criterion_main!(benches); - -// --- Benchmarks --- - -pub fn build_some_point2d(len: usize) -> Vec { - use rand::Rng as _; - let mut rng = rand::thread_rng(); - - (0..len) - .map(|_| Position2D::new(rng.gen_range(0.0..10.0), rng.gen_range(0.0..10.0))) - .collect() -} - -/// Create `len` dummy colors -pub fn build_some_colors(len: usize) -> Vec { - (0..len).map(|i| Color::from(i as u32)).collect() -} - -/// Build a ([`Timeline`], [`TimeInt`]) tuple from `frame_nr` suitable for inserting in a [`re_log_types::TimePoint`]. -pub fn build_frame_nr(frame_nr: TimeInt) -> (Timeline, TimeInt) { - (Timeline::new("frame_nr", TimeType::Sequence), frame_nr) -} - -pub fn build_some_strings(len: usize) -> Vec { - use rand::Rng as _; - let mut rng = rand::thread_rng(); - - (0..len) - .map(|_| { - let ilen: usize = rng.gen_range(0..10000); - let s: String = rand::thread_rng() - .sample_iter(&rand::distributions::Alphanumeric) - .take(ilen) - .map(char::from) - .collect(); - Text::from(s) - }) - .collect() -} - -fn mono_points(c: &mut Criterion) { - // Each mono point gets logged at a different path - let paths = (0..NUM_POINTS) - .map(move |point_idx| entity_path!("points", Index::Sequence(point_idx as _))) - .collect_vec(); - let msgs = build_points_rows(&paths, 1); - - { - let mut group = c.benchmark_group("arrow_mono_points2"); - // Mono-insert is slow -- decrease the sample size - group.sample_size(10); - group.throughput(criterion::Throughput::Elements( - (NUM_POINTS * NUM_FRAMES_POINTS) as _, - )); - group.bench_function("insert", |b| { - b.iter(|| insert_rows(msgs.iter())); - }); - } - - { - let mut group = c.benchmark_group("arrow_mono_points2"); - group.throughput(criterion::Throughput::Elements(NUM_POINTS as _)); - let mut store = insert_rows(msgs.iter()); - group.bench_function("query", |b| { - b.iter(|| query_and_visit_points(&mut store, &paths)); - }); - } -} - -fn mono_strings(c: &mut Criterion) { - // Each mono string gets logged at a different path - let paths = (0..NUM_STRINGS) - .map(move |string_idx| entity_path!("strings", Index::Sequence(string_idx as _))) - .collect_vec(); - let msgs = build_strings_rows(&paths, 1); - - { - let mut group = c.benchmark_group("arrow_mono_strings2"); - group.sample_size(10); - group.throughput(criterion::Throughput::Elements( - (NUM_STRINGS * NUM_FRAMES_STRINGS) as _, - )); - group.bench_function("insert", |b| { - b.iter(|| insert_rows(msgs.iter())); - }); - } - - { - let mut group = c.benchmark_group("arrow_mono_strings2"); - group.throughput(criterion::Throughput::Elements(NUM_POINTS as _)); - let mut store = insert_rows(msgs.iter()); - group.bench_function("query", |b| { - b.iter(|| query_and_visit_strings(&mut store, &paths)); - }); - } -} - -fn batch_points(c: &mut Criterion) { - // Batch points are logged together at a single path - let paths = [EntityPath::from("points")]; - let msgs = build_points_rows(&paths, NUM_POINTS as _); - - { - let mut group = c.benchmark_group("arrow_batch_points2"); - group.throughput(criterion::Throughput::Elements( - (NUM_POINTS * NUM_FRAMES_POINTS) as _, - )); - group.bench_function("insert", |b| { - b.iter(|| insert_rows(msgs.iter())); - }); - } - - { - let mut group = c.benchmark_group("arrow_batch_points2"); - group.throughput(criterion::Throughput::Elements(NUM_POINTS as _)); - let mut store = insert_rows(msgs.iter()); - group.bench_function("query", |b| { - b.iter(|| query_and_visit_points(&mut store, &paths)); - }); - } -} - -fn batch_strings(c: &mut Criterion) { - // Batch strings are logged together at a single path - let paths = [EntityPath::from("points")]; - let msgs = build_strings_rows(&paths, NUM_STRINGS as _); - - { - let mut group = c.benchmark_group("arrow_batch_strings2"); - group.throughput(criterion::Throughput::Elements( - (NUM_STRINGS * NUM_FRAMES_STRINGS) as _, - )); - group.bench_function("insert", |b| { - b.iter(|| insert_rows(msgs.iter())); - }); - } - - { - let mut group = c.benchmark_group("arrow_batch_strings2"); - group.throughput(criterion::Throughput::Elements(NUM_POINTS as _)); - let mut store = insert_rows(msgs.iter()); - group.bench_function("query", |b| { - b.iter(|| query_and_visit_strings(&mut store, &paths)); - }); - } -} - -// --- Helpers --- - -fn build_points_rows(paths: &[EntityPath], pts: usize) -> Vec { - (0..NUM_FRAMES_POINTS) - .flat_map(move |frame_idx| { - paths.iter().map(move |path| { - let mut row = DataRow::from_cells2( - RowId::ZERO, - path.clone(), - [build_frame_nr((frame_idx as i64).into())], - pts as _, - (build_some_point2d(pts), build_some_colors(pts)), - ) - .unwrap(); - // NOTE: Using unsized cells will crash in debug mode, and benchmarks are run for 1 iteration, - // in debug mode, by the standard test harness. - if cfg!(debug_assertions) { - row.compute_all_size_bytes(); - } - row - }) - }) - .collect() -} - -fn build_strings_rows(paths: &[EntityPath], num_strings: usize) -> Vec { - (0..NUM_FRAMES_STRINGS) - .flat_map(move |frame_idx| { - paths.iter().map(move |path| { - let mut row = DataRow::from_cells2( - RowId::ZERO, - path.clone(), - [build_frame_nr((frame_idx as i64).into())], - num_strings as _, - // We still need to create points because they are the primary for the - // archetype query we want to do. We won't actually deserialize the points - // during the query -- we just need it for the primary keys. - // TODO(jleibs): switch this to use `TextEntry` once the new type has - // landed. - ( - build_some_point2d(num_strings), - build_some_strings(num_strings), - ), - ) - .unwrap(); - // NOTE: Using unsized cells will crash in debug mode, and benchmarks are run for 1 iteration, - // in debug mode, by the standard test harness. - if cfg!(debug_assertions) { - row.compute_all_size_bytes(); - } - row - }) - }) - .collect() -} - -fn insert_rows<'a>(msgs: impl Iterator) -> DataStore { - let mut store = DataStore::new(InstanceKey::name(), Default::default()); - msgs.for_each(|row| store.insert_row(row).unwrap()); - store -} - -struct SavePoint { - _pos: Position2D, - _color: Option, -} - -fn query_and_visit_points(store: &mut DataStore, paths: &[EntityPath]) -> Vec { - let timeline_frame_nr = Timeline::new("frame_nr", TimeType::Sequence); - let query = LatestAtQuery::new(timeline_frame_nr, (NUM_FRAMES_POINTS as i64 / 2).into()); - - let mut points = Vec::with_capacity(NUM_POINTS as _); - - // TODO(jleibs): Add Radius once we have support for it in field_types - for path in paths { - let arch_view = query_archetype::(store, &query, path).unwrap(); - itertools::izip!( - arch_view.iter_required_component::().unwrap(), - arch_view.iter_optional_component::().unwrap() - ) - .for_each(|(pos, color)| { - points.push(SavePoint { - _pos: pos, - _color: color, - }); - }); - } - assert_eq!(NUM_POINTS as usize, points.len()); - points -} - -struct SaveString { - _label: Option, -} - -fn query_and_visit_strings(store: &mut DataStore, paths: &[EntityPath]) -> Vec { - let timeline_frame_nr = Timeline::new("frame_nr", TimeType::Sequence); - let query = LatestAtQuery::new(timeline_frame_nr, (NUM_FRAMES_STRINGS as i64 / 2).into()); - - let mut strings = Vec::with_capacity(NUM_STRINGS as _); - - for path in paths { - let arch_view = query_archetype::(store, &query, path).unwrap(); - arch_view - .iter_optional_component::() - .unwrap() - .for_each(|label| { - strings.push(SaveString { _label: label }); - }); - } - assert_eq!(NUM_STRINGS as usize, strings.len()); - - criterion::black_box(strings) -} diff --git a/crates/re_query/examples/range.rs b/crates/re_query/examples/range.rs index 872c63dc7c5e..7454ff946925 100644 --- a/crates/re_query/examples/range.rs +++ b/crates/re_query/examples/range.rs @@ -5,9 +5,9 @@ //! ``` use re_arrow_store::{DataStore, RangeQuery, TimeRange}; -use re_components::datagen::{build_frame_nr, build_some_colors, build_some_positions2d}; -use re_log_types::{DataRow, EntityPath, RowId, TimeType}; +use re_log_types::{build_frame_nr, DataRow, EntityPath, RowId, TimeType}; use re_query::range_entity_with_primary; +use re_types::datagen::{build_some_colors, build_some_positions2d}; use re_types::{ components::{Color, InstanceKey, Position2D}, Loggable as _, diff --git a/crates/re_query/src/query.rs b/crates/re_query/src/query.rs index e5d9cd5e6d4c..a93d356ebebf 100644 --- a/crates/re_query/src/query.rs +++ b/crates/re_query/src/query.rs @@ -246,7 +246,7 @@ pub fn query_archetype( /// Helper used to create an example store we can use for querying in doctests pub fn __populate_example_store() -> DataStore { - use re_components::datagen::build_frame_nr; + use re_log_types::build_frame_nr; use re_types::components::{Color, Position2D}; let mut store = DataStore::new(InstanceKey::name(), Default::default()); diff --git a/crates/re_query/tests/archetype_query_tests.rs b/crates/re_query/tests/archetype_query_tests.rs index 3b571940b16c..f2e6ba7a9629 100644 --- a/crates/re_query/tests/archetype_query_tests.rs +++ b/crates/re_query/tests/archetype_query_tests.rs @@ -1,8 +1,7 @@ mod common; use re_arrow_store::DataStore; -use re_components::datagen::build_frame_nr; -use re_log_types::{DataRow, RowId}; +use re_log_types::{build_frame_nr, DataRow, RowId}; use re_query::query_archetype; use re_types::{ archetypes::Points2D, diff --git a/crates/re_query/tests/archetype_range_tests.rs b/crates/re_query/tests/archetype_range_tests.rs index 0488711d2590..ed7195a61dcd 100644 --- a/crates/re_query/tests/archetype_range_tests.rs +++ b/crates/re_query/tests/archetype_range_tests.rs @@ -1,8 +1,7 @@ mod common; use re_arrow_store::{DataStore, TimeInt, TimeRange}; -use re_components::datagen::build_frame_nr; -use re_log_types::{DataRow, EntityPath, RowId}; +use re_log_types::{build_frame_nr, DataRow, EntityPath, RowId}; use re_query::range_archetype; use re_types::{ archetypes::Points2D, diff --git a/crates/re_query/tests/query_tests.rs b/crates/re_query/tests/query_tests.rs index 015c0bd9beed..4d47a01bf735 100644 --- a/crates/re_query/tests/query_tests.rs +++ b/crates/re_query/tests/query_tests.rs @@ -1,8 +1,7 @@ mod common; use re_arrow_store::DataStore; -use re_components::datagen::build_frame_nr; -use re_log_types::{DataRow, RowId}; +use re_log_types::{build_frame_nr, DataRow, RowId}; use re_query::query_entity_with_primary; use re_types::{ components::{Color, InstanceKey, Position2D}, diff --git a/crates/re_query/tests/range_tests.rs b/crates/re_query/tests/range_tests.rs index 909847bdc1f8..bd837efc97a9 100644 --- a/crates/re_query/tests/range_tests.rs +++ b/crates/re_query/tests/range_tests.rs @@ -1,8 +1,7 @@ mod common; use re_arrow_store::{DataStore, TimeInt, TimeRange}; -use re_components::datagen::build_frame_nr; -use re_log_types::{DataRow, EntityPath, RowId}; +use re_log_types::{build_frame_nr, DataRow, EntityPath, RowId}; use re_query::range_entity_with_primary; use re_types::{ components::{Color, InstanceKey, Position2D}, diff --git a/crates/re_sdk/Cargo.toml b/crates/re_sdk/Cargo.toml index e4373a749a99..77c42410f903 100644 --- a/crates/re_sdk/Cargo.toml +++ b/crates/re_sdk/Cargo.toml @@ -25,10 +25,10 @@ default = ["demo", "glam", "image"] demo = [] ## Add support for some math operations using [`glam`](https://crates.io/crates/glam/). -glam = ["re_components/glam"] +glam = ["re_types/glam"] ## Integration with the [`image`](https://crates.io/crates/image/) crate, plus JPEG support.. -image = ["re_components/image"] +image = ["re_types/image"] ## Support serving a web viewer over HTTP. ## @@ -53,7 +53,6 @@ web_viewer = [ [dependencies] re_build_info.workspace = true -re_components.workspace = true re_log_encoding = { workspace = true, features = ["encoder"] } re_log_types.workspace = true re_log.workspace = true @@ -80,7 +79,7 @@ webbrowser = { version = "0.8", optional = true } [dev-dependencies] -re_components = { workspace = true, features = ["arrow_datagen"] } +re_types = { workspace = true, features = ["datagen"] } arrow2_convert.workspace = true ndarray.workspace = true diff --git a/crates/re_sdk/src/lib.rs b/crates/re_sdk/src/lib.rs index a034ad336404..2774c3b0455a 100644 --- a/crates/re_sdk/src/lib.rs +++ b/crates/re_sdk/src/lib.rs @@ -69,7 +69,6 @@ pub mod time { /// They all implement the [`Component`][`re_types::Component`] trait, /// and can be used in [`RecordingStream::log_component_batches`]. pub mod components { - pub use re_components::Quaternion; pub use re_types::components::{ AnnotationContext, Blob, ClassId, ClearIsRecursive, Color, DepthMeter, DisconnectedSpace, DrawOrder, HalfSizes2D, HalfSizes3D, InstanceKey, KeypointId, LineStrip2D, LineStrip3D, @@ -113,12 +112,6 @@ pub mod external { pub use re_log::external::*; pub use re_log_types::external::*; pub use re_types::external::*; - - #[cfg(feature = "glam")] - pub use re_components::external::glam; - - #[cfg(feature = "image")] - pub use re_components::external::image; } // ----- diff --git a/crates/re_sdk/src/recording_stream.rs b/crates/re_sdk/src/recording_stream.rs index 5000238f8a84..a93f944a5b04 100644 --- a/crates/re_sdk/src/recording_stream.rs +++ b/crates/re_sdk/src/recording_stream.rs @@ -1357,8 +1357,7 @@ impl RecordingStream { #[cfg(test)] mod tests { - use re_components::datagen::data_table_example; - use re_log_types::RowId; + use re_log_types::{DataTable, RowId}; use super::*; @@ -1378,7 +1377,7 @@ mod tests { let store_info = rec.store_info().cloned().unwrap(); - let mut table = data_table_example(false); + let mut table = DataTable::example(false); table.compute_all_size_bytes(); for row in table.to_rows() { rec.record_row(row.unwrap(), false); @@ -1445,7 +1444,7 @@ mod tests { let store_info = rec.store_info().cloned().unwrap(); - let mut table = data_table_example(false); + let mut table = DataTable::example(false); table.compute_all_size_bytes(); for row in table.to_rows() { rec.record_row(row.unwrap(), false); @@ -1525,7 +1524,7 @@ mod tests { let store_info = rec.store_info().cloned().unwrap(); - let mut table = data_table_example(false); + let mut table = DataTable::example(false); table.compute_all_size_bytes(); for row in table.to_rows() { rec.record_row(row.unwrap(), false); @@ -1579,7 +1578,7 @@ mod tests { .memory() .unwrap(); - let mut table = data_table_example(false); + let mut table = DataTable::example(false); table.compute_all_size_bytes(); for row in table.to_rows() { rec.record_row(row.unwrap(), false); diff --git a/crates/re_space_view_bar_chart/Cargo.toml b/crates/re_space_view_bar_chart/Cargo.toml index cea71764caab..9116f622012f 100644 --- a/crates/re_space_view_bar_chart/Cargo.toml +++ b/crates/re_space_view_bar_chart/Cargo.toml @@ -17,7 +17,6 @@ all-features = true [dependencies] re_arrow_store.workspace = true -re_components.workspace = true re_data_store.workspace = true re_log_types.workspace = true re_log.workspace = true diff --git a/crates/re_space_view_spatial/Cargo.toml b/crates/re_space_view_spatial/Cargo.toml index 6cae74ce2ae2..f1e97eee5830 100644 --- a/crates/re_space_view_spatial/Cargo.toml +++ b/crates/re_space_view_spatial/Cargo.toml @@ -17,7 +17,6 @@ all-features = true [dependencies] re_arrow_store.workspace = true -re_components = { workspace = true, features = ["ecolor", "glam", "image"] } re_data_store = { workspace = true, features = ["serde"] } re_data_ui.workspace = true re_error.workspace = true diff --git a/crates/re_space_view_tensor/Cargo.toml b/crates/re_space_view_tensor/Cargo.toml index 0503e74664e4..b9b2f436046e 100644 --- a/crates/re_space_view_tensor/Cargo.toml +++ b/crates/re_space_view_tensor/Cargo.toml @@ -17,7 +17,6 @@ all-features = true [dependencies] re_arrow_store.workspace = true -re_components.workspace = true re_data_store.workspace = true re_data_ui.workspace = true re_log_types.workspace = true diff --git a/crates/re_space_view_text_log/Cargo.toml b/crates/re_space_view_text_log/Cargo.toml index dca137b3bcf9..3f40acb31315 100644 --- a/crates/re_space_view_text_log/Cargo.toml +++ b/crates/re_space_view_text_log/Cargo.toml @@ -17,7 +17,6 @@ all-features = true [dependencies] re_arrow_store.workspace = true -re_components.workspace = true re_data_store.workspace = true re_data_ui.workspace = true re_log_types.workspace = true diff --git a/crates/re_space_view_time_series/Cargo.toml b/crates/re_space_view_time_series/Cargo.toml index 646a70b4e722..8f833ae7602f 100644 --- a/crates/re_space_view_time_series/Cargo.toml +++ b/crates/re_space_view_time_series/Cargo.toml @@ -17,7 +17,6 @@ all-features = true [dependencies] re_arrow_store.workspace = true -re_components.workspace = true re_log_types.workspace = true re_log.workspace = true re_renderer.workspace = true diff --git a/crates/re_types/Cargo.toml b/crates/re_types/Cargo.toml index 3d3ba08c9790..c00a479bfb4c 100644 --- a/crates/re_types/Cargo.toml +++ b/crates/re_types/Cargo.toml @@ -25,6 +25,10 @@ default = [] ## All features except `testing`. all = ["ecolor", "glam", "serde"] +## Enables the `datagen` module, which exposes a number of tools for generating random data for +## tests and benchmarks. +datagen = ["dep:rand"] + ## Enable color conversions. ecolor = ["dep:ecolor"] @@ -75,6 +79,7 @@ uuid = { version = "1.1", features = ["serde", "v4", "js"] } ecolor = { workspace = true, optional = true } glam = { workspace = true, optional = true } image = { workspace = true, optional = true, default-features = false } +rand = { workspace = true, optional = true, features = ["std", "std_rng"] } serde = { version = "1", optional = true, features = ["derive", "rc"] } zune-core = { version = "0.2", optional = true } zune-jpeg = { version = "0.3", optional = true } diff --git a/crates/re_types/src/datagen.rs b/crates/re_types/src/datagen.rs new file mode 100644 index 000000000000..37239d93e51e --- /dev/null +++ b/crates/re_types/src/datagen.rs @@ -0,0 +1,63 @@ +//! Generate random data for tests and benchmarks. + +// TODO(#1810): It really is time for whole module to disappear. + +use crate::components::InstanceKey; + +/// Create `len` dummy colors +pub fn build_some_colors(len: usize) -> Vec { + (0..len) + .map(|i| crate::components::Color::from(i as u32)) + .collect() +} + +/// Create `len` dummy `Position2D` +pub fn build_some_positions2d(len: usize) -> Vec { + use rand::Rng as _; + let mut rng = rand::thread_rng(); + + (0..len) + .map(|_| { + crate::components::Position2D::new(rng.gen_range(0.0..10.0), rng.gen_range(0.0..10.0)) + }) + .collect() +} + +/// Create `len` dummy `Vec3D` +pub fn build_some_vec3d(len: usize) -> Vec { + use rand::Rng as _; + let mut rng = rand::thread_rng(); + + (0..len) + .map(|_| { + crate::datatypes::Vec3D::new( + rng.gen_range(0.0..10.0), + rng.gen_range(0.0..10.0), + rng.gen_range(0.0..10.0), + ) + }) + .collect() +} + +/// Create `len` dummy `InstanceKey` keys. These keys will be sorted. +pub fn build_some_instances(num_instances: usize) -> Vec { + use rand::seq::SliceRandom; + let mut rng = rand::thread_rng(); + + // Allocate pool of 10x the potential instance keys, draw a random sampling, and then sort it + let mut instance_pool = (0..(num_instances * 10)).collect::>(); + let (rand_instances, _) = instance_pool.partial_shuffle(&mut rng, num_instances); + let mut sorted_instances = rand_instances.to_vec(); + sorted_instances.sort(); + + sorted_instances + .into_iter() + .map(|id| InstanceKey(id as u64)) + .collect() +} + +pub fn build_some_instances_from(instances: impl IntoIterator) -> Vec { + let mut instances = instances.into_iter().map(InstanceKey).collect::>(); + instances.sort(); + instances +} diff --git a/crates/re_types/src/lib.rs b/crates/re_types/src/lib.rs index 0da560ab5536..4005980717b1 100644 --- a/crates/re_types/src/lib.rs +++ b/crates/re_types/src/lib.rs @@ -135,6 +135,9 @@ pub use self::result::{ }; pub use self::size_bytes::SizeBytes; +#[cfg(feature = "datagen")] +pub mod datagen; + // --- mod arrow_buffer; @@ -149,6 +152,12 @@ pub mod external { #[cfg(feature = "ecolor")] pub use ecolor; + + #[cfg(feature = "glam")] + pub use glam; + + #[cfg(feature = "image")] + pub use image; } // TODO(jleibs): Should all of this go into `tensor_data_ext`? Don't have a good way to export diff --git a/crates/re_viewer/Cargo.toml b/crates/re_viewer/Cargo.toml index 50d9bffeca95..c64208d5cc45 100644 --- a/crates/re_viewer/Cargo.toml +++ b/crates/re_viewer/Cargo.toml @@ -40,7 +40,6 @@ webgl = ["re_renderer/webgl"] # Internal: re_arrow_store.workspace = true re_build_info.workspace = true -re_components = { workspace = true, features = ["ecolor", "image"] } re_data_source.workspace = true re_data_store = { workspace = true, features = ["serde"] } re_data_ui.workspace = true diff --git a/crates/re_viewer/src/lib.rs b/crates/re_viewer/src/lib.rs index 7af58e6bc255..ebee21a08060 100644 --- a/crates/re_viewer/src/lib.rs +++ b/crates/re_viewer/src/lib.rs @@ -29,9 +29,9 @@ pub mod external { pub use re_data_ui; pub use {eframe, egui}; pub use { - re_arrow_store, re_arrow_store::external::*, re_components, re_data_store, re_log, - re_log_types, re_memory, re_renderer, re_types, re_ui, re_viewer_context, - re_viewer_context::external::*, re_viewport, re_viewport::external::*, + re_arrow_store, re_arrow_store::external::*, re_data_store, re_log, re_log_types, + re_memory, re_renderer, re_types, re_ui, re_viewer_context, re_viewer_context::external::*, + re_viewport, re_viewport::external::*, }; } diff --git a/crates/re_viewer_context/Cargo.toml b/crates/re_viewer_context/Cargo.toml index 1439b7f31025..709c581c1d6f 100644 --- a/crates/re_viewer_context/Cargo.toml +++ b/crates/re_viewer_context/Cargo.toml @@ -17,7 +17,6 @@ all-features = true [dependencies] re_arrow_store.workspace = true -re_components = { workspace = true, features = ["ecolor", "glam", "image"] } re_data_source.workspace = true re_data_store = { workspace = true, features = ["serde"] } re_log_types.workspace = true @@ -26,7 +25,7 @@ re_query.workspace = true re_renderer.workspace = true re_string_interner.workspace = true re_tracing.workspace = true -re_types = { workspace = true, features = ["ecolor"] } +re_types = { workspace = true, features = ["ecolor", "glam", "image"] } re_ui.workspace = true ahash.workspace = true diff --git a/crates/re_viewport/Cargo.toml b/crates/re_viewport/Cargo.toml index f8787066087a..01d71ea527d5 100644 --- a/crates/re_viewport/Cargo.toml +++ b/crates/re_viewport/Cargo.toml @@ -17,7 +17,6 @@ all-features = true [dependencies] re_arrow_store.workspace = true -re_components = { workspace = true, features = ["ecolor", "glam", "image"] } re_data_store = { workspace = true, features = ["serde"] } re_data_ui.workspace = true re_log_types.workspace = true diff --git a/rerun_py/Cargo.toml b/rerun_py/Cargo.toml index 22b45b4b7076..742a62b97a9d 100644 --- a/rerun_py/Cargo.toml +++ b/rerun_py/Cargo.toml @@ -43,7 +43,6 @@ re_build_info.workspace = true re_error.workspace = true re_log.workspace = true re_log_types.workspace = true -re_components = { workspace = true, features = ["image"] } re_memory.workspace = true rerun = { workspace = true, features = ["analytics", "server", "sdk"] } re_web_viewer_server = { workspace = true, optional = true } diff --git a/rerun_py/src/arrow.rs b/rerun_py/src/arrow.rs index 464af4c746a3..421a54734b01 100644 --- a/rerun_py/src/arrow.rs +++ b/rerun_py/src/arrow.rs @@ -3,11 +3,7 @@ use arrow2::{array::Array, datatypes::Field, ffi}; use itertools::Itertools as _; use pyo3::{ - exceptions::{PyAttributeError, PyValueError}, - ffi::Py_uintptr_t, - types::PyDict, - types::{IntoPyDict, PyString}, - PyAny, PyResult, + exceptions::PyValueError, ffi::Py_uintptr_t, types::PyDict, types::PyString, PyAny, PyResult, }; use re_log_types::{DataCell, DataRow, EntityPath, RowId, TimePoint}; @@ -48,27 +44,6 @@ fn array_to_rust(arrow_array: &PyAny, name: Option<&str>) -> PyResult<(Box) -> PyResult<&PyDict> { - let pyarrow = py.import("pyarrow")?; - let pyarrow_field_cls = pyarrow - .dict() - .get_item("Field") - .ok_or_else(|| PyAttributeError::new_err("Module 'pyarrow' has no attribute 'Field'"))?; - - let fields = re_components::iter_registered_field_types() - .map(|field| { - let schema = Box::new(ffi::export_field_to_c(field)); - let schema_ptr = &*schema as *const ffi::ArrowSchema; - pyarrow_field_cls - .call_method1("_import_from_c", (schema_ptr as Py_uintptr_t,)) - .map(|f| (field.name.clone(), f)) - }) - .collect::, _>>()?; - - Ok(fields.into_py_dict(py)) -} - /// Build a [`DataRow`] given a '**kwargs'-style dictionary of component arrays. pub fn build_data_row_from_components( entity_path: &EntityPath, diff --git a/rerun_py/src/python_bridge.rs b/rerun_py/src/python_bridge.rs index 1d06852cec31..2e7d229cd31c 100644 --- a/rerun_py/src/python_bridge.rs +++ b/rerun_py/src/python_bridge.rs @@ -28,7 +28,7 @@ pub use rerun::{ components::{ AnnotationContext, Blob, ClassId, Color, DisconnectedSpace, DrawOrder, InstanceKey, KeypointId, LineStrip2D, LineStrip3D, Origin3D, OutOfTreeTransform3D, PinholeProjection, - Position2D, Position3D, Quaternion, Radius, Text, Transform3D, Vector3D, ViewCoordinates, + Position2D, Position3D, Radius, Text, Transform3D, Vector3D, ViewCoordinates, }, coordinates::{Axis3, Handedness, Sign, SignedAxis3}, datatypes::{AnnotationInfo, ClassDescription}, @@ -39,8 +39,6 @@ use re_web_viewer_server::WebViewerServerPort; #[cfg(feature = "web_viewer")] use re_ws_comms::RerunServerPort; -use crate::arrow::get_registered_component_names; - // --- FFI --- use once_cell::sync::OnceCell; @@ -100,7 +98,6 @@ fn rerun_bindings(_py: Python<'_>, m: &PyModule) -> PyResult<()> { // These two components are necessary for imports to work // TODO(jleibs): Refactor import logic so all we need is main - m.add_function(wrap_pyfunction!(get_registered_component_names, m)?)?; m.add_class::()?; m.add_class::()?; m.add_class::()?; From ce1ef1f2592687b97ad9a14abe516275254d7c2b Mon Sep 17 00:00:00 2001 From: Clement Rey Date: Tue, 26 Sep 2023 10:19:28 +0200 Subject: [PATCH 3/3] lints --- crates/re_arrow_store/src/arrow_util.rs | 2 ++ crates/re_log_types/src/lib.rs | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/re_arrow_store/src/arrow_util.rs b/crates/re_arrow_store/src/arrow_util.rs index b0cb2a7de78f..f644608dcbe9 100644 --- a/crates/re_arrow_store/src/arrow_util.rs +++ b/crates/re_arrow_store/src/arrow_util.rs @@ -232,6 +232,7 @@ fn test_clean_for_polars_nomodify() { let cleaned = cell.as_arrow_ref().clean_for_polars(); assert_eq!(cell.as_arrow_ref(), &*cleaned); + #[cfg(feature = "polars")] crate::polars_util::dataframe_from_cells(&[Some(cell)]).unwrap(); } @@ -258,5 +259,6 @@ fn test_clean_for_polars_modify() { let cleaned = cell.as_arrow_ref().clean_for_polars(); assert_ne!(cell.as_arrow_ref(), &*cleaned); + #[cfg(feature = "polars")] crate::polars_util::dataframe_from_cells(&[Some(cell)]).unwrap(); } diff --git a/crates/re_log_types/src/lib.rs b/crates/re_log_types/src/lib.rs index 4ed9ebb6a641..9f26b858bdb1 100644 --- a/crates/re_log_types/src/lib.rs +++ b/crates/re_log_types/src/lib.rs @@ -498,13 +498,13 @@ macro_rules! component_legacy_shim { // --- -/// Build a ([`Timeline`], [`TimeInt`]) tuple from `log_time` suitable for inserting in a [`re_log_types::TimePoint`]. +/// Build a ([`Timeline`], [`TimeInt`]) tuple from `log_time` suitable for inserting in a [`TimePoint`]. #[inline] pub fn build_log_time(log_time: Time) -> (Timeline, TimeInt) { (Timeline::log_time(), log_time.into()) } -/// Build a ([`Timeline`], [`TimeInt`]) tuple from `frame_nr` suitable for inserting in a [`re_log_types::TimePoint`]. +/// Build a ([`Timeline`], [`TimeInt`]) tuple from `frame_nr` suitable for inserting in a [`TimePoint`]. #[inline] pub fn build_frame_nr(frame_nr: TimeInt) -> (Timeline, TimeInt) { (Timeline::new("frame_nr", TimeType::Sequence), frame_nr)