Skip to content

Commit

Permalink
introduce DataCell, retire ComponentBundle
Browse files Browse the repository at this point in the history
  • Loading branch information
@teh-cmc
teh-cmc committed Mar 23, 2023
1 parent 2e64016 commit 70e7a65
Show file tree
Hide file tree
Showing 20 changed files with 764 additions and 749 deletions.
1 change: 1 addition & 0 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

149 changes: 70 additions & 79 deletions crates/re_arrow_store/src/arrow_util.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,103 +262,46 @@ impl ArrayExt for dyn Array {
#[test]
fn test_clean_for_polars_nomodify() {
use re_log_types::datagen::build_some_colors;
use re_log_types::msg_bundle::ComponentBundle;
use re_log_types::DataCell;

// Colors don't need polars cleaning
let bundle: ComponentBundle = build_some_colors(5).try_into().unwrap();
let cleaned = bundle.value_boxed().clean_for_polars();
assert_eq!(bundle.value_boxed(), cleaned);
let cell: DataCell = build_some_colors(5).try_into().unwrap();
let cleaned = cell.as_arrow().clean_for_polars();
assert_eq!(cell.as_arrow(), cleaned);
}

#[test]
fn test_clean_for_polars_modify() {
use re_log_types::msg_bundle::ComponentBundle;
use re_log_types::{Pinhole, Transform};
use re_log_types::{DataCell, Pinhole, Transform};
// transforms are a nice pathological type with both Unions and FixedSizeLists
let transforms = vec![Transform::Pinhole(Pinhole {
image_from_cam: [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]].into(),
resolution: None,
})];

let bundle: ComponentBundle = transforms.try_into().unwrap();
let cell: DataCell = transforms.try_into().unwrap();
assert_eq!(
*bundle.value_boxed().data_type(),
DataType::List(Box::new(Field::new(
"item",
DataType::Union(
vec![
Field::new("Unknown", DataType::Boolean, false),
Field::new(
"Rigid3",
DataType::Struct(vec![
Field::new(
"rotation",
DataType::FixedSizeList(
Box::new(Field::new("item", DataType::Float32, false)),
4
),
false
),
Field::new(
"translation",
DataType::FixedSizeList(
Box::new(Field::new("item", DataType::Float32, false)),
3
),
false
)
]),
false
),
Field::new(
"Pinhole",
DataType::Struct(vec![
Field::new(
"image_from_cam",
DataType::FixedSizeList(
Box::new(Field::new("item", DataType::Float32, false)),
9
),
false,
),
Field::new(
"resolution",
DataType::FixedSizeList(
Box::new(Field::new("item", DataType::Float32, false)),
2
),
true,
),
]),
false
)
],
None,
UnionMode::Dense
),
true
)))
);

let cleaned = bundle.value_boxed().clean_for_polars();

assert_eq!(
*cleaned.data_type(),
DataType::List(Box::new(Field::new(
"item",
DataType::Struct(vec![
*cell.datatype(),
DataType::Union(
vec![
Field::new("Unknown", DataType::Boolean, false),
Field::new(
"Rigid3",
DataType::Struct(vec![
Field::new(
"rotation",
DataType::List(Box::new(Field::new("item", DataType::Float32, false)),),
DataType::FixedSizeList(
Box::new(Field::new("item", DataType::Float32, false)),
4
),
false
),
Field::new(
"translation",
DataType::List(Box::new(Field::new("item", DataType::Float32, false)),),
DataType::FixedSizeList(
Box::new(Field::new("item", DataType::Float32, false)),
3
),
false
)
]),
Expand All @@ -369,19 +312,67 @@ fn test_clean_for_polars_modify() {
DataType::Struct(vec![
Field::new(
"image_from_cam",
DataType::List(Box::new(Field::new("item", DataType::Float32, false))),
DataType::FixedSizeList(
Box::new(Field::new("item", DataType::Float32, false)),
9
),
false,
),
Field::new(
"resolution",
DataType::List(Box::new(Field::new("item", DataType::Float32, false))),
DataType::FixedSizeList(
Box::new(Field::new("item", DataType::Float32, false)),
2
),
true,
),
]),
false
)
],),
true
)))
],
None,
UnionMode::Dense
),
);

let cleaned = cell.as_arrow().clean_for_polars();

assert_eq!(
*cleaned.data_type(),
DataType::Struct(vec![
Field::new("Unknown", DataType::Boolean, false),
Field::new(
"Rigid3",
DataType::Struct(vec![
Field::new(
"rotation",
DataType::List(Box::new(Field::new("item", DataType::Float32, false)),),
false
),
Field::new(
"translation",
DataType::List(Box::new(Field::new("item", DataType::Float32, false)),),
false
)
]),
false
),
Field::new(
"Pinhole",
DataType::Struct(vec![
Field::new(
"image_from_cam",
DataType::List(Box::new(Field::new("item", DataType::Float32, false))),
false,
),
Field::new(
"resolution",
DataType::List(Box::new(Field::new("item", DataType::Float32, false))),
true,
),
]),
false
)
],),
);
}
2 changes: 1 addition & 1 deletion crates/re_arrow_store/src/store.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ pub struct DataStore {

/// Used to cache auto-generated cluster components, i.e. `[0]`, `[0, 1]`, `[0, 1, 2]`, etc
/// so that they can be properly deduplicated.
pub(crate) cluster_comp_cache: IntMap<usize, RowIndex>,
pub(crate) cluster_comp_cache: IntMap<u32, RowIndex>,

/// Dedicated index tables for timeless data. Never garbage collected.
///
Expand Down
Loading

0 comments on commit 70e7a65

Please sign in to comment.