Skip to content

Commit

Permalink
introduce DataCell, retire ComponentBundle
Browse files Browse the repository at this point in the history
  • Loading branch information
@teh-cmc
teh-cmc committed Mar 23, 2023
1 parent 2e64016 commit 60912c1
Show file tree
Hide file tree
Showing 31 changed files with 842 additions and 827 deletions.
1 change: 1 addition & 0 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion crates/re_arrow_store/benches/data_store.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ fn insert_messages<'a>(
msgs: impl Iterator<Item = &'a MsgBundle>,
) -> DataStore {
let mut store = DataStore::new(cluster_key, config);
msgs.for_each(|msg_bundle| store.insert(msg_bundle).unwrap());
msgs.for_each(|msg_bundle| store.insert_row(msg_bundle).unwrap());
store
}

Expand Down
18 changes: 9 additions & 9 deletions crates/re_arrow_store/examples/dump_dataframe.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,48 +29,48 @@ fn main() {
let bundle1 = test_bundle!(ent_path @ [
build_frame_nr(1.into()), build_log_time(Time::now()),
] => [build_some_instances(2), build_some_rects(2)]);
store.insert(&bundle1).unwrap();
store.insert_row(&bundle1).unwrap();
}

for ent_path in &ent_paths {
let bundle2 = test_bundle!(ent_path @ [
build_frame_nr(2.into())
] => [build_some_instances(2), build_some_point2d(2)]);
store.insert(&bundle2).unwrap();
store.insert_row(&bundle2).unwrap();
// Insert timelessly too!
let bundle2 =
test_bundle!(ent_path @ [] => [build_some_instances(2), build_some_point2d(2)]);
store.insert(&bundle2).unwrap();
store.insert_row(&bundle2).unwrap();

let bundle3 = test_bundle!(ent_path @ [
build_frame_nr(3.into()), build_log_time(Time::now()),
] => [build_some_instances_from(25..29), build_some_point2d(4)]);
store.insert(&bundle3).unwrap();
store.insert_row(&bundle3).unwrap();
// Insert timelessly too!
let bundle3 = test_bundle!(ent_path @ [] => [build_some_instances_from(25..29), build_some_point2d(4)]);
store.insert(&bundle3).unwrap();
store.insert_row(&bundle3).unwrap();
}

for ent_path in &ent_paths {
let bundle4_1 = test_bundle!(ent_path @ [
build_frame_nr(4.into()), build_log_time(Time::now()),
] => [build_some_instances_from(20..23), build_some_rects(3)]);
store.insert(&bundle4_1).unwrap();
store.insert_row(&bundle4_1).unwrap();

let bundle4_15 = test_bundle!(ent_path @ [
build_frame_nr(4.into()),
] => [build_some_instances_from(20..23), build_some_point2d(3)]);
store.insert(&bundle4_15).unwrap();
store.insert_row(&bundle4_15).unwrap();

let bundle4_2 = test_bundle!(ent_path @ [
build_frame_nr(4.into()), build_log_time(Time::now()),
] => [build_some_instances_from(25..28), build_some_rects(3)]);
store.insert(&bundle4_2).unwrap();
store.insert_row(&bundle4_2).unwrap();

let bundle4_25 = test_bundle!(ent_path @ [
build_frame_nr(4.into()), build_log_time(Time::now()),
] => [build_some_instances_from(25..28), build_some_point2d(3)]);
store.insert(&bundle4_25).unwrap();
store.insert_row(&bundle4_25).unwrap();
}

let df = store.to_dataframe();
Expand Down
4 changes: 2 additions & 2 deletions crates/re_arrow_store/examples/latest_component.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,10 @@ fn main() {
let ent_path = EntityPath::from("my/entity");

let bundle = test_bundle!(ent_path @ [build_frame_nr(2.into())] => [build_some_rects(4)]);
store.insert(&bundle).unwrap();
store.insert_row(&bundle).unwrap();

let bundle = test_bundle!(ent_path @ [build_frame_nr(3.into())] => [build_some_point2d(2)]);
store.insert(&bundle).unwrap();
store.insert_row(&bundle).unwrap();

let timeline_frame_nr = Timeline::new("frame_nr", TimeType::Sequence);

Expand Down
4 changes: 2 additions & 2 deletions crates/re_arrow_store/examples/latest_components.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,10 @@ fn main() {
let ent_path = EntityPath::from("my/entity");

let bundle = test_bundle!(ent_path @ [build_frame_nr(2.into())] => [build_some_rects(4)]);
store.insert(&bundle).unwrap();
store.insert_row(&bundle).unwrap();

let bundle = test_bundle!(ent_path @ [build_frame_nr(3.into())] => [build_some_point2d(2)]);
store.insert(&bundle).unwrap();
store.insert_row(&bundle).unwrap();

let timeline_frame_nr = Timeline::new("frame_nr", TimeType::Sequence);
let df = latest_components(
Expand Down
14 changes: 7 additions & 7 deletions crates/re_arrow_store/examples/range_components.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,25 +24,25 @@ fn main() {
let frame4 = 4.into();

let bundle = test_bundle!(ent_path @ [build_frame_nr(frame1)] => [build_some_rects(2)]);
store.insert(&bundle).unwrap();
store.insert_row(&bundle).unwrap();

let bundle = test_bundle!(ent_path @ [build_frame_nr(frame2)] => [build_some_point2d(2)]);
store.insert(&bundle).unwrap();
store.insert_row(&bundle).unwrap();

let bundle = test_bundle!(ent_path @ [build_frame_nr(frame3)] => [build_some_point2d(4)]);
store.insert(&bundle).unwrap();
store.insert_row(&bundle).unwrap();

let bundle = test_bundle!(ent_path @ [build_frame_nr(frame4)] => [build_some_rects(3)]);
store.insert(&bundle).unwrap();
store.insert_row(&bundle).unwrap();

let bundle = test_bundle!(ent_path @ [build_frame_nr(frame4)] => [build_some_point2d(1)]);
store.insert(&bundle).unwrap();
store.insert_row(&bundle).unwrap();

let bundle = test_bundle!(ent_path @ [build_frame_nr(frame4)] => [build_some_rects(3)]);
store.insert(&bundle).unwrap();
store.insert_row(&bundle).unwrap();

let bundle = test_bundle!(ent_path @ [build_frame_nr(frame4)] => [build_some_point2d(3)]);
store.insert(&bundle).unwrap();
store.insert_row(&bundle).unwrap();

let timeline_frame_nr = Timeline::new("frame_nr", TimeType::Sequence);
let query = RangeQuery::new(timeline_frame_nr, TimeRange::new(2.into(), 4.into()));
Expand Down
149 changes: 70 additions & 79 deletions crates/re_arrow_store/src/arrow_util.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,103 +262,46 @@ impl ArrayExt for dyn Array {
#[test]
fn test_clean_for_polars_nomodify() {
use re_log_types::datagen::build_some_colors;
use re_log_types::msg_bundle::ComponentBundle;
use re_log_types::DataCell;

// Colors don't need polars cleaning
let bundle: ComponentBundle = build_some_colors(5).try_into().unwrap();
let cleaned = bundle.value_boxed().clean_for_polars();
assert_eq!(bundle.value_boxed(), cleaned);
let cell: DataCell = build_some_colors(5).try_into().unwrap();
let cleaned = cell.as_arrow().clean_for_polars();
assert_eq!(cell.as_arrow(), cleaned);
}

#[test]
fn test_clean_for_polars_modify() {
use re_log_types::msg_bundle::ComponentBundle;
use re_log_types::{Pinhole, Transform};
use re_log_types::{DataCell, Pinhole, Transform};
// transforms are a nice pathological type with both Unions and FixedSizeLists
let transforms = vec![Transform::Pinhole(Pinhole {
image_from_cam: [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]].into(),
resolution: None,
})];

let bundle: ComponentBundle = transforms.try_into().unwrap();
let cell: DataCell = transforms.try_into().unwrap();
assert_eq!(
*bundle.value_boxed().data_type(),
DataType::List(Box::new(Field::new(
"item",
DataType::Union(
vec![
Field::new("Unknown", DataType::Boolean, false),
Field::new(
"Rigid3",
DataType::Struct(vec![
Field::new(
"rotation",
DataType::FixedSizeList(
Box::new(Field::new("item", DataType::Float32, false)),
4
),
false
),
Field::new(
"translation",
DataType::FixedSizeList(
Box::new(Field::new("item", DataType::Float32, false)),
3
),
false
)
]),
false
),
Field::new(
"Pinhole",
DataType::Struct(vec![
Field::new(
"image_from_cam",
DataType::FixedSizeList(
Box::new(Field::new("item", DataType::Float32, false)),
9
),
false,
),
Field::new(
"resolution",
DataType::FixedSizeList(
Box::new(Field::new("item", DataType::Float32, false)),
2
),
true,
),
]),
false
)
],
None,
UnionMode::Dense
),
true
)))
);

let cleaned = bundle.value_boxed().clean_for_polars();

assert_eq!(
*cleaned.data_type(),
DataType::List(Box::new(Field::new(
"item",
DataType::Struct(vec![
*cell.datatype(),
DataType::Union(
vec![
Field::new("Unknown", DataType::Boolean, false),
Field::new(
"Rigid3",
DataType::Struct(vec![
Field::new(
"rotation",
DataType::List(Box::new(Field::new("item", DataType::Float32, false)),),
DataType::FixedSizeList(
Box::new(Field::new("item", DataType::Float32, false)),
4
),
false
),
Field::new(
"translation",
DataType::List(Box::new(Field::new("item", DataType::Float32, false)),),
DataType::FixedSizeList(
Box::new(Field::new("item", DataType::Float32, false)),
3
),
false
)
]),
Expand All @@ -369,19 +312,67 @@ fn test_clean_for_polars_modify() {
DataType::Struct(vec![
Field::new(
"image_from_cam",
DataType::List(Box::new(Field::new("item", DataType::Float32, false))),
DataType::FixedSizeList(
Box::new(Field::new("item", DataType::Float32, false)),
9
),
false,
),
Field::new(
"resolution",
DataType::List(Box::new(Field::new("item", DataType::Float32, false))),
DataType::FixedSizeList(
Box::new(Field::new("item", DataType::Float32, false)),
2
),
true,
),
]),
false
)
],),
true
)))
],
None,
UnionMode::Dense
),
);

let cleaned = cell.as_arrow().clean_for_polars();

assert_eq!(
*cleaned.data_type(),
DataType::Struct(vec![
Field::new("Unknown", DataType::Boolean, false),
Field::new(
"Rigid3",
DataType::Struct(vec![
Field::new(
"rotation",
DataType::List(Box::new(Field::new("item", DataType::Float32, false)),),
false
),
Field::new(
"translation",
DataType::List(Box::new(Field::new("item", DataType::Float32, false)),),
false
)
]),
false
),
Field::new(
"Pinhole",
DataType::Struct(vec![
Field::new(
"image_from_cam",
DataType::List(Box::new(Field::new("item", DataType::Float32, false))),
false,
),
Field::new(
"resolution",
DataType::List(Box::new(Field::new("item", DataType::Float32, false))),
true,
),
]),
false
)
],),
);
}
2 changes: 1 addition & 1 deletion crates/re_arrow_store/src/store.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ pub struct DataStore {

/// Used to cache auto-generated cluster components, i.e. `[0]`, `[0, 1]`, `[0, 1, 2]`, etc
/// so that they can be properly deduplicated.
pub(crate) cluster_comp_cache: IntMap<usize, RowIndex>,
pub(crate) cluster_comp_cache: IntMap<u32, RowIndex>,

/// Dedicated index tables for timeless data. Never garbage collected.
///
Expand Down
Loading

0 comments on commit 60912c1

Please sign in to comment.