SDK batching/revamp 3: sunset PythonSession

Cargo.toml

@@ -57,6 +57,7 @@ arrow2_convert = "0.4.2"
 cfg-if = "1.0"
 clap = "4.0"
 comfy-table = { version = "6.1", default-features = false }
+crossbeam = "0.8"
 ctrlc = { version = "3.0", features = ["termination"] }
 ecolor = "0.21.0"
 eframe = { version = "0.21.3", default-features = false }

crates/re_analytics/Cargo.toml

@@ -23,7 +23,7 @@ re_log.workspace = true
 
 # External dependencies:
 anyhow.workspace = true
-crossbeam = "0.8"
+crossbeam.workspace = true
 once_cell = "1.17"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"

crates/re_arrow_store/src/polars_util.rs

@@ -197,7 +197,7 @@ pub fn range_components<'a, const N: usize>(
 
 // --- Joins ---
 
-// TODO(#1619): none of this mess should be here
+// TODO(#1759): none of this mess should be here
 
 pub fn dataframe_from_cells<const N: usize>(
     cells: &[Option<DataCell>; N],

crates/re_arrow_store/src/store_dump.rs

@@ -1,6 +1,6 @@
-use ahash::HashMapExt;
+use std::collections::BTreeMap;
+
 use arrow2::Either;
-use nohash_hasher::IntMap;
 use re_log_types::{
     DataCellColumn, DataTable, ErasedTimeVec, RowIdVec, TableId, TimeRange, Timeline,
 };
@@ -51,7 +51,7 @@ impl DataStore {
                     .take(table.num_rows() as _)
                     .collect(),
                 col_num_instances: col_num_instances.clone(),
-                columns: columns.clone(), // shallow
+                columns: columns.clone().into_iter().collect(), // shallow
             }
         })
     }
@@ -92,7 +92,7 @@ impl DataStore {
                         .take(col_row_id.len())
                         .collect(),
                     col_num_instances: col_num_instances.clone(),
-                    columns: columns.clone(), // shallow
+                    columns: columns.clone().into_iter().collect(), // shallow
                 }
             })
         })
@@ -163,7 +163,7 @@ impl DataStore {
                     let col_num_instances =
                         filter_column(col_time, col_num_instances.iter(), time_filter).collect();
 
-                    let mut columns2 = IntMap::with_capacity(columns.len());
+                    let mut columns2 = BTreeMap::default();
                     for (component, column) in columns {
                         let column = filter_column(col_time, column.iter(), time_filter).collect();
                         columns2.insert(*component, DataCellColumn(column));

crates/re_arrow_store/src/store_polars.rs

@@ -17,7 +17,7 @@ use crate::{
 };
 
 // TODO(#1692): all of this stuff should be defined by Data{Cell,Row,Table}, not the store.
-// TODO(#1619): remove this and reimplement it on top of store serialization
+// TODO(#1759): remove this and reimplement it on top of store serialization
 
 // ---
 

crates/re_arrow_store/src/store_write.rs

@@ -15,9 +15,8 @@ use crate::{
     IndexedTable, PersistentIndexedTable,
 };
 
-// TODO(#1619):
-// - The store should insert column-per-column rather than row-per-row (purely a performance
-//   matter)
+// TODO(cmc): the store should insert column-per-column rather than row-per-row (purely a
+// performance matter).
 
 // --- Data store ---
 

crates/re_arrow_store/tests/data_store.rs

@@ -24,8 +24,6 @@ use re_log_types::{
     Timeline,
 };
 
-// TODO(#1619): introduce batching in the testing matrix
-
 // --- LatestComponentsAt ---
 
 #[test]

crates/re_data_store/src/log_db.rs

@@ -61,7 +61,7 @@ impl EntityDb {
         let mut table = DataTable::from_arrow_msg(msg)?;
         table.compute_all_size_bytes();
 
-        // TODO(#1619): batch all of this
+        // TODO(cmc): batch all of this
         for row in table.to_rows() {
             self.try_add_data_row(&row)?;
         }

crates/re_log_types/Cargo.toml

@@ -94,8 +94,10 @@ serde_bytes = { version = "0.11", optional = true }
 
 # Native dependencies:
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+crossbeam.workspace = true
 puffin.workspace = true
 
 
 [dev-dependencies]
 rmp-serde = "1.1"
+similar-asserts = "1.4.2"

crates/re_log_types/src/arrow_msg.rs

@@ -11,9 +11,6 @@ use arrow2::{array::Array, chunk::Chunk, datatypes::Schema};
 #[derive(Clone, Debug, PartialEq)]
 pub struct ArrowMsg {
     /// Unique identifier for the [`crate::DataTable`] in this message.
-    ///
-    /// NOTE(#1619): While we're in the process of transitioning towards end-to-end batching, the
-    /// `table_id` is always the same as the `row_id` as the first and only row.
     pub table_id: TableId,
 
     /// The maximum values for all timelines across the entire batch of data.

crates/re_log_types/src/data_cell.rs

@@ -134,7 +134,7 @@ pub struct DataCellInner {
 }
 
 // TODO(cmc): We should be able to build a cell from non-reference types.
-// TODO(#1619): We shouldn't have to specify the component name separately, this should be
+// TODO(#1696): We shouldn't have to specify the component name separately, this should be
 // part of the metadata by using an extension.
 // TODO(#1696): Check that the array is indeed a leaf / component type when building a cell from an
 // arrow payload.
@@ -533,6 +533,9 @@ impl DataCell {
             inner.compute_size_bytes();
             return true;
         }
+
+        re_log::error_once!("cell size could _not_ be computed");
+
         false
     }
 }

crates/re_log_types/src/data_row.rs

@@ -87,6 +87,13 @@ impl std::ops::IndexMut<usize> for DataCellRow {
     }
 }
 
+impl SizeBytes for DataCellRow {
+    #[inline]
+    fn heap_size_bytes(&self) -> u64 {
+        self.0.heap_size_bytes()
+    }
+}
+
 // ---
 
 /// A unique ID for a [`DataRow`].
@@ -120,12 +127,6 @@ impl RowId {
     pub fn random() -> Self {
         Self(re_tuid::Tuid::random())
     }
-
-    /// Temporary utility while we transition to batching. See #1619.
-    #[doc(hidden)]
-    pub fn into_table_id(self) -> TableId {
-        TableId(self.0)
-    }
 }
 
 impl SizeBytes for RowId {
@@ -315,13 +316,28 @@ impl DataRow {
         Self::try_from_cells(row_id, timepoint, entity_path, num_instances, cells).unwrap()
     }
 
-    /// Turns the `DataRow` into a single-row [`DataTable`] that carries the same ID.
-    ///
-    /// This only makes sense as part of our transition to batching. See #1619.
-    #[doc(hidden)]
+    /// Turns the `DataRow` into a single-row [`DataTable`].
     #[inline]
     pub fn into_table(self) -> DataTable {
-        DataTable::from_rows(self.row_id.into_table_id(), [self])
+        DataTable::from_rows(TableId::random(), [self])
+    }
+}
+
+impl SizeBytes for DataRow {
+    fn heap_size_bytes(&self) -> u64 {
+        let Self {
+            row_id,
+            timepoint,
+            entity_path,
+            num_instances,
+            cells,
+        } = self;
+
+        row_id.heap_size_bytes()
+            + timepoint.heap_size_bytes()
+            + entity_path.heap_size_bytes()
+            + num_instances.heap_size_bytes()
+            + cells.heap_size_bytes()
     }
 }
 

crates/re_log_types/src/data_table.rs

@@ -2,7 +2,7 @@ use std::collections::BTreeMap;
 
 use ahash::HashMap;
 use itertools::Itertools as _;
-use nohash_hasher::{IntMap, IntSet};
+use nohash_hasher::IntSet;
 use smallvec::SmallVec;
 
 use crate::{
@@ -156,12 +156,6 @@ impl TableId {
     pub fn random() -> Self {
         Self(re_tuid::Tuid::random())
     }
-
-    /// Temporary utility while we transition to batching. See #1619.
-    #[doc(hidden)]
-    pub fn into_row_id(self) -> RowId {
-        RowId(self.0)
-    }
 }
 
 impl SizeBytes for TableId {
@@ -353,7 +347,7 @@ pub struct DataTable {
     ///
     /// The cells are optional since not all rows will have data for every single component
     /// (i.e. the table is sparse).
-    pub columns: IntMap<ComponentName, DataCellColumn>,
+    pub columns: BTreeMap<ComponentName, DataCellColumn>,
 }
 
 impl DataTable {
@@ -424,7 +418,7 @@ impl DataTable {
         }
 
         // Pre-allocate all columns (one per component).
-        let mut columns = IntMap::default();
+        let mut columns = BTreeMap::default();
         for component in components {
             columns.insert(
                 component,
@@ -441,12 +435,6 @@ impl DataTable {
             }
         }
 
-        if col_row_id.len() > 1 {
-            re_log::warn_once!(
-                "batching features are not ready for use, use single-row data tables instead!"
-            );
-        }
-
         Self {
             table_id,
             col_row_id,