Merge branch 'cmc/datastore/revamp_msg_id' into cmc/datastore/revamp_…

…logmsg_store

.github/workflows/rust.yml

@@ -137,18 +137,20 @@ jobs:
           command: cranky
           args: --all-targets --all-features -- --deny warnings
 
-      - name: Check no default features
+      # --------------------------------------------------------------------------------
+      # Check a few important permutations of the feature flags for our `rerun` library:
+      - name: Check rerun with `--no-default-features``
         uses: actions-rs/cargo@v1
         with:
-          command: check
-          args: --locked --no-default-features --features __ci --lib
+          command: cranky
+          args: --locked -p rerun --no-default-features
 
-      # Check a few important permutations of the feature flags for our `rerun` library:
-      - name: Check rerun with --features sdk
+      - name: Check rerun with `--features sdk`
         uses: actions-rs/cargo@v1
         with:
-          command: check
-          args: --locked --no-default-features --features sdk
+          command: cranky
+          args: --locked -p rerun --no-default-features --features sdk
+      # --------------------------------------------------------------------------------
 
       - name: Test doc-tests
         uses: actions-rs/cargo@v1

Cargo.toml

@@ -33,6 +33,7 @@ re_error = { path = "crates/re_error", version = "0.4.0" }
 re_format = { path = "crates/re_format", version = "0.4.0" }
 re_int_histogram = { path = "crates/re_int_histogram", version = "0.4.0" }
 re_log = { path = "crates/re_log", version = "0.4.0" }
+re_log_encoding = { path = "crates/re_log_encoding", version = "0.4.0" }
 re_log_types = { path = "crates/re_log_types", version = "0.4.0" }
 re_memory = { path = "crates/re_memory", version = "0.4.0" }
 re_query = { path = "crates/re_query", version = "0.4.0" }

crates/re_arrow_store/src/arrow_util.rs

@@ -51,7 +51,14 @@ impl ArrayExt for dyn Array {
     ///
     /// Nested types are expanded and cleaned recursively
     fn clean_for_polars(&self) -> Box<dyn Array> {
-        match self.data_type() {
+        let datatype = self.data_type();
+        let datatype = if let DataType::Extension(_, inner, _) = datatype {
+            (**inner).clone()
+        } else {
+            datatype.clone()
+        };
+
+        match &datatype {
             DataType::List(field) => {
                 // Recursively clean the contents
                 let typed_arr = self.as_any().downcast_ref::<ListArray<i32>>().unwrap();

crates/re_data_store/Cargo.toml

@@ -26,10 +26,10 @@ serde = ["dep:serde", "re_log_types/serde"]
 [dependencies]
 re_arrow_store.workspace = true
 re_int_histogram.workspace = true
+re_log_encoding = { workspace = true, optional = true }
 re_log_types.workspace = true
 re_log.workspace = true
 re_smart_channel.workspace = true
-re_string_interner.workspace = true
 
 ahash.workspace = true
 document-features = "0.2"
@@ -47,12 +47,12 @@ puffin.workspace = true
 criterion = "0.4"
 mimalloc.workspace = true
 rand = "0.8"
-re_log_types = { workspace = true, features = ["load", "save"] }
+re_log_encoding = { workspace = true, features = ["decoder", "encoder"] }
 
 [lib]
 bench = false
 
 [[example]]
 name = "memory_usage"
 path = "examples/memory_usage.rs"
-required-features = ["re_log_types/load", "re_log_types/save"]
+required-features = ["re_log_encoding/decoder", "re_log_encoding/encoder"]

crates/re_data_store/examples/memory_usage.rs

@@ -48,7 +48,7 @@ fn live_bytes() -> usize {
 
 // ----------------------------------------------------------------------------
 
-use re_log_types::{entity_path, DataRow, RowId};
+use re_log_types::{entity_path, DataRow, RecordingId, RowId};
 
 fn main() {
     log_messages();
@@ -65,12 +65,12 @@ fn log_messages() {
 
     fn encode_log_msg(log_msg: &LogMsg) -> Vec<u8> {
         let mut bytes = vec![];
-        re_log_types::encoding::encode(std::iter::once(log_msg), &mut bytes).unwrap();
+        re_log_encoding::encoder::encode(std::iter::once(log_msg), &mut bytes).unwrap();
         bytes
     }
 
     fn decode_log_msg(mut bytes: &[u8]) -> LogMsg {
-        let mut messages = re_log_types::encoding::Decoder::new(&mut bytes)
+        let mut messages = re_log_encoding::decoder::Decoder::new(&mut bytes)
             .unwrap()
             .collect::<Result<Vec<LogMsg>, _>>()
             .unwrap();
@@ -91,6 +91,7 @@ fn log_messages() {
 
     const NUM_POINTS: usize = 1_000;
 
+    let recording_id = RecordingId::random();
     let timeline = Timeline::new_sequence("frame_nr");
     let mut time_point = TimePoint::default();
     time_point.insert(timeline, TimeInt::from(0));
@@ -116,7 +117,10 @@ fn log_messages() {
             .into_table(),
         );
         let table_bytes = live_bytes() - used_bytes_start;
-        let log_msg = Box::new(LogMsg::ArrowMsg(table.to_arrow_msg().unwrap()));
+        let log_msg = Box::new(LogMsg::ArrowMsg(
+            recording_id,
+            table.to_arrow_msg().unwrap(),
+        ));
         let log_msg_bytes = live_bytes() - used_bytes_start;
         println!("Arrow payload containing a Pos2 uses {table_bytes} bytes in RAM");
         let encoded = encode_log_msg(&log_msg);
@@ -139,7 +143,10 @@ fn log_messages() {
             .into_table(),
         );
         let table_bytes = live_bytes() - used_bytes_start;
-        let log_msg = Box::new(LogMsg::ArrowMsg(table.to_arrow_msg().unwrap()));
+        let log_msg = Box::new(LogMsg::ArrowMsg(
+            recording_id,
+            table.to_arrow_msg().unwrap(),
+        ));
         let log_msg_bytes = live_bytes() - used_bytes_start;
         println!("Arrow payload containing a Pos2 uses {table_bytes} bytes in RAM");
         let encoded = encode_log_msg(&log_msg);

crates/re_data_store/src/log_db.rs

@@ -212,15 +212,15 @@ impl LogDb {
 
         match &msg {
             LogMsg::BeginRecordingMsg(msg) => self.add_begin_recording_msg(msg),
-            LogMsg::EntityPathOpMsg(msg) => {
+            LogMsg::EntityPathOpMsg(_, msg) => {
                 let EntityPathOpMsg {
                     row_id,
                     time_point,
                     path_op,
                 } = msg;
                 self.entity_db.add_path_op(*row_id, time_point, path_op);
             }
-            LogMsg::ArrowMsg(inner) => self.entity_db.try_add_arrow_msg(inner)?,
+            LogMsg::ArrowMsg(_, inner) => self.entity_db.try_add_arrow_msg(inner)?,
             LogMsg::Goodbye(_) => {}
         }
 

crates/re_format/Cargo.toml

@@ -15,7 +15,9 @@ version.workspace = true
 [package.metadata.docs.rs]
 all-features = true
 
-
 [dependencies]
 arrow2.workspace = true
+arrow2_convert.workspace = true
 comfy-table.workspace = true
+parking_lot.workspace = true
+re_tuid.workspace = true