Basic persistence for blueprints

.github/workflows/labels.yml

@@ -29,4 +29,4 @@ jobs:
         with:
           mode: minimum
           count: 1
-          labels: "📊 analytics, 🪳 bug, C/C++ SDK, codegen/idl, 🧑‍💻 dev experience, dependencies, 📖 documentation, 💬 discussion, examples, 📉 performance, 🐍 python API, ⛃ re_datastore, 📺 re_viewer, 🔺 re_renderer, 🚜 refactor, ⛴ release, 🦀 rust SDK, 🔨 testing, ui, 🕸️ web"
+          labels: "📊 analytics, , 🟦 blueprint, 🪳 bug, C/C++ SDK, codegen/idl, 🧑‍💻 dev experience, dependencies, 📖 documentation, 💬 discussion, examples, 📉 performance, 🐍 python API, ⛃ re_datastore, 📺 re_viewer, 🔺 re_renderer, 🚜 refactor, ⛴ release, 🦀 rust SDK, 🔨 testing, ui, 🕸️ web"

crates/re_arrow_store/src/lib.rs

@@ -36,7 +36,7 @@ pub mod polars_util;
 pub mod test_util;
 
 pub use self::arrow_util::ArrayExt;
-pub use self::store::{DataStore, DataStoreConfig};
+pub use self::store::{DataStore, DataStoreConfig, StoreGeneration};
 pub use self::store_gc::GarbageCollectionTarget;
 pub use self::store_read::{LatestAtQuery, RangeQuery};
 pub use self::store_stats::{DataStoreRowStats, DataStoreStats};

crates/re_arrow_store/src/store.rs

@@ -155,6 +155,10 @@ impl std::ops::DerefMut for ClusterCellCache {
 
 // ---
 
+/// Incremented on each edit
+#[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)]
+pub struct StoreGeneration(u64);
+
 /// A complete data store: covers all timelines, all entities, everything.
 ///
 /// ## Debugging
@@ -264,6 +268,12 @@ impl DataStore {
         "rerun.insert_id".into()
     }
 
+    /// Return the current `StoreGeneration`. This can be used to determine whether the
+    /// database has been modified since the last time it was queried.
+    pub fn generation(&self) -> StoreGeneration {
+        StoreGeneration(self.insert_id)
+    }
+
     /// See [`Self::cluster_key`] for more information about the cluster key.
     pub fn cluster_key(&self) -> ComponentName {
         self.cluster_key

crates/re_data_store/src/editable_auto_value.rs

@@ -49,4 +49,14 @@ where
             self
         }
     }
+
+    /// Determine whether this `EditableAutoValue` has user-edits relative to another `EditableAutoValue`
+    /// If both values are `Auto`, then it is not considered edited.
+    pub fn has_edits(&self, other: &Self) -> bool {
+        match (self, other) {
+            (EditableAutoValue::UserEdited(s), EditableAutoValue::UserEdited(o)) => s != o,
+            (EditableAutoValue::Auto(_), EditableAutoValue::Auto(_)) => false,
+            _ => true,
+        }
+    }
 }

crates/re_data_store/src/entity_properties.rs

@@ -34,6 +34,17 @@ impl EntityPropertyMap {
     pub fn iter(&self) -> impl Iterator<Item = (&EntityPath, &EntityProperties)> {
         self.props.iter()
     }
+
+    /// Determine whether this `EntityPropertyMap` has user-edits relative to another `EntityPropertyMap`
+    pub fn has_edits(&self, other: &Self) -> bool {
+        self.props.len() != other.props.len()
+            || self.props.iter().any(|(key, val)| {
+                other
+                    .props
+                    .get(key)
+                    .map_or(true, |other_val| val.has_edits(other_val))
+            })
+    }
 }
 
 // ----------------------------------------------------------------------------
@@ -116,6 +127,29 @@ impl EntityProperties {
                 .clone(),
         }
     }
+
+    /// Determine whether this `EntityProperty` has user-edits relative to another `EntityProperty`
+    pub fn has_edits(&self, other: &Self) -> bool {
+        let Self {
+            visible,
+            visible_history,
+            interactive,
+            color_mapper,
+            pinhole_image_plane_distance,
+            backproject_depth,
+            depth_from_world_scale,
+            backproject_radius_scale,
+        } = self;
+
+        visible != &other.visible
+            || visible_history != &other.visible_history
+            || interactive != &other.interactive
+            || color_mapper.has_edits(&other.color_mapper)
+            || pinhole_image_plane_distance.has_edits(&other.pinhole_image_plane_distance)
+            || backproject_depth.has_edits(&other.backproject_depth)
+            || depth_from_world_scale.has_edits(&other.depth_from_world_scale)
+            || backproject_radius_scale.has_edits(&other.backproject_radius_scale)
+    }
 }
 
 // ----------------------------------------------------------------------------

crates/re_data_store/src/store_db.rs

@@ -292,6 +292,12 @@ impl StoreDb {
             + self.entity_db.data_store.num_temporal_rows() as usize
     }
 
+    /// Return the current `StoreGeneration`. This can be used to determine whether the
+    /// database has been modified since the last time it was queried.
+    pub fn generation(&self) -> re_arrow_store::StoreGeneration {
+        self.entity_db.data_store.generation()
+    }
+
     pub fn is_empty(&self) -> bool {
         self.recording_msg.is_none() && self.num_rows() == 0
     }

crates/re_space_view/src/data_blueprint.rs

@@ -7,7 +7,7 @@ use slotmap::SlotMap;
 use smallvec::{smallvec, SmallVec};
 
 /// A grouping of several data-blueprints.
-#[derive(Clone, Default, PartialEq, serde::Deserialize, serde::Serialize)]
+#[derive(Clone, Default, serde::Deserialize, serde::Serialize)]
 pub struct DataBlueprintGroup {
     pub display_name: String,
 
@@ -31,9 +31,29 @@ pub struct DataBlueprintGroup {
     pub entities: BTreeSet<EntityPath>,
 }
 
+impl DataBlueprintGroup {
+    /// Determine whether this `DataBlueprints` has user-edits relative to another `DataBlueprints`
+    fn has_edits(&self, other: &Self) -> bool {
+        let Self {
+            display_name,
+            properties_individual,
+            properties_projected: _,
+            parent,
+            children,
+            entities,
+        } = self;
+
+        display_name != &other.display_name
+            || properties_individual.has_edits(&other.properties_individual)
+            || parent != &other.parent
+            || children != &other.children
+            || entities != &other.entities
+    }
+}
+
 /// Data blueprints for all entity paths in a space view.
-#[derive(Clone, Default, PartialEq, serde::Deserialize, serde::Serialize)]
-struct DataBlueprints {
+#[derive(Clone, Default, serde::Deserialize, serde::Serialize)]
+pub struct DataBlueprints {
     /// Individual settings. Mutate this.
     individual: EntityPropertyMap,
 
@@ -44,6 +64,19 @@ struct DataBlueprints {
     projected: EntityPropertyMap,
 }
 
+// Manually implement `PartialEq` since projected is serde skip
+impl DataBlueprints {
+    /// Determine whether this `DataBlueprints` has user-edits relative to another `DataBlueprints`
+    fn has_edits(&self, other: &Self) -> bool {
+        let Self {
+            individual,
+            projected: _,
+        } = self;
+
+        individual.has_edits(&other.individual)
+    }
+}
+
 /// Tree of all data blueprint groups for a single space view.
 #[derive(Clone, serde::Deserialize, serde::Serialize)]
 pub struct DataBlueprintTree {
@@ -71,9 +104,9 @@ pub struct DataBlueprintTree {
     data_blueprints: DataBlueprints,
 }
 
-// Manually implement PartialEq since slotmap doesn't
-impl PartialEq for DataBlueprintTree {
-    fn eq(&self, other: &Self) -> bool {
+/// Determine whether this `DataBlueprintTree` has user-edits relative to another `DataBlueprintTree`
+impl DataBlueprintTree {
+    pub fn has_edits(&self, other: &Self) -> bool {
         let Self {
             groups,
             path_to_group,
@@ -82,12 +115,17 @@ impl PartialEq for DataBlueprintTree {
             data_blueprints,
         } = self;
 
-        // Note: this could fail unexpectedly if slotmap iteration order is unstable.
-        groups.iter().zip(other.groups.iter()).all(|(x, y)| x == y)
-            && *path_to_group == other.path_to_group
-            && *entity_paths == other.entity_paths
-            && *root_group_handle == other.root_group_handle
-            && *data_blueprints == other.data_blueprints
+        groups.len() != other.groups.len()
+            || groups.iter().any(|(key, val)| {
+                other
+                    .groups
+                    .get(key)
+                    .map_or(true, |other_val| val.has_edits(other_val))
+            })
+            || *path_to_group != other.path_to_group
+            || *entity_paths != other.entity_paths
+            || *root_group_handle != other.root_group_handle
+            || data_blueprints.has_edits(&other.data_blueprints)
     }
 }
 

crates/re_viewer/Cargo.toml

@@ -75,6 +75,7 @@ arrow2.workspace = true
 arrow2_convert.workspace = true
 bytemuck.workspace = true
 cfg-if.workspace = true
+directories-next = "2.0.0"
 eframe = { workspace = true, default-features = false, features = [
   "default_fonts",
   "persistence",

crates/re_viewer/src/app.rs

@@ -275,7 +275,8 @@ impl App {
             }
             #[cfg(not(target_arch = "wasm32"))]
             SystemCommand::LoadRrd(path) => {
-                if let Some(rrd) = crate::loading::load_file_path(&path) {
+                let with_notification = true;
+                if let Some(rrd) = crate::loading::load_file_path(&path, with_notification) {
                     store_hub.add_bundle(rrd);
                 }
             }
@@ -763,7 +764,8 @@ impl App {
 
             #[cfg(not(target_arch = "wasm32"))]
             if let Some(path) = &file.path {
-                if let Some(rrd) = crate::loading::load_file_path(path) {
+                let with_notification = true;
+                if let Some(rrd) = crate::loading::load_file_path(path, with_notification) {
                     self.on_rrd_loaded(store_hub, rrd);
                 }
             }
@@ -778,7 +780,20 @@ impl eframe::App for App {
 
     fn save(&mut self, storage: &mut dyn eframe::Storage) {
         if self.startup_options.persist_state {
+            // Save the app state
             eframe::set_value(storage, eframe::APP_KEY, &self.state);
+
+            // Save the blueprints
+            // TODO(2579): implement web-storage for blueprints as well
+            #[cfg(not(target_arch = "wasm32"))]
+            if let Some(hub) = &mut self.store_hub {
+                match hub.persist_app_blueprints() {
+                    Ok(f) => f,
+                    Err(err) => {
+                        re_log::error!("Saving blueprints failed: {err}");
+                    }
+                };
+            }
         }
     }
 
@@ -1032,6 +1047,8 @@ fn save(
     store_context: Option<&StoreContext<'_>>,
     loop_selection: Option<(re_data_store::Timeline, re_log_types::TimeRangeF)>,
 ) {
+    use crate::saving::save_database_to_file;
+
     let Some(store_db) = store_context.as_ref().and_then(|view| view.recording) else {
             // NOTE: Can only happen if saving through the command palette.
             re_log::error!("No data to save!");
@@ -1062,67 +1079,3 @@ fn save(
         }
     }
 }
-
-/// Returns a closure that, when run, will save the contents of the current database
-/// to disk, at the specified `path`.
-///
-/// If `time_selection` is specified, then only data for that specific timeline over that
-/// specific time range will be accounted for.
-#[cfg(not(target_arch = "wasm32"))]
-fn save_database_to_file(
-    store_db: &StoreDb,
-    path: std::path::PathBuf,
-    time_selection: Option<(re_data_store::Timeline, re_log_types::TimeRangeF)>,
-) -> anyhow::Result<impl FnOnce() -> anyhow::Result<std::path::PathBuf>> {
-    use itertools::Itertools as _;
-    use re_arrow_store::TimeRange;
-
-    re_tracing::profile_scope!("dump_messages");
-
-    let begin_rec_msg = store_db
-        .recording_msg()
-        .map(|msg| LogMsg::SetStoreInfo(msg.clone()));
-
-    let ent_op_msgs = store_db
-        .iter_entity_op_msgs()
-        .map(|msg| LogMsg::EntityPathOpMsg(store_db.store_id().clone(), msg.clone()))
-        .collect_vec();
-
-    let time_filter = time_selection.map(|(timeline, range)| {
-        (
-            timeline,
-            TimeRange::new(range.min.floor(), range.max.ceil()),
-        )
-    });
-    let data_msgs: Result<Vec<_>, _> = store_db
-        .entity_db
-        .data_store
-        .to_data_tables(time_filter)
-        .map(|table| {
-            table
-                .to_arrow_msg()
-                .map(|msg| LogMsg::ArrowMsg(store_db.store_id().clone(), msg))
-        })
-        .collect();
-
-    use anyhow::Context as _;
-    let data_msgs = data_msgs.with_context(|| "Failed to export to data tables")?;
-
-    let msgs = std::iter::once(begin_rec_msg)
-        .flatten() // option
-        .chain(ent_op_msgs)
-        .chain(data_msgs);
-
-    Ok(move || {
-        re_tracing::profile_scope!("save_to_file");
-
-        use anyhow::Context as _;
-        let file = std::fs::File::create(path.as_path())
-            .with_context(|| format!("Failed to create file at {path:?}"))?;
-
-        let encoding_options = re_log_encoding::EncodingOptions::COMPRESSED;
-        re_log_encoding::encoder::encode_owned(encoding_options, msgs, file)
-            .map(|_| path)
-            .context("Message encode")
-    })
-}

crates/re_viewer/src/lib.rs

@@ -13,6 +13,7 @@ mod loading;
 #[cfg(not(target_arch = "wasm32"))]
 mod profiler;
 mod remote_viewer_app;
+mod saving;
 mod screenshotter;
 mod store_hub;
 mod ui;

crates/re_viewer/src/loading.rs

@@ -2,19 +2,23 @@ use crate::StoreBundle;
 
 #[cfg(not(target_arch = "wasm32"))]
 #[must_use]
-pub fn load_file_path(path: &std::path::Path) -> Option<StoreBundle> {
+pub fn load_file_path(path: &std::path::Path, with_notifications: bool) -> Option<StoreBundle> {
     fn load_file_path_impl(path: &std::path::Path) -> anyhow::Result<StoreBundle> {
         re_tracing::profile_function!();
         use anyhow::Context as _;
         let file = std::fs::File::open(path).context("Failed to open file")?;
         StoreBundle::from_rrd(file)
     }
 
-    re_log::info!("Loading {path:?}…");
+    if with_notifications {
+        re_log::info!("Loading {path:?}…");
+    }
 
     match load_file_path_impl(path) {
         Ok(mut rrd) => {
-            re_log::info!("Loaded {path:?}");
+            if with_notifications {
+                re_log::info!("Loaded {path:?}");
+            }
             for store_db in rrd.store_dbs_mut() {
                 store_db.data_source = Some(re_smart_channel::SmartChannelSource::Files {
                     paths: vec![path.into()],