From 23b0c403be1ccac5e58d8ec6c89d118ea1f6107f Mon Sep 17 00:00:00 2001 From: Oscar Beaumont Date: Mon, 24 Jul 2023 18:59:57 +0800 Subject: [PATCH 1/7] [ENG-925] - `Arc`/`Arc` (#1124) * `Arc` + `Arc` * fix specta version * remove unneccessary derefs --------- Co-authored-by: Brendan Allan --- Cargo.lock | 8 ++-- Cargo.toml | 2 +- core/src/api/search.rs | 4 +- core/src/api/tags.rs | 4 +- core/src/api/utils/library.rs | 4 +- core/src/job/manager.rs | 13 +++--- core/src/job/mod.rs | 2 +- core/src/job/worker.rs | 6 +-- core/src/lib.rs | 5 +-- core/src/library/config.rs | 2 +- core/src/library/library.rs | 3 +- core/src/library/manager.rs | 30 +++++++------- core/src/location/indexer/mod.rs | 4 +- core/src/location/manager/helpers.rs | 17 ++++---- core/src/location/manager/mod.rs | 24 +++++------ core/src/location/manager/watcher/linux.rs | 5 ++- core/src/location/manager/watcher/macos.rs | 6 +-- core/src/location/manager/watcher/mod.rs | 7 ++-- core/src/location/manager/watcher/utils.rs | 11 ++--- core/src/location/manager/watcher/windows.rs | 5 ++- core/src/location/mod.rs | 41 +++++++++++-------- .../file_identifier/file_identifier_job.rs | 2 +- core/src/object/file_identifier/shallow.rs | 2 +- core/src/object/fs/copy.rs | 2 +- core/src/object/fs/cut.rs | 2 +- core/src/object/fs/decrypt.rs | 2 +- core/src/object/fs/delete.rs | 2 +- core/src/object/fs/encrypt.rs | 4 +- core/src/object/fs/erase.rs | 2 +- core/src/object/preview/thumbnail/shallow.rs | 2 +- .../preview/thumbnail/thumbnailer_job.rs | 2 +- core/src/object/validation/validator_job.rs | 4 +- core/src/p2p/pairing/mod.rs | 2 +- 33 files changed, 121 insertions(+), 110 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8fdfceae35c3..d00e4a601827 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7872,8 +7872,8 @@ dependencies = [ [[package]] name = "specta" -version = "1.0.4" -source = "git+https://github.com/oscartbeaumont/specta?rev=2fc97ec8178ba27da1c80c0faaf43cb0db95955f#2fc97ec8178ba27da1c80c0faaf43cb0db95955f" +version = "1.0.5" +source = "git+https://github.com/oscartbeaumont/specta?rev=4bc6e46fc8747cd8d8a07597c1fe13c52aa16a41#4bc6e46fc8747cd8d8a07597c1fe13c52aa16a41" dependencies = [ "bigdecimal", "chrono", @@ -7894,8 +7894,8 @@ dependencies = [ [[package]] name = "specta-macros" -version = "1.0.4" -source = "git+https://github.com/oscartbeaumont/specta?rev=2fc97ec8178ba27da1c80c0faaf43cb0db95955f#2fc97ec8178ba27da1c80c0faaf43cb0db95955f" +version = "1.0.5" +source = "git+https://github.com/oscartbeaumont/specta?rev=4bc6e46fc8747cd8d8a07597c1fe13c52aa16a41#4bc6e46fc8747cd8d8a07597c1fe13c52aa16a41" dependencies = [ "Inflector", "itertools", diff --git a/Cargo.toml b/Cargo.toml index eb2fb8b16608..0081489f542b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,7 +52,7 @@ if-watch = { git = "https://github.com/oscartbeaumont/if-watch", rev = "782eb7b2 mdns-sd = { git = "https://github.com/oscartbeaumont/mdns-sd", rev = "45515a98e9e408c102871abaa5a9bff3bee0cbe8" } # TODO: Do upstream PR httpz = { git = "https://github.com/oscartbeaumont/httpz", rev = "a5185f2ed2fdefeb2f582dce38a692a1bf76d1d6" } -specta = { git = "https://github.com/oscartbeaumont/specta", rev = "2fc97ec8178ba27da1c80c0faaf43cb0db95955f" } +specta = { git = "https://github.com/oscartbeaumont/specta", rev = "4bc6e46fc8747cd8d8a07597c1fe13c52aa16a41" } rspc = { git = "https://github.com/oscartbeaumont/rspc", rev = "adebce542049b982dd251466d4144f4d57e92177" } tauri-specta = { git = "https://github.com/oscartbeaumont/tauri-specta", rev = "c964bef228a90a66effc18cefcba6859c45a8e08" } diff --git a/core/src/api/search.rs b/core/src/api/search.rs index 1c82f0cae4cf..17f2914ee69a 100644 --- a/core/src/api/search.rs +++ b/core/src/api/search.rs @@ -292,7 +292,7 @@ pub fn mount() -> AlphaRouter { cursor, filter, }| async move { - let Library { db, .. } = &library; + let Library { db, .. } = library.as_ref(); let take = take.unwrap_or(100); @@ -368,7 +368,7 @@ pub fn mount() -> AlphaRouter { cursor, filter, }| async move { - let Library { db, .. } = &library; + let Library { db, .. } = library.as_ref(); let take = take.unwrap_or(100); diff --git a/core/src/api/tags.rs b/core/src/api/tags.rs index 8a1a031728c0..a41427c307cf 100644 --- a/core/src/api/tags.rs +++ b/core/src/api/tags.rs @@ -66,7 +66,7 @@ pub(crate) fn mount() -> AlphaRouter { R.with2(library()) .mutation(|(_, library), args: TagAssignArgs| async move { - let Library { db, .. } = &library; + let Library { db, .. } = library.as_ref(); if args.unassign { db.tag_on_object() @@ -107,7 +107,7 @@ pub(crate) fn mount() -> AlphaRouter { R.with2(library()) .mutation(|(_, library), args: TagUpdateArgs| async move { - let Library { sync, db, .. } = &library; + let Library { sync, db, .. } = library.as_ref(); let tag = db .tag() diff --git a/core/src/api/utils/library.rs b/core/src/api/utils/library.rs index fa11ff059cb9..0333f733a356 100644 --- a/core/src/api/utils/library.rs +++ b/core/src/api/utils/library.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use rspc::{ alpha::{ unstable::{MwArgMapper, MwArgMapperMiddleware}, @@ -30,7 +32,7 @@ impl MwArgMapper for LibraryArgsLike { } } -pub(crate) fn library() -> impl MwV3 { +pub(crate) fn library() -> impl MwV3)> { MwArgMapperMiddleware::::new().mount(|mw, ctx: Ctx, library_id| async move { let library = ctx .library_manager diff --git a/core/src/job/manager.rs b/core/src/job/manager.rs index 50f770925bb2..4c01d71b60c5 100644 --- a/core/src/job/manager.rs +++ b/core/src/job/manager.rs @@ -31,7 +31,7 @@ use super::{JobManagerError, JobReport, JobStatus, StatefulJob}; const MAX_WORKERS: usize = 1; pub enum JobManagerEvent { - IngestJob(Library, Box), + IngestJob(Arc, Box), Shutdown(oneshot::Sender<()>), } /// JobManager handles queueing and executing jobs using the `DynJob` @@ -85,7 +85,7 @@ impl JobManager { /// Ingests a new job and dispatches it if possible, queues it otherwise. pub async fn ingest( self: Arc, - library: &Library, + library: &Arc, job: Box>, ) -> Result<(), JobManagerError> { let job_hash = job.hash(); @@ -109,7 +109,7 @@ impl JobManager { } /// Dispatches a job to a worker if under MAX_WORKERS limit, queues it otherwise. - async fn dispatch(self: Arc, library: &Library, mut job: Box) { + async fn dispatch(self: Arc, library: &Arc, mut job: Box) { let mut running_workers = self.running_workers.write().await; let mut job_report = job .report_mut() @@ -151,7 +151,7 @@ impl JobManager { pub async fn complete( self: Arc, - library: &Library, + library: &Arc, worker_id: Uuid, job_hash: u64, next_job: Option>, @@ -238,7 +238,10 @@ impl JobManager { /// when the core was shut down. /// - It will resume jobs that contain data and cancel jobs that do not. /// - Prevents jobs from being stuck in a paused/running state - pub async fn cold_resume(self: Arc, library: &Library) -> Result<(), JobManagerError> { + pub async fn cold_resume( + self: Arc, + library: &Arc, + ) -> Result<(), JobManagerError> { // Include the Queued status in the initial find condition let find_condition = vec![or(vec![ job::status::equals(Some(JobStatus::Paused as i32)), diff --git a/core/src/job/mod.rs b/core/src/job/mod.rs index d0b921f4a47c..427ffa7d33e0 100644 --- a/core/src/job/mod.rs +++ b/core/src/job/mod.rs @@ -219,7 +219,7 @@ impl Job { })) } - pub async fn spawn(self, library: &Library) -> Result<(), JobManagerError> { + pub async fn spawn(self, library: &Arc) -> Result<(), JobManagerError> { library .node_context .job_manager diff --git a/core/src/job/worker.rs b/core/src/job/worker.rs index ab1cf4b075f1..5ca34de6ff6e 100644 --- a/core/src/job/worker.rs +++ b/core/src/job/worker.rs @@ -51,7 +51,7 @@ pub enum WorkerCommand { } pub struct WorkerContext { - pub library: Library, + pub library: Arc, pub(super) events_tx: mpsc::UnboundedSender, } @@ -100,7 +100,7 @@ impl Worker { id: Uuid, mut job: Box, mut report: JobReport, - library: Library, + library: Arc, job_manager: Arc, ) -> Result { let (commands_tx, commands_rx) = mpsc::channel(8); @@ -294,7 +294,7 @@ impl Worker { report_watch_tx: Arc>, start_time: DateTime, commands_rx: mpsc::Receiver, - library: Library, + library: Arc, ) { let (events_tx, mut events_rx) = mpsc::unbounded_channel(); diff --git a/core/src/lib.rs b/core/src/lib.rs index 02b98dcfbb44..c813469e6c15 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -43,7 +43,6 @@ pub(crate) mod sync; pub(crate) mod util; pub(crate) mod volume; -#[derive(Clone)] pub struct NodeContext { pub config: Arc, pub job_manager: Arc, @@ -91,14 +90,14 @@ impl Node { debug!("Initialised 'LocationManager'..."); let library_manager = LibraryManager::new( data_dir.join("libraries"), - NodeContext { + Arc::new(NodeContext { config: config.clone(), job_manager: job_manager.clone(), location_manager: location_manager.clone(), // p2p: p2p.clone(), event_bus_tx: event_bus.0.clone(), notifications: notifications.clone(), - }, + }), ) .await?; debug!("Initialised 'LibraryManager'..."); diff --git a/core/src/library/config.rs b/core/src/library/config.rs index 04e02092972f..6e0d0fd4c0d3 100644 --- a/core/src/library/config.rs +++ b/core/src/library/config.rs @@ -23,7 +23,7 @@ use uuid::Uuid; use super::name::LibraryName; /// LibraryConfig holds the configuration for a specific library. This is stored as a '{uuid}.sdlibrary' file. -#[derive(Debug, Serialize, Deserialize, Clone, Type)] +#[derive(Debug, Clone, Serialize, Deserialize, Type)] pub struct LibraryConfig { /// name is the display name of the library. This is used in the UI and is set by the user. pub name: LibraryName, diff --git a/core/src/library/library.rs b/core/src/library/library.rs index d17b78fc994b..16c3eabd667a 100644 --- a/core/src/library/library.rs +++ b/core/src/library/library.rs @@ -35,7 +35,6 @@ use uuid::Uuid; use super::{LibraryConfig, LibraryManagerError}; /// LibraryContext holds context for a library which can be passed around the application. -#[derive(Clone)] pub struct Library { /// id holds the ID of the current library. pub id: Uuid, @@ -47,7 +46,7 @@ pub struct Library { /// key manager that provides encryption keys to functions that require them // pub key_manager: Arc, /// node_context holds the node context for the node which this library is running on. - pub node_context: NodeContext, + pub node_context: Arc, /// p2p identity pub identity: Arc, pub orphan_remover: OrphanRemoverActor, diff --git a/core/src/library/manager.rs b/core/src/library/manager.rs index e13d062f0fac..f38ea94fd413 100644 --- a/core/src/library/manager.rs +++ b/core/src/library/manager.rs @@ -59,9 +59,9 @@ pub struct LibraryManager { /// libraries_dir holds the path to the directory where libraries are stored. libraries_dir: PathBuf, /// libraries holds the list of libraries which are currently loaded into the node. - libraries: RwLock>, + libraries: RwLock>>, /// node_context holds the context for the node which this library manager is running on. - node_context: NodeContext, + node_context: Arc, /// on load subscribers subscribers: RwLock>>, } @@ -115,7 +115,7 @@ impl From for rspc::Error { impl LibraryManager { pub(crate) async fn new( libraries_dir: PathBuf, - node_context: NodeContext, + node_context: Arc, ) -> Result, LibraryManagerError> { fs::create_dir_all(&libraries_dir) .await @@ -277,7 +277,7 @@ impl LibraryManager { Ok(LibraryConfigWrapped { uuid: id, config }) } - pub(crate) async fn get_all_libraries(&self) -> Vec { + pub(crate) async fn get_all_libraries(&self) -> Vec> { self.libraries.read().await.clone() } @@ -311,19 +311,17 @@ impl LibraryManager { .ok_or(LibraryManagerError::LibraryNotFound)?; // update the library + let mut config = library.config.clone(); if let Some(name) = name { - library.config.name = name; + config.name = name; } match description { MaybeUndefined::Undefined => {} - MaybeUndefined::Null => library.config.description = None, - MaybeUndefined::Value(description) => library.config.description = Some(description), + MaybeUndefined::Null => config.description = None, + MaybeUndefined::Value(description) => config.description = Some(description), } - LibraryConfig::save( - &library.config, - &self.libraries_dir.join(format!("{id}.sdlibrary")), - )?; + LibraryConfig::save(&config, &self.libraries_dir.join(format!("{id}.sdlibrary")))?; invalidate_query!(library, "library.list"); @@ -387,7 +385,7 @@ impl LibraryManager { } // get_ctx will return the library context for the given library id. - pub async fn get_library(&self, library_id: Uuid) -> Option { + pub async fn get_library(&self, library_id: Uuid) -> Option> { self.libraries .read() .await @@ -401,11 +399,11 @@ impl LibraryManager { id: Uuid, db_path: impl AsRef, config_path: PathBuf, - node_context: NodeContext, + node_context: Arc, subscribers: &RwLock>>, create: Option, should_seed: bool, - ) -> Result { + ) -> Result, LibraryManagerError> { let db_path = db_path.as_ref(); let db_url = format!( "file:{}?socket_timeout=15&connection_limit=1", @@ -472,7 +470,7 @@ impl LibraryManager { ) .await; - let library = Library { + let library = Arc::new(Library { id, config, // key_manager, @@ -488,7 +486,7 @@ impl LibraryManager { db, node_context, identity, - }; + }); if should_seed { library.orphan_remover.invoke().await; diff --git a/core/src/location/indexer/mod.rs b/core/src/location/indexer/mod.rs index 6ecdf80b4b55..f8449462dfe4 100644 --- a/core/src/location/indexer/mod.rs +++ b/core/src/location/indexer/mod.rs @@ -87,7 +87,7 @@ async fn execute_indexer_save_step( save_step: &IndexerJobSaveStep, library: &Library, ) -> Result { - let Library { sync, db, .. } = &library; + let Library { sync, db, .. } = library; let (sync_stuff, paths): (Vec<_>, Vec<_>) = save_step .walked @@ -189,7 +189,7 @@ async fn execute_indexer_update_step( update_step: &IndexerJobUpdateStep, library: &Library, ) -> Result { - let Library { sync, db, .. } = &library; + let Library { sync, db, .. } = library; let (sync_stuff, paths_to_update): (Vec<_>, Vec<_>) = update_step .to_update diff --git a/core/src/location/manager/helpers.rs b/core/src/location/manager/helpers.rs index 679ab981d2d0..6370d18f99d4 100644 --- a/core/src/location/manager/helpers.rs +++ b/core/src/location/manager/helpers.rs @@ -3,6 +3,7 @@ use crate::{library::Library, prisma::location, util::db::maybe_missing}; use std::{ collections::{HashMap, HashSet}, path::{Path, PathBuf}, + sync::Arc, time::Duration, }; @@ -50,8 +51,8 @@ pub(super) async fn check_online( pub(super) async fn location_check_sleep( location_id: location::id::Type, - library: Library, -) -> (location::id::Type, Library) { + library: Arc, +) -> (location::id::Type, Arc) { sleep(LOCATION_CHECK_INTERVAL).await; (location_id, library) } @@ -131,7 +132,7 @@ pub(super) async fn get_location( pub(super) async fn handle_remove_location_request( location_id: location::id::Type, - library: Library, + library: Arc, response_tx: oneshot::Sender>, forced_unwatch: &mut HashSet, locations_watched: &mut HashMap, @@ -172,7 +173,7 @@ pub(super) async fn handle_remove_location_request( pub(super) async fn handle_stop_watcher_request( location_id: location::id::Type, - library: Library, + library: Arc, response_tx: oneshot::Sender>, forced_unwatch: &mut HashSet, locations_watched: &mut HashMap, @@ -180,7 +181,7 @@ pub(super) async fn handle_stop_watcher_request( ) { async fn inner( location_id: location::id::Type, - library: Library, + library: Arc, forced_unwatch: &mut HashSet, locations_watched: &mut HashMap, locations_unwatched: &mut HashMap, @@ -215,7 +216,7 @@ pub(super) async fn handle_stop_watcher_request( pub(super) async fn handle_reinit_watcher_request( location_id: location::id::Type, - library: Library, + library: Arc, response_tx: oneshot::Sender>, forced_unwatch: &mut HashSet, locations_watched: &mut HashMap, @@ -223,7 +224,7 @@ pub(super) async fn handle_reinit_watcher_request( ) { async fn inner( location_id: location::id::Type, - library: Library, + library: Arc, forced_unwatch: &mut HashSet, locations_watched: &mut HashMap, locations_unwatched: &mut HashMap, @@ -258,7 +259,7 @@ pub(super) async fn handle_reinit_watcher_request( pub(super) fn handle_ignore_path_request( location_id: location::id::Type, - library: Library, + library: Arc, path: PathBuf, ignore: bool, response_tx: oneshot::Sender>, diff --git a/core/src/location/manager/mod.rs b/core/src/location/manager/mod.rs index 441bb751903f..a544d88069b4 100644 --- a/core/src/location/manager/mod.rs +++ b/core/src/location/manager/mod.rs @@ -42,7 +42,7 @@ enum ManagementMessageAction { #[allow(dead_code)] pub struct LocationManagementMessage { location_id: location::id::Type, - library: Library, + library: Arc, action: ManagementMessageAction, response_tx: oneshot::Sender>, } @@ -59,7 +59,7 @@ enum WatcherManagementMessageAction { #[allow(dead_code)] pub struct WatcherManagementMessage { location_id: location::id::Type, - library: Library, + library: Arc, action: WatcherManagementMessageAction, response_tx: oneshot::Sender>, } @@ -168,7 +168,7 @@ impl LocationManager { async fn location_management_message( &self, location_id: location::id::Type, - library: Library, + library: Arc, action: ManagementMessageAction, ) -> Result<(), LocationManagerError> { #[cfg(feature = "location-watcher")] @@ -196,7 +196,7 @@ impl LocationManager { async fn watcher_management_message( &self, location_id: location::id::Type, - library: Library, + library: Arc, action: WatcherManagementMessageAction, ) -> Result<(), LocationManagerError> { #[cfg(feature = "location-watcher")] @@ -222,7 +222,7 @@ impl LocationManager { pub async fn add( &self, location_id: location::id::Type, - library: Library, + library: Arc, ) -> Result<(), LocationManagerError> { self.location_management_message(location_id, library, ManagementMessageAction::Add) .await @@ -231,7 +231,7 @@ impl LocationManager { pub async fn remove( &self, location_id: location::id::Type, - library: Library, + library: Arc, ) -> Result<(), LocationManagerError> { self.location_management_message(location_id, library, ManagementMessageAction::Remove) .await @@ -240,7 +240,7 @@ impl LocationManager { pub async fn stop_watcher( &self, location_id: location::id::Type, - library: Library, + library: Arc, ) -> Result<(), LocationManagerError> { self.watcher_management_message(location_id, library, WatcherManagementMessageAction::Stop) .await @@ -249,7 +249,7 @@ impl LocationManager { pub async fn reinit_watcher( &self, location_id: location::id::Type, - library: Library, + library: Arc, ) -> Result<(), LocationManagerError> { self.watcher_management_message( location_id, @@ -262,7 +262,7 @@ impl LocationManager { pub async fn temporary_stop( &self, location_id: location::id::Type, - library: Library, + library: Arc, ) -> Result { self.stop_watcher(location_id, library.clone()).await?; @@ -276,7 +276,7 @@ impl LocationManager { pub async fn temporary_ignore_events_for_path( &self, location_id: location::id::Type, - library: Library, + library: Arc, path: impl AsRef, ) -> Result { let path = path.as_ref().to_path_buf(); @@ -558,7 +558,7 @@ impl Drop for LocationManager { pub struct StopWatcherGuard<'m> { manager: &'m LocationManager, location_id: location::id::Type, - library: Option, + library: Option>, } impl Drop for StopWatcherGuard<'_> { @@ -580,7 +580,7 @@ pub struct IgnoreEventsForPathGuard<'m> { manager: &'m LocationManager, path: Option, location_id: location::id::Type, - library: Option, + library: Option>, } impl Drop for IgnoreEventsForPathGuard<'_> { diff --git a/core/src/location/manager/watcher/linux.rs b/core/src/location/manager/watcher/linux.rs index 2ffa438e254e..83916111e5c4 100644 --- a/core/src/location/manager/watcher/linux.rs +++ b/core/src/location/manager/watcher/linux.rs @@ -14,6 +14,7 @@ use crate::{ use std::{ collections::{BTreeMap, HashMap}, path::PathBuf, + sync::Arc, }; use async_trait::async_trait; @@ -32,7 +33,7 @@ use super::{ #[derive(Debug)] pub(super) struct LinuxEventHandler<'lib> { location_id: location::id::Type, - library: &'lib Library, + library: &'lib Arc, last_check_rename: Instant, rename_from: HashMap, rename_from_buffer: Vec<(PathBuf, Instant)>, @@ -42,7 +43,7 @@ pub(super) struct LinuxEventHandler<'lib> { #[async_trait] impl<'lib> EventHandler<'lib> for LinuxEventHandler<'lib> { - fn new(location_id: location::id::Type, library: &'lib Library) -> Self { + fn new(location_id: location::id::Type, library: &'lib Arc) -> Self { Self { location_id, library, diff --git a/core/src/location/manager/watcher/macos.rs b/core/src/location/manager/watcher/macos.rs index a1ef6f41bb24..8b430ea1ec70 100644 --- a/core/src/location/manager/watcher/macos.rs +++ b/core/src/location/manager/watcher/macos.rs @@ -22,7 +22,7 @@ use crate::{ util::error::FileIOError, }; -use std::{collections::HashMap, path::PathBuf}; +use std::{collections::HashMap, path::PathBuf, sync::Arc}; use async_trait::async_trait; use notify::{ @@ -43,7 +43,7 @@ use super::{ #[derive(Debug)] pub(super) struct MacOsEventHandler<'lib> { location_id: location::id::Type, - library: &'lib Library, + library: &'lib Arc, recently_created_files: HashMap, recently_created_files_buffer: Vec<(PathBuf, Instant)>, last_check_created_files: Instant, @@ -56,7 +56,7 @@ pub(super) struct MacOsEventHandler<'lib> { #[async_trait] impl<'lib> EventHandler<'lib> for MacOsEventHandler<'lib> { - fn new(location_id: location::id::Type, library: &'lib Library) -> Self + fn new(location_id: location::id::Type, library: &'lib Arc) -> Self where Self: Sized, { diff --git a/core/src/location/manager/watcher/mod.rs b/core/src/location/manager/watcher/mod.rs index 35e26cfd595f..339cbeb30d2d 100644 --- a/core/src/location/manager/watcher/mod.rs +++ b/core/src/location/manager/watcher/mod.rs @@ -3,6 +3,7 @@ use crate::{library::Library, prisma::location, util::db::maybe_missing}; use std::{ collections::HashSet, path::{Path, PathBuf}, + sync::Arc, time::Duration, }; @@ -47,7 +48,7 @@ const HUNDRED_MILLIS: Duration = Duration::from_millis(100); #[async_trait] trait EventHandler<'lib> { - fn new(location_id: location::id::Type, library: &'lib Library) -> Self + fn new(location_id: location::id::Type, library: &'lib Arc) -> Self where Self: Sized; @@ -72,7 +73,7 @@ pub(super) struct LocationWatcher { impl LocationWatcher { pub(super) async fn new( location: location::Data, - library: Library, + library: Arc, ) -> Result { let (events_tx, events_rx) = mpsc::unbounded_channel(); let (ignore_path_tx, ignore_path_rx) = mpsc::unbounded_channel(); @@ -119,7 +120,7 @@ impl LocationWatcher { async fn handle_watch_events( location_id: location::id::Type, location_pub_id: Uuid, - library: Library, + library: Arc, mut events_rx: mpsc::UnboundedReceiver>, mut ignore_path_rx: mpsc::UnboundedReceiver, mut stop_rx: oneshot::Receiver<()>, diff --git a/core/src/location/manager/watcher/utils.rs b/core/src/location/manager/watcher/utils.rs index e1a3b38551a9..2ec9868505d2 100644 --- a/core/src/location/manager/watcher/utils.rs +++ b/core/src/location/manager/watcher/utils.rs @@ -39,6 +39,7 @@ use std::{ fs::Metadata, path::{Path, PathBuf}, str::FromStr, + sync::Arc, }; use sd_file_ext::extensions::ImageExtension; @@ -67,7 +68,7 @@ pub(super) async fn create_dir( location_id: location::id::Type, path: impl AsRef, metadata: &Metadata, - library: &Library, + library: &Arc, ) -> Result<(), LocationManagerError> { let location = find_location(library, location_id) .include(location_with_indexer_rules::include()) @@ -144,7 +145,7 @@ pub(super) async fn create_file( location_id: location::id::Type, path: impl AsRef, metadata: &Metadata, - library: &Library, + library: &Arc, ) -> Result<(), LocationManagerError> { inner_create_file( location_id, @@ -161,7 +162,7 @@ async fn inner_create_file( location_path: impl AsRef, path: impl AsRef, metadata: &Metadata, - library: &Library, + library: &Arc, ) -> Result<(), LocationManagerError> { let path = path.as_ref(); let location_path = location_path.as_ref(); @@ -324,7 +325,7 @@ async fn inner_create_file( pub(super) async fn create_dir_or_file( location_id: location::id::Type, path: impl AsRef, - library: &Library, + library: &Arc, ) -> Result { let path = path.as_ref(); let metadata = fs::metadata(path) @@ -342,7 +343,7 @@ pub(super) async fn create_dir_or_file( pub(super) async fn update_file( location_id: location::id::Type, full_path: impl AsRef, - library: &Library, + library: &Arc, ) -> Result<(), LocationManagerError> { let full_path = full_path.as_ref(); let location_path = extract_location_path(location_id, library).await?; diff --git a/core/src/location/manager/watcher/windows.rs b/core/src/location/manager/watcher/windows.rs index 7c5c2fd43ee0..a2d9889330c6 100644 --- a/core/src/location/manager/watcher/windows.rs +++ b/core/src/location/manager/watcher/windows.rs @@ -18,6 +18,7 @@ use crate::{ use std::{ collections::{BTreeMap, HashMap}, path::PathBuf, + sync::Arc, }; use async_trait::async_trait; @@ -37,7 +38,7 @@ use super::{ #[derive(Debug)] pub(super) struct WindowsEventHandler<'lib> { location_id: location::id::Type, - library: &'lib Library, + library: &'lib Arc, last_check_recently_files: Instant, recently_created_files: BTreeMap, last_check_rename_and_remove: Instant, @@ -49,7 +50,7 @@ pub(super) struct WindowsEventHandler<'lib> { #[async_trait] impl<'lib> EventHandler<'lib> for WindowsEventHandler<'lib> { - fn new(location_id: location::id::Type, library: &'lib Library) -> Self + fn new(location_id: location::id::Type, library: &'lib Arc) -> Self where Self: Sized, { diff --git a/core/src/location/mod.rs b/core/src/location/mod.rs index c31641c47b77..dacdd45a090a 100644 --- a/core/src/location/mod.rs +++ b/core/src/location/mod.rs @@ -15,6 +15,7 @@ use crate::{ use std::{ collections::HashSet, path::{Component, Path, PathBuf}, + sync::Arc, }; use chrono::Utc; @@ -60,7 +61,7 @@ pub struct LocationCreateArgs { impl LocationCreateArgs { pub async fn create( self, - library: &Library, + library: &Arc, ) -> Result, LocationError> { let path_metadata = match fs::metadata(&self.path).await { Ok(metadata) => metadata, @@ -145,7 +146,7 @@ impl LocationCreateArgs { pub async fn add_library( self, - library: &Library, + library: &Arc, ) -> Result, LocationError> { let mut metadata = SpacedriveLocationMetadataFile::try_load(&self.path) .await? @@ -223,8 +224,8 @@ pub struct LocationUpdateArgs { } impl LocationUpdateArgs { - pub async fn update(self, library: &Library) -> Result<(), LocationError> { - let Library { sync, db, .. } = &library; + pub async fn update(self, library: &Arc) -> Result<(), LocationError> { + let Library { sync, db, .. } = &**library; let location = find_location(library, self.id) .include(location_with_indexer_rules::include()) @@ -341,10 +342,13 @@ impl LocationUpdateArgs { } pub fn find_location( - Library { db, .. }: &Library, + library: &Library, location_id: location::id::Type, ) -> location::FindUniqueQuery { - db.location().find_unique(location::id::equals(location_id)) + library + .db + .location() + .find_unique(location::id::equals(location_id)) } async fn link_location_and_indexer_rules( @@ -368,7 +372,7 @@ async fn link_location_and_indexer_rules( } pub async fn scan_location( - library: &Library, + library: &Arc, location: location_with_indexer_rules::Data, ) -> Result<(), JobManagerError> { // TODO(N): This isn't gonna work with removable media and this will likely permanently break if the DB is restored from a backup. @@ -400,7 +404,7 @@ pub async fn scan_location( #[cfg(feature = "location-watcher")] pub async fn scan_location_sub_path( - library: &Library, + library: &Arc, location: location_with_indexer_rules::Data, sub_path: impl AsRef, ) -> Result<(), JobManagerError> { @@ -437,7 +441,7 @@ pub async fn scan_location_sub_path( } pub async fn light_scan_location( - library: Library, + library: Arc, location: location_with_indexer_rules::Data, sub_path: impl AsRef, ) -> Result<(), JobError> { @@ -458,10 +462,10 @@ pub async fn light_scan_location( } pub async fn relink_location( - library: &Library, + library: &Arc, location_path: impl AsRef, ) -> Result<(), LocationError> { - let Library { db, id, sync, .. } = &library; + let Library { db, id, sync, .. } = &**library; let mut metadata = SpacedriveLocationMetadataFile::try_load(&location_path) .await? @@ -502,13 +506,13 @@ pub struct CreatedLocationResult { } async fn create_location( - library: &Library, + library: &Arc, location_pub_id: Uuid, location_path: impl AsRef, indexer_rules_ids: &[i32], dry_run: bool, ) -> Result, LocationError> { - let Library { db, sync, .. } = &library; + let Library { db, sync, .. } = &**library; let mut path = location_path.as_ref().to_path_buf(); @@ -640,11 +644,9 @@ async fn create_location( } pub async fn delete_location( - library: &Library, + library: &Arc, location_id: location::id::Type, ) -> Result<(), LocationError> { - let Library { db, .. } = library; - library .location_manager() .remove(location_id, library.clone()) @@ -652,14 +654,17 @@ pub async fn delete_location( delete_directory(library, location_id, None).await?; - db.indexer_rules_in_location() + library + .db + .indexer_rules_in_location() .delete_many(vec![indexer_rules_in_location::location_id::equals( location_id, )]) .exec() .await?; - let location = db + let location = library + .db .location() .delete(location::id::equals(location_id)) .exec() diff --git a/core/src/object/file_identifier/file_identifier_job.rs b/core/src/object/file_identifier/file_identifier_job.rs index ac1779e3faae..56a3feb949c4 100644 --- a/core/src/object/file_identifier/file_identifier_job.rs +++ b/core/src/object/file_identifier/file_identifier_job.rs @@ -82,7 +82,7 @@ impl StatefulJob for FileIdentifierJobInit { data: &mut Option, ) -> Result, JobError> { let init = self; - let Library { db, .. } = &ctx.library; + let Library { db, .. } = &*ctx.library; debug!("Identifying orphan File Paths..."); diff --git a/core/src/object/file_identifier/shallow.rs b/core/src/object/file_identifier/shallow.rs index 38aef6a064a0..d08807bc82e5 100644 --- a/core/src/object/file_identifier/shallow.rs +++ b/core/src/object/file_identifier/shallow.rs @@ -28,7 +28,7 @@ pub async fn shallow( sub_path: &PathBuf, library: &Library, ) -> Result<(), JobError> { - let Library { db, .. } = &library; + let Library { db, .. } = library; debug!("Identifying orphan File Paths..."); diff --git a/core/src/object/fs/copy.rs b/core/src/object/fs/copy.rs index 4662e614c50c..ddba276617e4 100644 --- a/core/src/object/fs/copy.rs +++ b/core/src/object/fs/copy.rs @@ -60,7 +60,7 @@ impl StatefulJob for FileCopierJobInit { data: &mut Option, ) -> Result, JobError> { let init = self; - let Library { db, .. } = &ctx.library; + let Library { db, .. } = &*ctx.library; let (sources_location_path, targets_location_path) = fetch_source_and_target_location_paths( diff --git a/core/src/object/fs/cut.rs b/core/src/object/fs/cut.rs index 259f5051a655..33809f15acde 100644 --- a/core/src/object/fs/cut.rs +++ b/core/src/object/fs/cut.rs @@ -48,7 +48,7 @@ impl StatefulJob for FileCutterJobInit { data: &mut Option, ) -> Result, JobError> { let init = self; - let Library { db, .. } = &ctx.library; + let Library { db, .. } = &*ctx.library; let (sources_location_path, targets_location_path) = fetch_source_and_target_location_paths( diff --git a/core/src/object/fs/decrypt.rs b/core/src/object/fs/decrypt.rs index 871c7ba1b71a..a8f8424f6b17 100644 --- a/core/src/object/fs/decrypt.rs +++ b/core/src/object/fs/decrypt.rs @@ -44,7 +44,7 @@ // } // async fn init(&self, ctx: WorkerContext, state: &mut JobState) -> Result<(), JobError> { -// let Library { db, .. } = &ctx.library; +// let Library { db, .. } = &*ctx.library; // state.steps = get_many_files_datas( // db, diff --git a/core/src/object/fs/delete.rs b/core/src/object/fs/delete.rs index 76239705609b..e35767ea1a9b 100644 --- a/core/src/object/fs/delete.rs +++ b/core/src/object/fs/delete.rs @@ -38,7 +38,7 @@ impl StatefulJob for FileDeleterJobInit { data: &mut Option, ) -> Result, JobError> { let init = self; - let Library { db, .. } = &ctx.library; + let Library { db, .. } = &*ctx.library; let steps = get_many_files_datas( db, diff --git a/core/src/object/fs/encrypt.rs b/core/src/object/fs/encrypt.rs index d1cfd251a3da..0ca16508756b 100644 --- a/core/src/object/fs/encrypt.rs +++ b/core/src/object/fs/encrypt.rs @@ -68,7 +68,7 @@ // } // async fn init(&self, ctx: WorkerContext, state: &mut JobState) -> Result<(), JobError> { -// let Library { db, .. } = &ctx.library; +// let Library { db, .. } = &*ctx.library; // state.steps = get_many_files_datas( // db, @@ -90,7 +90,7 @@ // ) -> Result<(), JobError> { // let step = &state.steps[0]; -// let Library { key_manager, .. } = &ctx.library; +// let Library { key_manager, .. } = &*ctx.library; // if !step.file_path.is_dir { // // handle overwriting checks, and making sure there's enough available space diff --git a/core/src/object/fs/erase.rs b/core/src/object/fs/erase.rs index 84b491e80076..f98ca6b77cfa 100644 --- a/core/src/object/fs/erase.rs +++ b/core/src/object/fs/erase.rs @@ -68,7 +68,7 @@ impl StatefulJob for FileEraserJobInit { data: &mut Option, ) -> Result, JobError> { let init = self; - let Library { db, .. } = &ctx.library; + let Library { db, .. } = &*ctx.library; let location_path = get_location_path_from_location_id(db, init.location_id).await?; diff --git a/core/src/object/preview/thumbnail/shallow.rs b/core/src/object/preview/thumbnail/shallow.rs index 044f4611319f..27be4b84726b 100644 --- a/core/src/object/preview/thumbnail/shallow.rs +++ b/core/src/object/preview/thumbnail/shallow.rs @@ -27,7 +27,7 @@ pub async fn shallow_thumbnailer( sub_path: &PathBuf, library: &Library, ) -> Result<(), JobError> { - let Library { db, .. } = &library; + let Library { db, .. } = library; let thumbnail_dir = init_thumbnail_dir(library.config().data_directory()).await?; diff --git a/core/src/object/preview/thumbnail/thumbnailer_job.rs b/core/src/object/preview/thumbnail/thumbnailer_job.rs index 876222e0b132..5822ed1c7a3e 100644 --- a/core/src/object/preview/thumbnail/thumbnailer_job.rs +++ b/core/src/object/preview/thumbnail/thumbnailer_job.rs @@ -83,7 +83,7 @@ impl StatefulJob for ThumbnailerJobInit { data: &mut Option, ) -> Result, JobError> { let init = self; - let Library { db, .. } = &ctx.library; + let Library { db, .. } = &*ctx.library; let thumbnail_dir = init_thumbnail_dir(ctx.library.config().data_directory()).await?; diff --git a/core/src/object/validation/validator_job.rs b/core/src/object/validation/validator_job.rs index da9188f821b0..b1ec3af4b7c0 100644 --- a/core/src/object/validation/validator_job.rs +++ b/core/src/object/validation/validator_job.rs @@ -66,7 +66,7 @@ impl StatefulJob for ObjectValidatorJobInit { data: &mut Option, ) -> Result, JobError> { let init = self; - let Library { db, .. } = &ctx.library; + let Library { db, .. } = &*ctx.library; let location_id = init.location.id; @@ -135,7 +135,7 @@ impl StatefulJob for ObjectValidatorJobInit { _: &Self::RunMetadata, ) -> Result, JobError> { let init = self; - let Library { db, sync, .. } = &ctx.library; + let Library { db, sync, .. } = &*ctx.library; // this is to skip files that already have checksums // i'm unsure what the desired behaviour is in this case diff --git a/core/src/p2p/pairing/mod.rs b/core/src/p2p/pairing/mod.rs index 5bcaca8b87ed..9d1ba8666458 100644 --- a/core/src/p2p/pairing/mod.rs +++ b/core/src/p2p/pairing/mod.rs @@ -244,7 +244,7 @@ impl PairingManager { .write_all( &PairingResponse::Accepted { library_id: library.id, - library_name: library.config.name.into(), + library_name: library.config.name.clone().into(), library_description: library.config.description.clone(), instances: library .db From 043b607ad4713ede79e0ddd57c9469606dbea0c2 Mon Sep 17 00:00:00 2001 From: Brendan Allan Date: Mon, 24 Jul 2023 23:26:00 +0800 Subject: [PATCH 2/7] CRDTOperation receiving (#1122) * operation receive + compare * cleanup + deduplication * operation receive + compare * cleanup + deduplication * sync route + operation grouping * tag assign sync * proper relation support in sync debug page * migration * separate core-sync + utils crates * separate p2p event loop from manager * cleanup library handling * clippy * feature gate sync messages properly * make migration not add required field --- Cargo.lock | 25 +- Cargo.toml | 4 + .../screens/onboarding/CreatingLibrary.tsx | 2 - core/Cargo.toml | 9 +- core/crates/sync/Cargo.toml | 21 ++ core/crates/sync/src/lib.rs | 357 ++++++++++++++++++ .../migration.sql | 15 + core/prisma/schema.prisma | 21 ++ core/src/api/libraries.rs | 2 + core/src/api/p2p.rs | 2 +- core/src/api/search.rs | 5 +- core/src/api/sync.rs | 3 +- core/src/api/tags.rs | 90 +++-- core/src/job/report.rs | 4 +- core/src/lib.rs | 14 +- core/src/library/config.rs | 9 +- core/src/library/library.rs | 56 ++- core/src/library/manager.rs | 155 +++----- core/src/location/file_path_helper/mod.rs | 9 +- core/src/location/indexer/mod.rs | 16 +- core/src/location/indexer/rules/mod.rs | 4 +- core/src/location/indexer/rules/seed.rs | 3 +- core/src/location/indexer/walk.rs | 6 +- core/src/location/manager/watcher/utils.rs | 7 +- core/src/location/mod.rs | 14 +- .../file_identifier/file_identifier_job.rs | 4 +- core/src/object/file_identifier/mod.rs | 29 +- core/src/object/file_identifier/shallow.rs | 4 +- core/src/object/tag/mod.rs | 5 +- core/src/object/validation/validator_job.rs | 12 +- core/src/p2p/p2p_manager.rs | 169 +++++---- core/src/p2p/pairing/mod.rs | 25 +- core/src/p2p/pairing/proto.rs | 1 + core/src/p2p/peer_metadata.rs | 23 +- core/src/preferences/kv.rs | 2 +- core/src/preferences/mod.rs | 2 +- core/src/sync/manager.rs | 220 ----------- core/src/sync/mod.rs | 5 - core/src/util/db.rs | 23 -- core/src/util/debug_initializer.rs | 3 +- crates/sync-generator/src/lib.rs | 13 +- crates/sync/src/crdt.rs | 62 ++- crates/sync/src/lib.rs | 157 ++++++++ crates/utils/Cargo.toml | 9 + crates/utils/src/lib.rs | 23 ++ .../$libraryId/Layout/Sidebar/Contents.tsx | 8 +- interface/app/$libraryId/sync.tsx | 211 +++++++++-- interface/app/onboarding/creating-library.tsx | 3 +- packages/client/src/core.ts | 2 +- packages/client/src/hooks/useFeatureFlag.tsx | 4 +- .../client/src/hooks/useNotifications.tsx | 2 +- 51 files changed, 1235 insertions(+), 639 deletions(-) create mode 100644 core/crates/sync/Cargo.toml create mode 100644 core/crates/sync/src/lib.rs create mode 100644 core/prisma/migrations/20230724131659_relation_operation/migration.sql delete mode 100644 core/src/sync/manager.rs delete mode 100644 core/src/sync/mod.rs create mode 100644 crates/utils/Cargo.toml create mode 100644 crates/utils/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index d00e4a601827..2e8e3e3419ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7102,6 +7102,7 @@ dependencies = [ "rmp-serde", "rmpv", "rspc", + "sd-core-sync", "sd-crypto", "sd-ffmpeg", "sd-file-ext", @@ -7109,6 +7110,7 @@ dependencies = [ "sd-p2p", "sd-prisma", "sd-sync", + "sd-utils", "serde", "serde-hashkey", "serde_json", @@ -7126,12 +7128,26 @@ dependencies = [ "tracing-appender", "tracing-subscriber 0.3.0", "tracing-test", - "uhlc", "uuid", "webp", "winapi-util", ] +[[package]] +name = "sd-core-sync" +version = "0.1.0" +dependencies = [ + "prisma-client-rust", + "sd-prisma", + "sd-sync", + "sd-utils", + "serde", + "serde_json", + "tokio", + "uhlc", + "uuid", +] + [[package]] name = "sd-crypto" version = "0.0.0" @@ -7328,6 +7344,13 @@ dependencies = [ "thiserror", ] +[[package]] +name = "sd-utils" +version = "0.1.0" +dependencies = [ + "uuid", +] + [[package]] name = "sdp" version = "0.5.3" diff --git a/Cargo.toml b/Cargo.toml index 0081489f542b..83680a706425 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ resolver = "2" members = [ "core", + "core/crates/*", "crates/*", # "crates/p2p/tunnel", # "crates/p2p/tunnel/utils", @@ -42,6 +43,9 @@ tauri-specta = { version = "1.0.2" } swift-rs = { version = "1.0.5" } tokio = { version = "1.28.2" } +uuid = { version = "1.3.3", features = ["v4", "serde"] } +serde = { version = "1.0" } +serde_json = { version = "1.0" } [patch.crates-io] # We use this patch so we can compile for the IOS simulator on M1 diff --git a/apps/mobile/src/screens/onboarding/CreatingLibrary.tsx b/apps/mobile/src/screens/onboarding/CreatingLibrary.tsx index c3b8d6473dae..5b97060861ef 100644 --- a/apps/mobile/src/screens/onboarding/CreatingLibrary.tsx +++ b/apps/mobile/src/screens/onboarding/CreatingLibrary.tsx @@ -50,8 +50,6 @@ const CreatingLibraryScreen = ({ navigation }: OnboardingStackScreenProps<'Creat const create = async () => { telemetryStore.shareTelemetry = obStore.shareTelemetry; createLibrary.mutate({ name: obStore.newLibraryName }); - - return; }; useEffect(() => { diff --git a/core/Cargo.toml b/core/Cargo.toml index e39d8618aa23..b6a40d3e661c 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -15,7 +15,6 @@ mobile = [] # This feature controls whether the Spacedrive Core contains functionality which requires FFmpeg. ffmpeg = ["dep:sd-ffmpeg"] location-watcher = ["dep:notify"] -sync-messages = [] heif = ["dep:sd-heif"] [dependencies] @@ -31,6 +30,9 @@ sd-file-ext = { path = "../crates/file-ext" } sd-sync = { path = "../crates/sync" } sd-p2p = { path = "../crates/p2p", features = ["specta", "serde"] } sd-prisma = { path = "../crates/prisma" } +sd-utils = { path = "../crates/utils" } + +sd-core-sync = { path = "./crates/sync" } rspc = { workspace = true, features = [ "uuid", @@ -54,14 +56,14 @@ tokio = { workspace = true, features = [ base64 = "0.21.2" serde = { version = "1.0", features = ["derive"] } chrono = { version = "0.4.25", features = ["serde"] } -serde_json = "1.0" +serde_json = { workspace = true } futures = "0.3" rmp = "^0.8.11" rmp-serde = "^1.1.1" rmpv = "^1.0.0" blake3 = "1.3.3" hostname = "0.3.1" -uuid = { version = "1.3.3", features = ["v4", "serde"] } +uuid = { workspace = true } sysinfo = "0.28.4" thiserror = "1.0.40" include_dir = { version = "0.7.3", features = ["glob"] } @@ -78,7 +80,6 @@ ctor = "0.1.26" globset = { version = "^0.4.10", features = ["serde1"] } itertools = "^0.10.5" enumflags2 = "0.7.7" -uhlc = "0.5.2" http-range = "0.1.5" mini-moka = "0.10.0" serde_with = "2.3.3" diff --git a/core/crates/sync/Cargo.toml b/core/crates/sync/Cargo.toml new file mode 100644 index 000000000000..217a37306283 --- /dev/null +++ b/core/crates/sync/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "sd-core-sync" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[features] +default = [] +emit-messages = [] + +[dependencies] +sd-prisma = { path = "../../../crates/prisma" } +sd-sync = { path = "../../../crates/sync" } +sd-utils = { path = "../../../crates/utils" } + +prisma-client-rust = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tokio = { workspace = true } +uuid = { workspace = true } +uhlc = "0.5.2" diff --git a/core/crates/sync/src/lib.rs b/core/crates/sync/src/lib.rs new file mode 100644 index 000000000000..ab72cb099ea8 --- /dev/null +++ b/core/crates/sync/src/lib.rs @@ -0,0 +1,357 @@ +#![allow(clippy::unwrap_used, clippy::panic)] // TODO: Brendan remove this once you've got error handling here + +use sd_prisma::{prisma::*, prisma_sync::ModelSyncData}; +use sd_sync::*; +use sd_utils::uuid_to_bytes; + +use std::{ + collections::{BTreeMap, HashMap}, + sync::Arc, +}; + +use serde_json::to_vec; +use tokio::sync::broadcast::{self, Receiver, Sender}; +use uhlc::{HLCBuilder, Timestamp, HLC, NTP64}; +use uuid::Uuid; + +pub use sd_prisma::prisma_sync; + +#[derive(Clone)] +pub enum SyncMessage { + Ingested(CRDTOperation), + Created(CRDTOperation), +} + +pub struct SyncManager { + db: Arc, + instance: Uuid, + _clocks: HashMap, + clock: HLC, + pub tx: Sender, +} + +impl SyncManager { + pub fn new(db: &Arc, instance: Uuid) -> (Self, Receiver) { + let (tx, rx) = broadcast::channel(64); + + ( + Self { + db: db.clone(), + instance, + clock: HLCBuilder::new().with_id(instance.into()).build(), + _clocks: Default::default(), + tx, + }, + rx, + ) + } + + pub async fn write_ops<'item, I: prisma_client_rust::BatchItem<'item>>( + &self, + tx: &PrismaClient, + (_ops, queries): (Vec, I), + ) -> prisma_client_rust::Result<::ReturnValue> { + #[cfg(feature = "emit-messages")] + let res = { + macro_rules! variant { + ($var:ident, $variant:ident, $fn:ident) => { + let $var = _ops + .iter() + .filter_map(|op| match &op.typ { + CRDTOperationType::$variant(inner) => { + Some($fn(&op, &inner).to_query(tx)) + } + _ => None, + }) + .collect::>(); + }; + } + + variant!(shared, Shared, shared_op_db); + variant!(relation, Relation, relation_op_db); + + let (res, _) = tx._batch((queries, (shared, relation))).await?; + + for op in _ops { + self.tx.send(SyncMessage::Created(op)).ok(); + } + + res + }; + #[cfg(not(feature = "emit-messages"))] + let res = tx._batch([queries]).await?.remove(0); + + Ok(res) + } + + #[allow(unused_variables)] + pub async fn write_op<'item, Q: prisma_client_rust::BatchItem<'item>>( + &self, + tx: &PrismaClient, + op: CRDTOperation, + query: Q, + ) -> prisma_client_rust::Result<::ReturnValue> { + #[cfg(feature = "emit-messages")] + let ret = { + macro_rules! exec { + ($fn:ident, $inner:ident) => { + tx._batch(($fn(&op, $inner).to_query(tx), query)).await?.1 + }; + } + + let ret = match &op.typ { + CRDTOperationType::Shared(inner) => exec!(shared_op_db, inner), + CRDTOperationType::Relation(inner) => exec!(relation_op_db, inner), + }; + + self.tx.send(SyncMessage::Created(op)).ok(); + + ret + }; + #[cfg(not(feature = "emit-messages"))] + let ret = tx._batch(vec![query]).await?.remove(0); + + Ok(ret) + } + + pub async fn get_ops(&self) -> prisma_client_rust::Result> { + let Self { db, .. } = self; + + shared_operation::include!(shared_include { + instance: select { pub_id } + }); + relation_operation::include!(relation_include { + instance: select { pub_id } + }); + + enum DbOperation { + Shared(shared_include::Data), + Relation(relation_include::Data), + } + + impl DbOperation { + fn timestamp(&self) -> NTP64 { + NTP64(match self { + Self::Shared(op) => op.timestamp, + Self::Relation(op) => op.timestamp, + } as u64) + } + + fn id(&self) -> Uuid { + Uuid::from_slice(match self { + Self::Shared(op) => &op.id, + Self::Relation(op) => &op.id, + }) + .unwrap() + } + + fn instance(&self) -> Uuid { + Uuid::from_slice(match self { + Self::Shared(op) => &op.instance.pub_id, + Self::Relation(op) => &op.instance.pub_id, + }) + .unwrap() + } + + fn into_operation(self) -> CRDTOperation { + CRDTOperation { + id: self.id(), + instance: self.instance(), + timestamp: self.timestamp(), + typ: match self { + Self::Shared(op) => CRDTOperationType::Shared(SharedOperation { + record_id: serde_json::from_slice(&op.record_id).unwrap(), + model: op.model, + data: serde_json::from_slice(&op.data).unwrap(), + }), + Self::Relation(op) => CRDTOperationType::Relation(RelationOperation { + relation: op.relation, + data: serde_json::from_slice(&op.data).unwrap(), + relation_item: serde_json::from_slice(&op.item_id).unwrap(), + relation_group: serde_json::from_slice(&op.group_id).unwrap(), + }), + }, + } + } + } + + let (shared, relation) = db + ._batch(( + db.shared_operation() + .find_many(vec![]) + .include(shared_include::include()), + db.relation_operation() + .find_many(vec![]) + .include(relation_include::include()), + )) + .await?; + + let mut ops = BTreeMap::new(); + + ops.extend( + shared + .into_iter() + .map(DbOperation::Shared) + .map(|op| (op.timestamp(), op)), + ); + ops.extend( + relation + .into_iter() + .map(DbOperation::Relation) + .map(|op| (op.timestamp(), op)), + ); + + Ok(ops + .into_values() + .map(DbOperation::into_operation) + .rev() + .collect()) + } + + pub async fn apply_op(&self, op: CRDTOperation) -> prisma_client_rust::Result<()> { + ModelSyncData::from_op(op.typ.clone()) + .unwrap() + .exec(&self.db) + .await?; + + match &op.typ { + CRDTOperationType::Shared(shared_op) => { + shared_op_db(&op, shared_op) + .to_query(&self.db) + .exec() + .await?; + } + CRDTOperationType::Relation(relation_op) => { + relation_op_db(&op, relation_op) + .to_query(&self.db) + .exec() + .await?; + } + } + + self.tx.send(SyncMessage::Ingested(op.clone())).ok(); + + Ok(()) + } + + async fn compare_message(&self, op: &CRDTOperation) -> bool { + let old_timestamp = match &op.typ { + CRDTOperationType::Shared(shared_op) => { + let newer_op = self + .db + .shared_operation() + .find_first(vec![ + shared_operation::timestamp::gte(op.timestamp.as_u64() as i64), + shared_operation::model::equals(shared_op.model.to_string()), + shared_operation::record_id::equals( + serde_json::to_vec(&shared_op.record_id).unwrap(), + ), + shared_operation::kind::equals(shared_op.kind().to_string()), + ]) + .order_by(shared_operation::timestamp::order(SortOrder::Desc)) + .exec() + .await + .unwrap(); + + newer_op.map(|newer_op| newer_op.timestamp) + } + CRDTOperationType::Relation(relation_op) => { + let newer_op = self + .db + .relation_operation() + .find_first(vec![ + relation_operation::timestamp::gte(op.timestamp.as_u64() as i64), + relation_operation::relation::equals(relation_op.relation.to_string()), + relation_operation::item_id::equals( + serde_json::to_vec(&relation_op.relation_item).unwrap(), + ), + relation_operation::kind::equals(relation_op.kind().to_string()), + ]) + .order_by(relation_operation::timestamp::order(SortOrder::Desc)) + .exec() + .await + .unwrap(); + + newer_op.map(|newer_op| newer_op.timestamp) + } + }; + + old_timestamp + .map(|old| old != op.timestamp.as_u64() as i64) + .unwrap_or_default() + } + + pub async fn receive_crdt_operation(&mut self, op: CRDTOperation) { + self.clock + .update_with_timestamp(&Timestamp::new(op.timestamp, op.instance.into())) + .ok(); + + let timestamp = self + ._clocks + .entry(op.instance) + .or_insert_with(|| op.timestamp); + + if *timestamp < op.timestamp { + *timestamp = op.timestamp; + } + + let op_timestamp = op.timestamp; + let op_instance = op.instance; + + let is_old = self.compare_message(&op).await; + + if !is_old { + self.apply_op(op).await.ok(); + } + + self.db + .instance() + .update( + instance::pub_id::equals(uuid_to_bytes(op_instance)), + vec![instance::timestamp::set(Some(op_timestamp.as_u64() as i64))], + ) + .exec() + .await + .ok(); + } +} + +fn shared_op_db(op: &CRDTOperation, shared_op: &SharedOperation) -> shared_operation::Create { + shared_operation::Create { + id: op.id.as_bytes().to_vec(), + timestamp: op.timestamp.0 as i64, + instance: instance::pub_id::equals(op.instance.as_bytes().to_vec()), + kind: shared_op.kind().to_string(), + data: to_vec(&shared_op.data).unwrap(), + model: shared_op.model.to_string(), + record_id: to_vec(&shared_op.record_id).unwrap(), + _params: vec![], + } +} + +fn relation_op_db( + op: &CRDTOperation, + relation_op: &RelationOperation, +) -> relation_operation::Create { + relation_operation::Create { + id: op.id.as_bytes().to_vec(), + timestamp: op.timestamp.0 as i64, + instance: instance::pub_id::equals(op.instance.as_bytes().to_vec()), + kind: relation_op.kind().to_string(), + data: to_vec(&relation_op.data).unwrap(), + relation: relation_op.relation.to_string(), + item_id: to_vec(&relation_op.relation_item).unwrap(), + group_id: to_vec(&relation_op.relation_group).unwrap(), + _params: vec![], + } +} + +impl OperationFactory for SyncManager { + fn get_clock(&self) -> &HLC { + &self.clock + } + + fn get_instance(&self) -> Uuid { + self.instance + } +} diff --git a/core/prisma/migrations/20230724131659_relation_operation/migration.sql b/core/prisma/migrations/20230724131659_relation_operation/migration.sql new file mode 100644 index 000000000000..4acfb50ffbb7 --- /dev/null +++ b/core/prisma/migrations/20230724131659_relation_operation/migration.sql @@ -0,0 +1,15 @@ +-- AlterTable +ALTER TABLE "instance" ADD COLUMN "timestamp" BIGINT; + +-- CreateTable +CREATE TABLE "relation_operation" ( + "id" BLOB NOT NULL PRIMARY KEY, + "timestamp" BIGINT NOT NULL, + "relation" TEXT NOT NULL, + "item_id" BLOB NOT NULL, + "group_id" BLOB NOT NULL, + "kind" TEXT NOT NULL, + "data" BLOB NOT NULL, + "instance_id" INTEGER NOT NULL, + CONSTRAINT "relation_operation_instance_id_fkey" FOREIGN KEY ("instance_id") REFERENCES "instance" ("id") ON DELETE RESTRICT ON UPDATE CASCADE +); diff --git a/core/prisma/schema.prisma b/core/prisma/schema.prisma index afcdaea14e71..0b9ca14148e7 100644 --- a/core/prisma/schema.prisma +++ b/core/prisma/schema.prisma @@ -36,6 +36,23 @@ model SharedOperation { @@map("shared_operation") } +model RelationOperation { + id Bytes @id + timestamp BigInt + relation String + + item_id Bytes + group_id Bytes + + kind String + data Bytes + + instance_id Int + instance Instance @relation(fields: [instance_id], references: [id]) + + @@map("relation_operation") +} + /// @deprecated: This model has to exist solely for backwards compatibility. model Node { id Int @id @default(autoincrement()) @@ -67,9 +84,13 @@ model Instance { last_seen DateTime // Time core started for owner, last P2P message for P2P node date_created DateTime + // clock timestamp for sync + timestamp BigInt? + // attestation Bytes SharedOperation SharedOperation[] + RelationOperation RelationOperation[] @@map("instance") } diff --git a/core/src/api/libraries.rs b/core/src/api/libraries.rs index d293240a9587..fc1bbc681cd2 100644 --- a/core/src/api/libraries.rs +++ b/core/src/api/libraries.rs @@ -94,6 +94,8 @@ pub(crate) fn mount() -> AlphaRouter { .create(args.name, None, ctx.config.get().await) .await?; + debug!("Created library {}", new_library.uuid); + Ok(new_library) }) }) diff --git a/core/src/api/p2p.rs b/core/src/api/p2p.rs index aac6dba69683..f40021187796 100644 --- a/core/src/api/p2p.rs +++ b/core/src/api/p2p.rs @@ -80,7 +80,7 @@ pub(crate) fn mount() -> AlphaRouter { ctx.p2p .pairing .clone() - .originator(id, ctx.config.get().await) + .originator(id, ctx.config.get().await, ctx.library_manager.clone()) .await }) }) diff --git a/core/src/api/search.rs b/core/src/api/search.rs index 17f2914ee69a..c0f50cdc2662 100644 --- a/core/src/api/search.rs +++ b/core/src/api/search.rs @@ -10,7 +10,6 @@ use crate::{ }, object::preview::get_thumb_key, prisma::{self, file_path, location, object, tag, tag_on_object, PrismaClient}, - util::db::chain_optional_iter, }; use std::collections::BTreeSet; @@ -159,7 +158,7 @@ impl FilePathFilterArgs { { use file_path::*; - Ok(chain_optional_iter( + Ok(sd_utils::chain_optional_iter( self.search .unwrap_or_default() .split(' ') @@ -248,7 +247,7 @@ impl ObjectFilterArgs { fn into_params(self) -> Vec { use object::*; - chain_optional_iter( + sd_utils::chain_optional_iter( [], [ self.hidden.to_param(), diff --git a/core/src/api/sync.rs b/core/src/api/sync.rs index ccf63667b6f9..1ea144fca58d 100644 --- a/core/src/api/sync.rs +++ b/core/src/api/sync.rs @@ -1,6 +1,5 @@ use rspc::alpha::AlphaRouter; - -use crate::sync::SyncMessage; +use sd_core_sync::SyncMessage; use super::{utils::library, Ctx, R}; diff --git a/core/src/api/tags.rs b/core/src/api/tags.rs index a41427c307cf..dc10c1cf7806 100644 --- a/core/src/api/tags.rs +++ b/core/src/api/tags.rs @@ -1,5 +1,7 @@ use chrono::Utc; use rspc::{alpha::AlphaRouter, ErrorCode}; +use sd_prisma::prisma_sync; +use sd_sync::OperationFactory; use serde::Deserialize; use specta::Type; @@ -9,8 +11,7 @@ use crate::{ invalidate_query, library::Library, object::tag::TagCreateArgs, - prisma::{tag, tag_on_object}, - sync::{self, OperationFactory}, + prisma::{object, tag, tag_on_object}, }; use super::{utils::library, Ctx, R}; @@ -66,29 +67,72 @@ pub(crate) fn mount() -> AlphaRouter { R.with2(library()) .mutation(|(_, library), args: TagAssignArgs| async move { - let Library { db, .. } = library.as_ref(); + let Library { db, sync, .. } = library.as_ref(); + + let (tag, objects) = db + ._batch(( + db.tag() + .find_unique(tag::id::equals(args.tag_id)) + .select(tag::select!({ pub_id })), + db.object() + .find_many(vec![object::id::in_vec(args.object_ids)]) + .select(object::select!({ id pub_id })), + )) + .await?; + + let tag = tag.ok_or_else(|| { + rspc::Error::new(ErrorCode::NotFound, "Tag not found".to_string()) + })?; + + macro_rules! sync_id { + ($object:ident) => { + prisma_sync::tag_on_object::SyncId { + tag: prisma_sync::tag::SyncId { + pub_id: tag.pub_id.clone(), + }, + object: prisma_sync::object::SyncId { + pub_id: $object.pub_id.clone(), + }, + } + }; + } if args.unassign { - db.tag_on_object() - .delete_many(vec![ - tag_on_object::tag_id::equals(args.tag_id), - tag_on_object::object_id::in_vec(args.object_ids), - ]) - .exec() - .await?; - } else { - db.tag_on_object() - .create_many( - args.object_ids - .iter() - .map(|&object_id| tag_on_object::CreateUnchecked { - tag_id: args.tag_id, - object_id, - _params: vec![], - }) + let query = db.tag_on_object().delete_many(vec![ + tag_on_object::tag_id::equals(args.tag_id), + tag_on_object::object_id::in_vec( + objects.iter().map(|o| o.id).collect(), + ), + ]); + + sync.write_ops( + db, + ( + objects + .into_iter() + .map(|object| sync.relation_delete(sync_id!(object))) .collect(), - ) - .exec() + query, + ), + ) + .await?; + } else { + let (sync_ops, db_creates) = objects.into_iter().fold( + (vec![], vec![]), + |(mut sync_ops, mut db_creates), object| { + db_creates.push(tag_on_object::CreateUnchecked { + tag_id: args.tag_id, + object_id: object.id, + _params: vec![], + }); + + sync_ops.extend(sync.relation_create(sync_id!(object), [])); + + (sync_ops, db_creates) + }, + ); + + sync.write_ops(db, (sync_ops, db.tag_on_object().create_many(db_creates))) .await?; } @@ -139,7 +183,7 @@ pub(crate) fn mount() -> AlphaRouter { .flatten() .map(|(k, v)| { sync.shared_update( - sync::tag::SyncId { + prisma_sync::tag::SyncId { pub_id: tag.pub_id.clone(), }, k, diff --git a/core/src/job/report.rs b/core/src/job/report.rs index 9f89e4130097..862fefd06de7 100644 --- a/core/src/job/report.rs +++ b/core/src/job/report.rs @@ -1,7 +1,7 @@ use crate::{ library::Library, prisma::job, - util::db::{chain_optional_iter, maybe_missing, MissingFieldError}, + util::db::{maybe_missing, MissingFieldError}, }; use std::fmt::{Display, Formatter}; @@ -203,7 +203,7 @@ impl JobReport { .job() .create( self.id.as_bytes().to_vec(), - chain_optional_iter( + sd_utils::chain_optional_iter( [ job::name::set(Some(self.name.clone())), job::action::set(self.action.clone()), diff --git a/core/src/lib.rs b/core/src/lib.rs index c813469e6c15..5fa41baac872 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -39,7 +39,6 @@ pub(crate) mod node; pub(crate) mod object; pub(crate) mod p2p; pub(crate) mod preferences; -pub(crate) mod sync; pub(crate) mod util; pub(crate) mod volume; @@ -47,6 +46,7 @@ pub struct NodeContext { pub config: Arc, pub job_manager: Arc, pub location_manager: Arc, + pub p2p: Arc, pub event_bus_tx: broadcast::Sender, pub notifications: Arc, } @@ -81,28 +81,32 @@ impl Node { debug!("Initialised 'NodeConfigManager'..."); let job_manager = JobManager::new(); - debug!("Initialised 'JobManager'..."); let notifications = NotificationManager::new(); + debug!("Initialised 'NotificationManager'..."); let location_manager = LocationManager::new(); debug!("Initialised 'LocationManager'..."); + + let (p2p, p2p_stream) = P2PManager::new(config.clone()).await?; + debug!("Initialised 'P2PManager'..."); + let library_manager = LibraryManager::new( data_dir.join("libraries"), Arc::new(NodeContext { config: config.clone(), job_manager: job_manager.clone(), location_manager: location_manager.clone(), - // p2p: p2p.clone(), + p2p: p2p.clone(), event_bus_tx: event_bus.0.clone(), notifications: notifications.clone(), }), ) .await?; debug!("Initialised 'LibraryManager'..."); - let p2p = P2PManager::new(config.clone(), library_manager.clone()).await?; - debug!("Initialised 'P2PManager'..."); + + p2p.start(p2p_stream, library_manager.clone()); #[cfg(debug_assertions)] if let Some(init_data) = init_data { diff --git a/core/src/library/config.rs b/core/src/library/config.rs index 6e0d0fd4c0d3..288bd07e6f9b 100644 --- a/core/src/library/config.rs +++ b/core/src/library/config.rs @@ -2,7 +2,7 @@ use crate::{ node::{NodeConfig, Platform}, prisma::{file_path, indexer_rule, PrismaClient}, util::{ - db::{maybe_missing, uuid_to_bytes}, + db::maybe_missing, migrator::{Migrate, MigratorError}, }, }; @@ -65,9 +65,9 @@ impl Migrate for LibraryConfig { .map(|(i, name)| { db.indexer_rule().update_many( vec![indexer_rule::name::equals(Some(name))], - vec![indexer_rule::pub_id::set(uuid_to_bytes(Uuid::from_u128( - i as u128, - )))], + vec![indexer_rule::pub_id::set(sd_utils::uuid_to_bytes( + Uuid::from_u128(i as u128), + ))], ) }) .collect::>(), @@ -185,6 +185,7 @@ impl Migrate for LibraryConfig { node_platform: Platform::current() as i32, last_seen: now, date_created: node.map(|n| n.date_created).unwrap_or_else(|| now), + // timestamp: Default::default(), // TODO: Source this properly! _params: vec![], } .to_query(db) diff --git a/core/src/library/library.rs b/core/src/library/library.rs index 16c3eabd667a..63aa7b444327 100644 --- a/core/src/library/library.rs +++ b/core/src/library/library.rs @@ -9,11 +9,11 @@ use crate::{ }, node::NodeConfigManager, object::{ - orphan_remover::OrphanRemoverActor, preview::get_thumbnail_path, + orphan_remover::OrphanRemoverActor, + preview::{get_thumbnail_path, THUMBNAIL_CACHE_DIR_NAME}, thumbnail_remover::ThumbnailRemoverActor, }, prisma::{file_path, location, PrismaClient}, - sync::SyncManager, util::{db::maybe_missing, error::FileIOError}, NodeContext, }; @@ -26,13 +26,14 @@ use std::{ }; use chrono::{DateTime, Utc}; +use sd_core_sync::{SyncManager, SyncMessage}; use sd_p2p::spacetunnel::Identity; use sd_prisma::prisma::notification; use tokio::{fs, io}; use tracing::warn; use uuid::Uuid; -use super::{LibraryConfig, LibraryManagerError}; +use super::{LibraryConfig, LibraryManager, LibraryManagerError}; /// LibraryContext holds context for a library which can be passed around the application. pub struct Library { @@ -42,7 +43,7 @@ pub struct Library { pub config: LibraryConfig, /// db holds the database client for the current library. pub db: Arc, - pub sync: Arc, + pub sync: Arc, /// key manager that provides encryption keys to functions that require them // pub key_manager: Arc, /// node_context holds the node context for the node which this library is running on. @@ -66,6 +67,53 @@ impl Debug for Library { } impl Library { + pub fn new( + id: Uuid, + instance_id: Uuid, + config: LibraryConfig, + identity: Arc, + db: Arc, + library_manager: Arc, + // node_context: Arc, + ) -> Self { + let (sync_manager, mut sync_rx) = SyncManager::new(&db, instance_id); + let node_context = library_manager.node_context.clone(); + + let library = Self { + orphan_remover: OrphanRemoverActor::spawn(db.clone()), + thumbnail_remover: ThumbnailRemoverActor::spawn( + db.clone(), + node_context + .config + .data_directory() + .join(THUMBNAIL_CACHE_DIR_NAME), + ), + id, + db, + config, + node_context, + // key_manager, + sync: Arc::new(sync_manager), + identity: identity.clone(), + }; + + tokio::spawn({ + async move { + while let Ok(op) = sync_rx.recv().await { + let SyncMessage::Created(op) = op else { continue; }; + + library_manager + .node_context + .p2p + .broadcast_sync_events(id, &identity, vec![op], &library_manager) + .await; + } + } + }); + + library + } + pub(crate) fn emit(&self, event: CoreEvent) { if let Err(e) = self.node_context.event_bus_tx.send(event) { warn!("Error sending event to event bus: {e:?}"); diff --git a/core/src/library/manager.rs b/core/src/library/manager.rs index f38ea94fd413..bd8f18ecaabe 100644 --- a/core/src/library/manager.rs +++ b/core/src/library/manager.rs @@ -2,12 +2,8 @@ use crate::{ invalidate_query, location::{indexer, LocationManagerError}, node::{NodeConfig, Platform}, - object::{ - orphan_remover::OrphanRemoverActor, preview::THUMBNAIL_CACHE_DIR_NAME, tag, - thumbnail_remover::ThumbnailRemoverActor, - }, + object::tag, prisma::location, - sync::{SyncManager, SyncMessage}, util::{ db::{self, MissingFieldError}, error::{FileIOError, NonUtf8PathError}, @@ -27,33 +23,12 @@ use chrono::Utc; use sd_p2p::spacetunnel::{Identity, IdentityErr}; use sd_prisma::prisma::instance; use thiserror::Error; -use tokio::{ - fs, io, - sync::{broadcast, RwLock}, - try_join, -}; +use tokio::{fs, io, sync::RwLock, try_join}; use tracing::{debug, error, info, warn}; use uuid::Uuid; use super::{Library, LibraryConfig, LibraryConfigWrapped, LibraryName}; -pub enum SubscriberEvent { - Load(Uuid, Arc, broadcast::Receiver), -} - -impl Clone for SubscriberEvent { - fn clone(&self) -> Self { - match self { - Self::Load(id, identity, receiver) => { - Self::Load(*id, identity.clone(), receiver.resubscribe()) - } - } - } -} - -pub trait SubscriberFn: Fn(SubscriberEvent) + Send + Sync + 'static {} -impl SubscriberFn for F {} - /// LibraryManager is a singleton that manages all libraries for a node. pub struct LibraryManager { /// libraries_dir holds the path to the directory where libraries are stored. @@ -61,9 +36,7 @@ pub struct LibraryManager { /// libraries holds the list of libraries which are currently loaded into the node. libraries: RwLock>>, /// node_context holds the context for the node which this library manager is running on. - node_context: Arc, - /// on load subscribers - subscribers: RwLock>>, + pub node_context: Arc, } #[derive(Error, Debug)] @@ -121,12 +94,16 @@ impl LibraryManager { .await .map_err(|e| FileIOError::from((&libraries_dir, e)))?; - let mut libraries = Vec::new(); - let subscribers = RwLock::new(Vec::new()); let mut read_dir = fs::read_dir(&libraries_dir) .await .map_err(|e| FileIOError::from((&libraries_dir, e)))?; + let this = Arc::new(Self { + libraries_dir: libraries_dir.clone(), + libraries: Default::default(), + node_context, + }); + while let Some(entry) = read_dir .next_entry() .await @@ -164,44 +141,17 @@ impl LibraryManager { Err(e) => return Err(FileIOError::from((db_path, e)).into()), } - libraries.push( - Self::load( - library_id, - &db_path, - config_path, - node_context.clone(), - &subscribers, - None, - true, - ) - .await?, - ); + this.load(library_id, &db_path, config_path, None, true) + .await?; } } - Ok(Arc::new(Self { - libraries: RwLock::new(libraries), - libraries_dir, - node_context, - subscribers, - })) - } - - /// subscribe to library events - pub(crate) async fn subscribe(&self, f: F) { - self.subscribers.write().await.push(Box::new(f)); - } - - async fn emit(subscribers: &RwLock>>, event: SubscriberEvent) { - let subscribers = subscribers.read().await; - for subscriber in subscribers.iter() { - subscriber(event.clone()); - } + Ok(this) } /// create creates a new library with the given config and mounts it into the running [LibraryManager]. pub(crate) async fn create( - &self, + self: &Arc, name: LibraryName, description: Option, node_cfg: NodeConfig, @@ -211,7 +161,7 @@ impl LibraryManager { } pub(crate) async fn create_with_uuid( - &self, + self: &Arc, id: Uuid, name: LibraryName, description: Option, @@ -240,25 +190,25 @@ impl LibraryManager { ); let now = Utc::now().fixed_offset(); - let library = Self::load( - id, - self.libraries_dir.join(format!("{id}.db")), - config_path, - self.node_context.clone(), - &self.subscribers, - Some(instance::Create { - pub_id: Uuid::new_v4().as_bytes().to_vec(), - identity: Identity::new().to_bytes(), - node_id: node_cfg.id.as_bytes().to_vec(), - node_name: node_cfg.name.clone(), - node_platform: Platform::current() as i32, - last_seen: now, - date_created: now, - _params: vec![instance::id::set(config.instance_id)], - }), - should_seed, - ) - .await?; + let library = self + .load( + id, + self.libraries_dir.join(format!("{id}.db")), + config_path, + Some(instance::Create { + pub_id: Uuid::new_v4().as_bytes().to_vec(), + identity: Identity::new().to_bytes(), + node_id: node_cfg.id.as_bytes().to_vec(), + node_name: node_cfg.name.clone(), + node_platform: Platform::current() as i32, + last_seen: now, + date_created: now, + // timestamp: Default::default(), // TODO: Source this properly! + _params: vec![instance::id::set(config.instance_id)], + }), + should_seed, + ) + .await?; debug!("Loaded library '{id:?}'"); @@ -270,8 +220,6 @@ impl LibraryManager { invalidate_query!(library, "library.list"); - self.libraries.write().await.push(library); - debug!("Pushed library into manager '{id:?}'"); Ok(LibraryConfigWrapped { uuid: id, config }) @@ -293,9 +241,9 @@ impl LibraryManager { .collect() } - pub(crate) async fn get_all_instances(&self) -> Vec { - vec![] // TODO: Cache in memory - } + // pub(crate) async fn get_all_instances(&self) -> Vec { + // vec![] // TODO: Cache in memory + // } pub(crate) async fn edit( &self, @@ -396,11 +344,10 @@ impl LibraryManager { /// load the library from a given path async fn load( + self: &Arc, id: Uuid, db_path: impl AsRef, config_path: PathBuf, - node_context: Arc, - subscribers: &RwLock>>, create: Option, should_seed: bool, ) -> Result, LibraryManagerError> { @@ -417,7 +364,7 @@ impl LibraryManager { create.to_query(&db).exec().await?; } - let node_config = node_context.config.get().await; + let node_config = self.node_context.config.get().await; let config = LibraryConfig::load_and_migrate(&config_path, &(node_config.clone(), db.clone())) .await?; @@ -462,31 +409,15 @@ impl LibraryManager { // let key_manager = Arc::new(KeyManager::new(vec![]).await?); // seed_keymanager(&db, &key_manager).await?; - let (sync_manager, sync_rx) = SyncManager::new(&db, instance_id); - - Self::emit( - subscribers, - SubscriberEvent::Load(id, identity.clone(), sync_rx), - ) - .await; - - let library = Arc::new(Library { + let library = Arc::new(Library::new( id, + instance_id, config, + identity, // key_manager, - sync: Arc::new(sync_manager), - orphan_remover: OrphanRemoverActor::spawn(db.clone()), - thumbnail_remover: ThumbnailRemoverActor::spawn( - db.clone(), - node_context - .config - .data_directory() - .join(THUMBNAIL_CACHE_DIR_NAME), - ), db, - node_context, - identity, - }); + self.clone(), + )); if should_seed { library.orphan_remover.invoke().await; diff --git a/core/src/location/file_path_helper/mod.rs b/core/src/location/file_path_helper/mod.rs index 082aa2925156..66c9f5ab4356 100644 --- a/core/src/location/file_path_helper/mod.rs +++ b/core/src/location/file_path_helper/mod.rs @@ -182,9 +182,8 @@ pub async fn create_file_path( metadata: FilePathMetadata, ) -> Result { use crate::util::db::{device_to_db, inode_to_db}; - use crate::{sync, util::db::uuid_to_bytes}; - use sd_prisma::prisma; + use sd_prisma::{prisma, prisma_sync}; use sd_sync::OperationFactory; use serde_json::json; use uuid::Uuid; @@ -203,7 +202,7 @@ pub async fn create_file_path( vec![ ( location::NAME, - json!(sync::location::SyncId { + json!(prisma_sync::location::SyncId { pub_id: location.pub_id }), ), @@ -223,14 +222,14 @@ pub async fn create_file_path( ] }; - let pub_id = uuid_to_bytes(Uuid::new_v4()); + let pub_id = sd_utils::uuid_to_bytes(Uuid::new_v4()); let created_path = sync .write_ops( db, ( sync.shared_create( - sync::file_path::SyncId { + prisma_sync::file_path::SyncId { pub_id: pub_id.clone(), }, params, diff --git a/core/src/location/indexer/mod.rs b/core/src/location/indexer/mod.rs index f8449462dfe4..7a0497e7c277 100644 --- a/core/src/location/indexer/mod.rs +++ b/core/src/location/indexer/mod.rs @@ -1,9 +1,8 @@ use crate::{ library::Library, prisma::{file_path, location, PrismaClient}, - sync, util::{ - db::{device_to_db, inode_to_db, uuid_to_bytes}, + db::{device_to_db, inode_to_db}, error::FileIOError, }, }; @@ -12,6 +11,7 @@ use std::path::Path; use chrono::Utc; use rspc::ErrorCode; +use sd_prisma::prisma_sync; use sd_sync::*; use serde::{Deserialize, Serialize}; use serde_json::json; @@ -103,13 +103,13 @@ async fn execute_indexer_save_step( use file_path::*; - let pub_id = uuid_to_bytes(entry.pub_id); + let pub_id = sd_utils::uuid_to_bytes(entry.pub_id); let (sync_params, db_params): (Vec<_>, Vec<_>) = [ ( ( location::NAME, - json!(sync::location::SyncId { + json!(prisma_sync::location::SyncId { pub_id: pub_id.clone() }), ), @@ -160,8 +160,8 @@ async fn execute_indexer_save_step( ( sync.shared_create( - sync::file_path::SyncId { - pub_id: uuid_to_bytes(entry.pub_id), + prisma_sync::file_path::SyncId { + pub_id: sd_utils::uuid_to_bytes(entry.pub_id), }, sync_params, ), @@ -199,7 +199,7 @@ async fn execute_indexer_update_step( use file_path::*; - let pub_id = uuid_to_bytes(entry.pub_id); + let pub_id = sd_utils::uuid_to_bytes(entry.pub_id); let (sync_params, db_params): (Vec<_>, Vec<_>) = [ // As this file was updated while Spacedrive was offline, we mark the object_id as null @@ -243,7 +243,7 @@ async fn execute_indexer_update_step( .into_iter() .map(|(field, value)| { sync.shared_update( - sync::file_path::SyncId { + prisma_sync::file_path::SyncId { pub_id: pub_id.clone(), }, field, diff --git a/core/src/location/indexer/rules/mod.rs b/core/src/location/indexer/rules/mod.rs index 2b286c123b41..449c57a9be19 100644 --- a/core/src/location/indexer/rules/mod.rs +++ b/core/src/location/indexer/rules/mod.rs @@ -4,7 +4,7 @@ use crate::{ library::Library, prisma::indexer_rule, util::{ - db::{maybe_missing, uuid_to_bytes, MissingFieldError}, + db::{maybe_missing, MissingFieldError}, error::{FileIOError, NonUtf8PathError}, }, }; @@ -135,7 +135,7 @@ impl IndexerRuleCreateArgs { .db .indexer_rule() .create( - uuid_to_bytes(generate_pub_id()), + sd_utils::uuid_to_bytes(generate_pub_id()), vec![ name::set(Some(self.name)), rules_per_kind::set(Some(rules_data)), diff --git a/core/src/location/indexer/rules/seed.rs b/core/src/location/indexer/rules/seed.rs index 40489569aacd..d2ed421b1b3c 100644 --- a/core/src/location/indexer/rules/seed.rs +++ b/core/src/location/indexer/rules/seed.rs @@ -1,7 +1,6 @@ use crate::{ library::Library, location::indexer::rules::{IndexerRuleError, RulePerKind}, - util::db::uuid_to_bytes, }; use chrono::Utc; use sd_prisma::prisma::indexer_rule; @@ -29,7 +28,7 @@ pub async fn new_or_existing_library(library: &Library) -> Result<(), SeederErro .into_iter() .enumerate() { - let pub_id = uuid_to_bytes(Uuid::from_u128(i as u128)); + let pub_id = sd_utils::uuid_to_bytes(Uuid::from_u128(i as u128)); let rules = rmp_serde::to_vec_named(&rule.rules).map_err(IndexerRuleError::from)?; use indexer_rule::*; diff --git a/core/src/location/indexer/walk.rs b/core/src/location/indexer/walk.rs index cd64a98c9797..85b370055700 100644 --- a/core/src/location/indexer/walk.rs +++ b/core/src/location/indexer/walk.rs @@ -5,7 +5,7 @@ use crate::{ }, prisma::file_path, util::{ - db::{device_from_db, from_bytes_to_uuid, inode_from_db}, + db::{device_from_db, inode_from_db}, error::FileIOError, }, }; @@ -385,7 +385,9 @@ where ) - *date_modified > Duration::milliseconds(1) { - to_update.push((from_bytes_to_uuid(&file_path.pub_id), entry).into()); + to_update.push( + (sd_utils::from_bytes_to_uuid(&file_path.pub_id), entry).into(), + ); } } diff --git a/core/src/location/manager/watcher/utils.rs b/core/src/location/manager/watcher/utils.rs index 2ec9868505d2..aab522331d64 100644 --- a/core/src/location/manager/watcher/utils.rs +++ b/core/src/location/manager/watcher/utils.rs @@ -20,7 +20,6 @@ use crate::{ validation::hash::file_checksum, }, prisma::{file_path, location, object}, - sync::{self, OperationFactory}, util::{ db::{device_from_db, device_to_db, inode_from_db, inode_to_db, maybe_missing}, error::FileIOError, @@ -47,6 +46,8 @@ use sd_file_ext::extensions::ImageExtension; use chrono::{DateTime, Local, Utc}; use notify::Event; use prisma_client_rust::{raw, PrismaValue}; +use sd_prisma::prisma_sync; +use sd_sync::OperationFactory; use serde_json::json; use tokio::{fs, io::ErrorKind}; use tracing::{debug, error, trace, warn}; @@ -508,7 +509,7 @@ async fn inner_update_file( .into_iter() .map(|(field, value)| { sync.shared_update( - sync::file_path::SyncId { + prisma_sync::file_path::SyncId { pub_id: file_path.pub_id.clone(), }, field, @@ -544,7 +545,7 @@ async fn inner_update_file( sync.write_op( db, sync.shared_update( - sync::object::SyncId { + prisma_sync::object::SyncId { pub_id: object.pub_id.clone(), }, object::kind::NAME, diff --git a/core/src/location/mod.rs b/core/src/location/mod.rs index dacdd45a090a..004e5b5da77d 100644 --- a/core/src/location/mod.rs +++ b/core/src/location/mod.rs @@ -8,8 +8,7 @@ use crate::{ preview::{shallow_thumbnailer, thumbnailer_job::ThumbnailerJobInit}, }, prisma::{file_path, indexer_rules_in_location, location, PrismaClient}, - sync, - util::{db::chain_optional_iter, error::FileIOError}, + util::error::FileIOError, }; use std::{ @@ -22,6 +21,7 @@ use chrono::Utc; use futures::future::TryFutureExt; use normpath::PathExt; use prisma_client_rust::{operator::and, or, QueryError}; +use sd_prisma::prisma_sync; use sd_sync::*; use serde::Deserialize; use serde_json::json; @@ -274,7 +274,7 @@ impl LocationUpdateArgs { .into_iter() .map(|p| { sync.shared_update( - sync::location::SyncId { + prisma_sync::location::SyncId { pub_id: location.pub_id.clone(), }, p.0, @@ -483,7 +483,7 @@ pub async fn relink_location( sync.write_op( db, sync.shared_update( - sync::location::SyncId { + prisma_sync::location::SyncId { pub_id: pub_id.clone(), }, location::path::NAME, @@ -588,7 +588,7 @@ async fn create_location( db, ( sync.shared_create( - sync::location::SyncId { + prisma_sync::location::SyncId { pub_id: location_pub_id.as_bytes().to_vec(), }, [ @@ -597,7 +597,7 @@ async fn create_location( (location::date_created::NAME, json!(date_created)), ( location::instance_id::NAME, - json!(sync::instance::SyncId { + json!(prisma_sync::instance::SyncId { pub_id: vec![], // id: library.config.instance_id, }), @@ -697,7 +697,7 @@ pub async fn delete_directory( ) -> Result<(), QueryError> { let Library { db, .. } = library; - let children_params = chain_optional_iter( + let children_params = sd_utils::chain_optional_iter( [file_path::location_id::equals(Some(location_id))], [parent_iso_file_path.and_then(|parent| { parent diff --git a/core/src/object/file_identifier/file_identifier_job.rs b/core/src/object/file_identifier/file_identifier_job.rs index 56a3feb949c4..96798e709f49 100644 --- a/core/src/object/file_identifier/file_identifier_job.rs +++ b/core/src/object/file_identifier/file_identifier_job.rs @@ -9,7 +9,7 @@ use crate::{ file_path_for_file_identifier, IsolatedFilePathData, }, prisma::{file_path, location, PrismaClient, SortOrder}, - util::db::{chain_optional_iter, maybe_missing}, + util::db::maybe_missing, }; use std::{ @@ -238,7 +238,7 @@ fn orphan_path_filters( file_path_id: Option, maybe_sub_iso_file_path: &Option>, ) -> Vec { - chain_optional_iter( + sd_utils::chain_optional_iter( [ file_path::object_id::equals(None), file_path::is_dir::equals(Some(false)), diff --git a/core/src/object/file_identifier/mod.rs b/core/src/object/file_identifier/mod.rs index 6934599c76e3..d0de8b1f5cba 100644 --- a/core/src/object/file_identifier/mod.rs +++ b/core/src/object/file_identifier/mod.rs @@ -6,14 +6,13 @@ use crate::{ }, object::{cas::generate_cas_id, object_for_file_identifier}, prisma::{file_path, location, object, PrismaClient}, - sync::{self, CRDTOperation, OperationFactory, SyncManager}, - util::{ - db::{maybe_missing, uuid_to_bytes}, - error::FileIOError, - }, + util::{db::maybe_missing, error::FileIOError}, }; +use sd_core_sync::SyncManager; use sd_file_ext::{extensions::Extension, kind::ObjectKind}; +use sd_prisma::prisma_sync; +use sd_sync::{CRDTOperation, OperationFactory}; use std::{ collections::{HashMap, HashSet}, @@ -138,14 +137,14 @@ async fn identifier_job_step( .map(|(pub_id, (meta, _))| { ( sync.shared_update( - sync::file_path::SyncId { - pub_id: uuid_to_bytes(*pub_id), + prisma_sync::file_path::SyncId { + pub_id: sd_utils::uuid_to_bytes(*pub_id), }, file_path::cas_id::NAME, json!(&meta.cas_id), ), db.file_path().update( - file_path::pub_id::equals(uuid_to_bytes(*pub_id)), + file_path::pub_id::equals(sd_utils::uuid_to_bytes(*pub_id)), vec![file_path::cas_id::set(Some(meta.cas_id.clone()))], ), ) @@ -230,8 +229,8 @@ async fn identifier_job_step( .map(|(file_path_pub_id, (meta, fp))| { let object_pub_id = Uuid::new_v4(); - let sync_id = || sync::object::SyncId { - pub_id: uuid_to_bytes(object_pub_id), + let sync_id = || prisma_sync::object::SyncId { + pub_id: sd_utils::uuid_to_bytes(object_pub_id), }; let kind = meta.kind as i32; @@ -251,7 +250,7 @@ async fn identifier_job_step( let object_creation_args = ( sync.shared_create(sync_id(), sync_params), - object::create_unchecked(uuid_to_bytes(object_pub_id), db_params), + object::create_unchecked(sd_utils::uuid_to_bytes(object_pub_id), db_params), ); (object_creation_args, { @@ -319,16 +318,16 @@ fn file_path_object_connect_ops<'db>( ( sync.shared_update( - sync::file_path::SyncId { - pub_id: uuid_to_bytes(file_path_id), + prisma_sync::file_path::SyncId { + pub_id: sd_utils::uuid_to_bytes(file_path_id), }, file_path::object::NAME, - json!(sync::object::SyncId { + json!(prisma_sync::object::SyncId { pub_id: vec_id.clone() }), ), db.file_path().update( - file_path::pub_id::equals(uuid_to_bytes(file_path_id)), + file_path::pub_id::equals(sd_utils::uuid_to_bytes(file_path_id)), vec![file_path::object::connect(object::pub_id::equals(vec_id))], ), ) diff --git a/core/src/object/file_identifier/shallow.rs b/core/src/object/file_identifier/shallow.rs index d08807bc82e5..2fd0f4f1ac38 100644 --- a/core/src/object/file_identifier/shallow.rs +++ b/core/src/object/file_identifier/shallow.rs @@ -7,7 +7,7 @@ use crate::{ file_path_for_file_identifier, IsolatedFilePathData, }, prisma::{file_path, location, PrismaClient, SortOrder}, - util::db::{chain_optional_iter, maybe_missing}, + util::db::maybe_missing, }; use std::path::{Path, PathBuf}; @@ -120,7 +120,7 @@ fn orphan_path_filters( file_path_id: Option, sub_iso_file_path: &IsolatedFilePathData<'_>, ) -> Vec { - chain_optional_iter( + sd_utils::chain_optional_iter( [ file_path::object_id::equals(None), file_path::is_dir::equals(Some(false)), diff --git a/core/src/object/tag/mod.rs b/core/src/object/tag/mod.rs index 70dce817485a..ae51cd5b7ac1 100644 --- a/core/src/object/tag/mod.rs +++ b/core/src/object/tag/mod.rs @@ -1,6 +1,7 @@ pub mod seed; use chrono::{DateTime, FixedOffset, Utc}; +use sd_prisma::prisma_sync; use sd_sync::*; use serde::Deserialize; use serde_json::json; @@ -8,7 +9,7 @@ use specta::Type; use uuid::Uuid; -use crate::{library::Library, prisma::tag, sync}; +use crate::{library::Library, prisma::tag}; #[derive(Type, Deserialize, Clone)] pub struct TagCreateArgs { @@ -28,7 +29,7 @@ impl TagCreateArgs { db, ( sync.shared_create( - sync::tag::SyncId { + prisma_sync::tag::SyncId { pub_id: pub_id.clone(), }, [ diff --git a/core/src/object/validation/validator_job.rs b/core/src/object/validation/validator_job.rs index b1ec3af4b7c0..c77038c59d32 100644 --- a/core/src/object/validation/validator_job.rs +++ b/core/src/object/validation/validator_job.rs @@ -8,11 +8,7 @@ use crate::{ file_path_for_object_validator, IsolatedFilePathData, }, prisma::{file_path, location}, - sync::{self, OperationFactory}, - util::{ - db::{chain_optional_iter, maybe_missing}, - error::FileIOError, - }, + util::{db::maybe_missing, error::FileIOError}, }; use std::{ @@ -20,6 +16,8 @@ use std::{ path::{Path, PathBuf}, }; +use sd_prisma::prisma_sync; +use sd_sync::OperationFactory; use serde::{Deserialize, Serialize}; use serde_json::json; use tracing::info; @@ -101,7 +99,7 @@ impl StatefulJob for ObjectValidatorJobInit { let steps = db .file_path() - .find_many(chain_optional_iter( + .find_many(sd_utils::chain_optional_iter( [ file_path::location_id::equals(Some(init.location.id)), file_path::is_dir::equals(Some(false)), @@ -153,7 +151,7 @@ impl StatefulJob for ObjectValidatorJobInit { sync.write_op( db, sync.shared_update( - sync::file_path::SyncId { + prisma_sync::file_path::SyncId { pub_id: file_path.pub_id.clone(), }, file_path::integrity_checksum::NAME, diff --git a/core/src/p2p/p2p_manager.rs b/core/src/p2p/p2p_manager.rs index 0d6c223849ce..dd28876279a0 100644 --- a/core/src/p2p/p2p_manager.rs +++ b/core/src/p2p/p2p_manager.rs @@ -10,7 +10,7 @@ use futures::Stream; use sd_p2p::{ spaceblock::{BlockSize, SpaceblockRequest, Transfer}, spacetunnel::{Identity, Tunnel}, - Event, Manager, ManagerError, MetadataManager, PeerId, + Event, Manager, ManagerError, ManagerStream, MetadataManager, PeerId, }; use sd_sync::CRDTOperation; use serde::Serialize; @@ -25,10 +25,9 @@ use tracing::{debug, error, info, warn}; use uuid::Uuid; use crate::{ - library::{LibraryManager, SubscriberEvent}, + library::LibraryManager, node::{NodeConfig, NodeConfigManager}, p2p::{OperatingSystem, SPACEDRIVE_APP_ID}, - sync::SyncMessage, }; use super::{Header, PairingManager, PairingStatus, PeerMetadata}; @@ -69,25 +68,23 @@ pub struct P2PManager { pub metadata_manager: Arc>, pub spacedrop_progress: Arc>>>, pub pairing: Arc, - library_manager: Arc, } impl P2PManager { pub async fn new( node_config: Arc, - library_manager: Arc, - ) -> Result, ManagerError> { + ) -> Result<(Arc, ManagerStream), ManagerError> { let (config, keypair) = { let config = node_config.get().await; ( - Self::config_to_metadata(&config, &library_manager).await, + Self::config_to_metadata(&config /* , &library_manager */).await, config.keypair, ) }; let metadata_manager = MetadataManager::new(config); - let (manager, mut stream) = + let (manager, stream) = Manager::new(SPACEDRIVE_APP_ID, &keypair, metadata_manager.clone()).await?; info!( @@ -102,13 +99,55 @@ impl P2PManager { let spacedrop_pairing_reqs = Arc::new(Mutex::new(HashMap::new())); let spacedrop_progress = Arc::new(Mutex::new(HashMap::new())); - let pairing = PairingManager::new(manager.clone(), tx.clone(), library_manager.clone()); + let pairing = PairingManager::new(manager.clone(), tx.clone()); + + // TODO: proper shutdown + // https://docs.rs/ctrlc/latest/ctrlc/ + // https://docs.rs/system_shutdown/latest/system_shutdown/ + + let this = Arc::new(Self { + pairing, + events: (tx, rx), + manager, + spacedrop_pairing_reqs, + metadata_manager, + spacedrop_progress, + }); + + // library_manager + // .subscribe({ + // let this = this.clone(); + // move |event| match event { + // SubscriberEvent::Load(library_id, library_identity, mut sync_rx) => { + // let this = this.clone(); + // } + // } + // }) + // .await; + + // TODO: Probs remove this once connection timeout/keepalive are working correctly tokio::spawn({ - let events = tx.clone(); - let spacedrop_pairing_reqs = spacedrop_pairing_reqs.clone(); - let spacedrop_progress = spacedrop_progress.clone(); - let library_manager = library_manager.clone(); - let pairing = pairing.clone(); + let this = this.clone(); + async move { + loop { + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + this.ping().await; + } + } + }); + + Ok((this, stream)) + } + pub fn start( + &self, + mut stream: ManagerStream, + library_manager: Arc, + ) { + tokio::spawn({ + let events = self.events.0.clone(); + let spacedrop_pairing_reqs = self.spacedrop_pairing_reqs.clone(); + let spacedrop_progress = self.spacedrop_progress.clone(); + let pairing = self.pairing.clone(); async move { let mut shutdown = false; @@ -204,7 +243,9 @@ impl P2PManager { }; } Header::Pair => { - pairing.responder(event.peer_id, stream).await; + pairing + .responder(event.peer_id, stream, library_manager) + .await; } Header::Sync(library_id) => { let mut stream = Tunnel::from_stream(stream).await.unwrap(); @@ -228,14 +269,12 @@ impl P2PManager { }; for op in operations { - library.sync.ingest_op(op).await.unwrap_or_else( - |err| { - error!( - "error ingesting operation for library '{}': {err:?}", - library.id - ); - }, - ); + library.sync.apply_op(op).await.unwrap_or_else(|err| { + error!( + "error ingesting operation for library '{}': {err:?}", + library.id + ); + }); } } } @@ -259,57 +298,11 @@ impl P2PManager { } } }); - - // TODO: proper shutdown - // https://docs.rs/ctrlc/latest/ctrlc/ - // https://docs.rs/system_shutdown/latest/system_shutdown/ - - let this = Arc::new(Self { - pairing, - events: (tx, rx), - manager, - spacedrop_pairing_reqs, - metadata_manager, - spacedrop_progress, - library_manager: library_manager.clone(), - }); - - library_manager - .subscribe({ - let this = this.clone(); - move |event| match event { - SubscriberEvent::Load(library_id, library_identity, mut sync_rx) => { - let this = this.clone(); - tokio::spawn(async move { - while let Ok(op) = sync_rx.recv().await { - let SyncMessage::Created(op) = op else { continue; }; - - this.broadcast_sync_events(library_id, &library_identity, vec![op]) - .await; - } - }); - } - } - }) - .await; - - // TODO: Probs remove this once connection timeout/keepalive are working correctly - tokio::spawn({ - let this = this.clone(); - async move { - loop { - tokio::time::sleep(std::time::Duration::from_secs(5)).await; - this.ping().await; - } - } - }); - - Ok(this) } async fn config_to_metadata( config: &NodeConfig, - library_manager: &LibraryManager, + // library_manager: &LibraryManager, ) -> PeerMetadata { PeerMetadata { name: config.name.clone(), @@ -317,24 +310,27 @@ impl P2PManager { version: Some(env!("CARGO_PKG_VERSION").to_string()), email: config.p2p_email.clone(), img_url: config.p2p_img_url.clone(), - instances: library_manager - .get_all_instances() - .await - .into_iter() - .filter_map(|i| { - Identity::from_bytes(&i.identity) - .map(|i| hex::encode(i.public_key().to_bytes())) - .ok() - }) - .collect(), + // instances: library_manager + // .get_all_instances() + // .await + // .into_iter() + // .filter_map(|i| { + // Identity::from_bytes(&i.identity) + // .map(|i| hex::encode(i.public_key().to_bytes())) + // .ok() + // }) + // .collect(), } } #[allow(unused)] // TODO: Should probs be using this - pub async fn update_metadata(&self, node_config_manager: &NodeConfigManager) { - self.metadata_manager.update( - Self::config_to_metadata(&node_config_manager.get().await, &self.library_manager).await, - ); + pub async fn update_metadata( + &self, + node_config_manager: &NodeConfigManager, + library_manager: &LibraryManager, + ) { + self.metadata_manager + .update(Self::config_to_metadata(&node_config_manager.get().await).await); } pub async fn accept_spacedrop(&self, id: Uuid, path: String) { @@ -358,7 +354,10 @@ impl P2PManager { library_id: Uuid, _identity: &Identity, event: Vec, + library_manager: &LibraryManager, ) { + println!("broadcasting sync events!"); + let mut buf = match rmp_serde::to_vec_named(&event) { Ok(buf) => buf, Err(e) => { @@ -374,7 +373,7 @@ impl P2PManager { // TODO: Establish a connection to them - let _library = self.library_manager.get_library(library_id).await.unwrap(); + let _library = library_manager.get_library(library_id).await.unwrap(); todo!(); diff --git a/core/src/p2p/pairing/mod.rs b/core/src/p2p/pairing/mod.rs index 9d1ba8666458..70809cf98ac2 100644 --- a/core/src/p2p/pairing/mod.rs +++ b/core/src/p2p/pairing/mod.rs @@ -38,21 +38,21 @@ pub struct PairingManager { events_tx: broadcast::Sender, pairing_response: RwLock>>, manager: Arc>, - library_manager: Arc, + // library_manager: Arc, } impl PairingManager { pub fn new( manager: Arc>, events_tx: broadcast::Sender, - library_manager: Arc, + // library_manager: Arc, ) -> Arc { Arc::new(Self { id: AtomicU16::new(0), events_tx, pairing_response: RwLock::new(HashMap::new()), manager, - library_manager, + // library_manager, }) } @@ -70,7 +70,12 @@ impl PairingManager { // TODO: Error handling - pub async fn originator(self: Arc, peer_id: PeerId, node_config: NodeConfig) -> u16 { + pub async fn originator( + self: Arc, + peer_id: PeerId, + node_config: NodeConfig, + library_manager: Arc, + ) -> u16 { // TODO: Timeout for max number of pairings in a time period let pairing_id = self.id.fetch_add(1, Ordering::SeqCst); @@ -119,8 +124,7 @@ impl PairingManager { // TODO: Future - Library in pairing state // TODO: Create library - if self - .library_manager + if library_manager .get_all_libraries() .await .into_iter() @@ -134,8 +138,7 @@ impl PairingManager { return; } - let library_config = self - .library_manager + let library_config = library_manager .create_with_uuid( library_id, LibraryName::new(library_name).unwrap(), @@ -145,8 +148,7 @@ impl PairingManager { ) .await .unwrap(); - let library = self - .library_manager + let library = library_manager .get_library(library_config.uuid) .await .unwrap(); @@ -207,6 +209,7 @@ impl PairingManager { self: Arc, peer_id: PeerId, mut stream: impl AsyncRead + AsyncWrite + Unpin, + library_manager: Arc, ) { let pairing_id = self.id.fetch_add(1, Ordering::SeqCst); self.emit_progress(pairing_id, PairingStatus::EstablishingConnection); @@ -239,7 +242,7 @@ impl PairingManager { }; info!("The user accepted pairing '{pairing_id}' for library '{library_id}'!"); - let library = self.library_manager.get_library(library_id).await.unwrap(); + let library = library_manager.get_library(library_id).await.unwrap(); stream .write_all( &PairingResponse::Accepted { diff --git a/core/src/p2p/pairing/proto.rs b/core/src/p2p/pairing/proto.rs index 2019e6092fb0..8d1c8d3612f5 100644 --- a/core/src/p2p/pairing/proto.rs +++ b/core/src/p2p/pairing/proto.rs @@ -40,6 +40,7 @@ impl From for instance::CreateUnchecked { node_platform: i.node_platform as i32, last_seen: i.last_seen.into(), date_created: i.date_created.into(), + // timestamp: Default::default(), // TODO: Source this properly! _params: vec![], } } diff --git a/core/src/p2p/peer_metadata.rs b/core/src/p2p/peer_metadata.rs index 6aaa673cfdf4..ccddfca80573 100644 --- a/core/src/p2p/peer_metadata.rs +++ b/core/src/p2p/peer_metadata.rs @@ -1,6 +1,5 @@ use std::{collections::HashMap, env, str::FromStr}; -use itertools::Itertools; use sd_p2p::Metadata; use serde::{Deserialize, Serialize}; use specta::Type; @@ -14,7 +13,7 @@ pub struct PeerMetadata { pub(super) version: Option, pub(super) email: Option, pub(super) img_url: Option, - pub(super) instances: Vec, + // pub(super) instances: Vec, } impl Metadata for PeerMetadata { @@ -33,7 +32,7 @@ impl Metadata for PeerMetadata { if let Some(img_url) = self.img_url { map.insert("img_url".to_owned(), img_url); } - map.insert("instances".to_owned(), self.instances.into_iter().join(",")); + // map.insert("instances".to_owned(), self.instances.into_iter().join(",")); map } @@ -56,15 +55,15 @@ impl Metadata for PeerMetadata { version: data.get("version").map(|v| v.to_owned()), email: data.get("email").map(|v| v.to_owned()), img_url: data.get("img_url").map(|v| v.to_owned()), - instances: data - .get("instances") - .ok_or_else(|| { - "DNS record for field 'instances' missing. Unable to decode 'PeerMetadata'!" - .to_owned() - })? - .split(',') - .map(|s| s.parse().map_err(|_| "Unable to parse instance 'Uuid'!")) - .collect::, _>>()?, + // instances: data + // .get("instances") + // .ok_or_else(|| { + // "DNS record for field 'instances' missing. Unable to decode 'PeerMetadata'!" + // .to_owned() + // })? + // .split(',') + // .map(|s| s.parse().map_err(|_| "Unable to parse instance 'Uuid'!")) + // .collect::, _>>()?, }) } } diff --git a/core/src/preferences/kv.rs b/core/src/preferences/kv.rs index f7927a239633..8ca78875d1cd 100644 --- a/core/src/preferences/kv.rs +++ b/core/src/preferences/kv.rs @@ -107,7 +107,7 @@ impl PreferenceKVs { self } - pub fn to_upserts(self, db: &PrismaClient) -> Vec { + pub fn into_upserts(self, db: &PrismaClient) -> Vec { self.0 .into_iter() .map(|(key, value)| { diff --git a/core/src/preferences/mod.rs b/core/src/preferences/mod.rs index 29b3465b400a..7dabf1d6166e 100644 --- a/core/src/preferences/mod.rs +++ b/core/src/preferences/mod.rs @@ -25,7 +25,7 @@ impl LibraryPreferences { pub async fn write(self, db: &PrismaClient) -> prisma_client_rust::Result<()> { let kvs = self.to_kvs(); - db._batch(kvs.to_upserts(db)).await?; + db._batch(kvs.into_upserts(db)).await?; Ok(()) } diff --git a/core/src/sync/manager.rs b/core/src/sync/manager.rs deleted file mode 100644 index 76039a864f7d..000000000000 --- a/core/src/sync/manager.rs +++ /dev/null @@ -1,220 +0,0 @@ -#![allow(clippy::unwrap_used, clippy::panic)] // TODO: Brendan remove this once you've got error handling here - -use crate::prisma::*; -use sd_sync::*; - -use std::{collections::HashMap, sync::Arc}; - -use serde_json::to_vec; -use tokio::sync::broadcast::{self, Receiver, Sender}; -use uhlc::{HLCBuilder, HLC, NTP64}; -use uuid::Uuid; - -use super::ModelSyncData; - -#[derive(Clone)] -pub enum SyncMessage { - Ingested(CRDTOperation), - Created(CRDTOperation), -} - -pub struct SyncManager { - db: Arc, - instance: Uuid, - _clocks: HashMap, - clock: HLC, - pub tx: Sender, -} - -impl SyncManager { - pub fn new(db: &Arc, instance: Uuid) -> (Self, Receiver) { - let (tx, rx) = broadcast::channel(64); - - ( - Self { - db: db.clone(), - instance, - clock: HLCBuilder::new().with_id(instance.into()).build(), - _clocks: Default::default(), - tx, - }, - rx, - ) - } - - pub async fn write_ops<'item, I: prisma_client_rust::BatchItem<'item>>( - &self, - tx: &PrismaClient, - (_ops, queries): (Vec, I), - ) -> prisma_client_rust::Result<::ReturnValue> { - #[cfg(feature = "sync-messages")] - let res = { - let shared = _ops - .iter() - .filter_map(|op| match &op.typ { - CRDTOperationType::Shared(shared_op) => { - let kind = match &shared_op.data { - SharedOperationData::Create => "c", - SharedOperationData::Update { .. } => "u", - SharedOperationData::Delete => "d", - }; - - Some(tx.shared_operation().create( - op.id.as_bytes().to_vec(), - op.timestamp.0 as i64, - shared_op.model.to_string(), - to_vec(&shared_op.record_id).unwrap(), - kind.to_string(), - to_vec(&shared_op.data).unwrap(), - instance::pub_id::equals(op.instance.as_bytes().to_vec()), - vec![], - )) - } - _ => None, - }) - .collect::>(); - - let (res, _) = tx._batch((queries, shared)).await?; - - for op in _ops { - self.tx.send(SyncMessage::Created(op)).ok(); - } - - res - }; - #[cfg(not(feature = "sync-messages"))] - let res = tx._batch([queries]).await?.remove(0); - - Ok(res) - } - - #[allow(unused_variables)] - pub async fn write_op<'item, Q: prisma_client_rust::BatchItem<'item>>( - &self, - tx: &PrismaClient, - op: CRDTOperation, - query: Q, - ) -> prisma_client_rust::Result<::ReturnValue> { - #[cfg(feature = "sync-messages")] - let ret = { - let ret = match &op.typ { - CRDTOperationType::Shared(shared_op) => { - let kind = match &shared_op.data { - SharedOperationData::Create => "c", - SharedOperationData::Update { .. } => "u", - SharedOperationData::Delete => "d", - }; - - tx._batch(( - tx.shared_operation().create( - op.id.as_bytes().to_vec(), - op.timestamp.0 as i64, - shared_op.model.to_string(), - to_vec(&shared_op.record_id).unwrap(), - kind.to_string(), - to_vec(&shared_op.data).unwrap(), - instance::pub_id::equals(op.instance.as_bytes().to_vec()), - vec![], - ), - query, - )) - .await? - .1 - } - _ => todo!(), - }; - - self.tx.send(SyncMessage::Created(op)).ok(); - - ret - }; - #[cfg(not(feature = "sync-messages"))] - let ret = tx._batch(vec![query]).await?.remove(0); - - Ok(ret) - } - - pub async fn get_ops(&self) -> prisma_client_rust::Result> { - Ok(self - .db - .shared_operation() - .find_many(vec![]) - .order_by(shared_operation::timestamp::order(SortOrder::Asc)) - .include(shared_operation::include!({ instance: select { - pub_id - } })) - .exec() - .await? - .into_iter() - .flat_map(|op| { - Some(CRDTOperation { - id: Uuid::from_slice(&op.id).ok()?, - instance: Uuid::from_slice(&op.instance.pub_id).ok()?, - timestamp: NTP64(op.timestamp as u64), - typ: CRDTOperationType::Shared(SharedOperation { - record_id: serde_json::from_slice(&op.record_id).ok()?, - model: op.model, - data: serde_json::from_slice(&op.data).ok()?, - }), - }) - }) - .collect()) - } - - pub async fn ingest_op(&self, op: CRDTOperation) -> prisma_client_rust::Result<()> { - let db = &self.db; - - if db - .instance() - .find_unique(instance::pub_id::equals(op.instance.as_bytes().to_vec())) - .exec() - .await? - .is_none() - { - panic!("Node is not paired!") - } - - let msg = SyncMessage::Ingested(op.clone()); - - ModelSyncData::from_op(op.typ.clone()) - .unwrap() - .exec(db) - .await?; - - if let CRDTOperationType::Shared(shared_op) = op.typ { - let kind = match &shared_op.data { - SharedOperationData::Create => "c", - SharedOperationData::Update { .. } => "u", - SharedOperationData::Delete => "d", - }; - - db.shared_operation() - .create( - op.id.as_bytes().to_vec(), - op.timestamp.0 as i64, - shared_op.model.to_string(), - to_vec(&shared_op.record_id).unwrap(), - kind.to_string(), - to_vec(&shared_op.data).unwrap(), - instance::pub_id::equals(op.instance.as_bytes().to_vec()), - vec![], - ) - .exec() - .await?; - } - - self.tx.send(msg).ok(); - - Ok(()) - } -} - -impl OperationFactory for SyncManager { - fn get_clock(&self) -> &HLC { - &self.clock - } - - fn get_instance(&self) -> Uuid { - self.instance - } -} diff --git a/core/src/sync/mod.rs b/core/src/sync/mod.rs deleted file mode 100644 index ca23770219db..000000000000 --- a/core/src/sync/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -mod manager; - -pub use crate::prisma_sync::*; -pub use manager::*; -pub use sd_sync::*; diff --git a/core/src/util/db.rs b/core/src/util/db.rs index df43a24e04f1..f071682693c4 100644 --- a/core/src/util/db.rs +++ b/core/src/util/db.rs @@ -1,7 +1,6 @@ use crate::prisma::{self, PrismaClient}; use prisma_client_rust::{migrations::*, NewClientError}; use thiserror::Error; -use uuid::Uuid; /// MigrationError represents an error that occurring while opening a initialising and running migrations on the database. #[derive(Error, Debug)] @@ -58,28 +57,6 @@ pub async fn load_and_migrate(db_url: &str) -> Result`, -/// removing any `None` values in the process -pub fn chain_optional_iter( - required: impl IntoIterator, - optional: impl IntoIterator>, -) -> Vec { - required - .into_iter() - .map(Some) - .chain(optional) - .flatten() - .collect() -} - -pub fn uuid_to_bytes(uuid: Uuid) -> Vec { - uuid.as_bytes().to_vec() -} - -pub fn from_bytes_to_uuid(bytes: &[u8]) -> Uuid { - Uuid::from_slice(bytes).expect("corrupted uuid in database") -} - pub fn inode_from_db(db_inode: &[u8]) -> u64 { u64::from_le_bytes(db_inode.try_into().expect("corrupted inode in database")) } diff --git a/core/src/util/debug_initializer.rs b/core/src/util/debug_initializer.rs index 378e44da17c2..6f4fbd82d11e 100644 --- a/core/src/util/debug_initializer.rs +++ b/core/src/util/debug_initializer.rs @@ -3,6 +3,7 @@ use std::{ io, path::{Path, PathBuf}, + sync::Arc, time::Duration, }; @@ -96,7 +97,7 @@ impl InitConfig { pub async fn apply( self, - library_manager: &LibraryManager, + library_manager: &Arc, node_cfg: NodeConfig, ) -> Result<(), InitConfigError> { info!("Initializing app from file: {:?}", self.path); diff --git a/crates/sync-generator/src/lib.rs b/crates/sync-generator/src/lib.rs index 2c3a3f0e5693..5dbee5571f9a 100644 --- a/crates/sync-generator/src/lib.rs +++ b/crates/sync-generator/src/lib.rs @@ -43,7 +43,7 @@ impl<'a> ModelSyncType<'a> { .field("id") .and_then(|field| match field { AttributeFieldValue::Single(s) => Some(s), - AttributeFieldValue::List(l) => None, + AttributeFieldValue::List(_) => None, }) .and_then(|name| model.fields().find(|f| f.name() == *name))?; @@ -58,20 +58,20 @@ impl<'a> ModelSyncType<'a> { attr.field(name) .and_then(|field| match field { AttributeFieldValue::Single(s) => Some(*s), - AttributeFieldValue::List(l) => None, + AttributeFieldValue::List(_) => None, }) .and_then(|name| { match model .fields() .find(|f| f.name() == name) - .expect(&format!("'{name}' field not found")) + .unwrap_or_else(|| panic!("'{name}' field not found")) .refine() { RefinedFieldWalker::Relation(r) => Some(r), _ => None, } }) - .expect(&format!("'{name}' must be a relation field")) + .unwrap_or_else(|| panic!("'{name}' must be a relation field")) }; Self::Relation { @@ -88,10 +88,9 @@ impl<'a> ModelSyncType<'a> { fn sync_id(&self) -> Vec { match self { // Self::Owned { id } => id.clone(), - Self::Local { id } => vec![id.clone()], - Self::Shared { id } => vec![id.clone()], + Self::Local { id } => vec![*id], + Self::Shared { id } => vec![*id], Self::Relation { group, item } => vec![(*group).into(), (*item).into()], - _ => vec![], } } } diff --git a/crates/sync/src/crdt.rs b/crates/sync/src/crdt.rs index e985b3cc0571..4ff63e3893d1 100644 --- a/crates/sync/src/crdt.rs +++ b/crates/sync/src/crdt.rs @@ -6,16 +6,22 @@ use specta::Type; use uhlc::NTP64; use uuid::Uuid; -#[derive(Serialize, Deserialize, Clone, Debug, Type)] -pub enum RelationOperationData { - #[serde(rename = "c")] +pub enum OperationKind<'a> { Create, - #[serde(rename = "u")] - Update { field: String, value: Value }, - #[serde(rename = "d")] + Update(&'a str), Delete, } +impl std::fmt::Display for OperationKind<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + OperationKind::Create => write!(f, "c"), + OperationKind::Update(field) => write!(f, "u:{}", field), + OperationKind::Delete => write!(f, "d"), + } + } +} + #[derive(Serialize, Deserialize, Clone, Debug, Type)] pub struct RelationOperation { pub relation_item: Value, @@ -24,8 +30,14 @@ pub struct RelationOperation { pub data: RelationOperationData, } +impl RelationOperation { + pub fn kind(&self) -> OperationKind { + self.data.as_kind() + } +} + #[derive(Serialize, Deserialize, Clone, Debug, Type)] -pub enum SharedOperationData { +pub enum RelationOperationData { #[serde(rename = "c")] Create, #[serde(rename = "u")] @@ -34,6 +46,16 @@ pub enum SharedOperationData { Delete, } +impl RelationOperationData { + fn as_kind(&self) -> OperationKind { + match self { + Self::Create => OperationKind::Create, + Self::Update { field, .. } => OperationKind::Update(field), + Self::Delete => OperationKind::Delete, + } + } +} + #[derive(Serialize, Deserialize, Clone, Debug, Type)] pub struct SharedOperation { pub record_id: Value, @@ -41,6 +63,32 @@ pub struct SharedOperation { pub data: SharedOperationData, } +impl SharedOperation { + pub fn kind(&self) -> OperationKind { + self.data.as_kind() + } +} + +#[derive(Serialize, Deserialize, Clone, Debug, Type)] +pub enum SharedOperationData { + #[serde(rename = "c")] + Create, + #[serde(rename = "u")] + Update { field: String, value: Value }, + #[serde(rename = "d")] + Delete, +} + +impl SharedOperationData { + fn as_kind(&self) -> OperationKind { + match self { + Self::Create => OperationKind::Create, + Self::Update { field, .. } => OperationKind::Update(field), + Self::Delete => OperationKind::Delete, + } + } +} + // #[derive(Serialize, Deserialize, Clone, Debug, Type)] // pub enum OwnedOperationData { // Create(BTreeMap), diff --git a/crates/sync/src/lib.rs b/crates/sync/src/lib.rs index 603ccb12764d..7fb007988b32 100644 --- a/crates/sync/src/lib.rs +++ b/crates/sync/src/lib.rs @@ -5,3 +5,160 @@ mod model_traits; pub use crdt::*; pub use factory::*; pub use model_traits::*; + +// fn compare_messages(&self, operations: Vec) -> Vec<(CRDTOperation, bool)> { +// operations +// .into_iter() +// .map(|op| (op.id, op)) +// .collect::>() +// .into_iter() +// .filter_map(|(_, op)| { +// match &op.typ { +// CRDTOperationType::Owned(_) => { +// self._operations.iter().find(|find_op| match &find_op.typ { +// CRDTOperationType::Owned(_) => { +// find_op.timestamp >= op.timestamp && find_op.node == op.node +// } +// _ => false, +// }) +// } +// CRDTOperationType::Shared(shared_op) => { +// self._operations.iter().find(|find_op| match &find_op.typ { +// CRDTOperationType::Shared(find_shared_op) => { +// shared_op.model == find_shared_op.model +// && shared_op.record_id == find_shared_op.record_id +// && find_op.timestamp >= op.timestamp +// } +// _ => false, +// }) +// } +// CRDTOperationType::Relation(relation_op) => { +// self._operations.iter().find(|find_op| match &find_op.typ { +// CRDTOperationType::Relation(find_relation_op) => { +// relation_op.relation == find_relation_op.relation +// && relation_op.relation_item == find_relation_op.relation_item +// && relation_op.relation_group == find_relation_op.relation_group +// } +// _ => false, +// }) +// } +// } +// .map(|old_op| (old_op.timestamp != op.timestamp).then_some(true)) +// .unwrap_or(Some(false)) +// .map(|old| (op, old)) +// }) +// .collect() +// } + +// pub fn receive_crdt_operations(&mut self, ops: Vec) { +// for op in &ops { +// self._clock +// .update_with_timestamp(&Timestamp::new(op.timestamp, op.node.into())) +// .ok(); + +// self._clocks.insert(op.node, op.timestamp); +// } + +// for (op, old) in self.compare_messages(ops) { +// let push_op = op.clone(); + +// if !old { +// match op.typ { +// CRDTOperationType::Shared(shared_op) => match shared_op.model.as_str() { +// "Object" => { +// let id = shared_op.record_id; + +// match shared_op.data { +// SharedOperationData::Create(SharedOperationCreateData::Atomic) => { +// self.objects.insert( +// id, +// Object { +// id, +// ..Default::default() +// }, +// ); +// } +// SharedOperationData::Update { field, value } => { +// let mut file = self.objects.get_mut(&id).unwrap(); + +// match field.as_str() { +// "name" => { +// file.name = from_value(value).unwrap(); +// } +// _ => unreachable!(), +// } +// } +// SharedOperationData::Delete => { +// self.objects.remove(&id).unwrap(); +// } +// _ => {} +// } +// } +// _ => unreachable!(), +// }, +// CRDTOperationType::Owned(owned_op) => match owned_op.model.as_str() { +// "FilePath" => { +// for item in owned_op.items { +// let id = from_value(item.id).unwrap(); + +// match item.data { +// OwnedOperationData::Create(data) => { +// self.file_paths.insert( +// id, +// from_value(Value::Object(data.into_iter().collect())) +// .unwrap(), +// ); +// } +// OwnedOperationData::Update(data) => { +// let obj = self.file_paths.get_mut(&id).unwrap(); + +// for (key, value) in data { +// match key.as_str() { +// "path" => obj.path = from_value(value).unwrap(), +// "file" => obj.file = from_value(value).unwrap(), +// _ => unreachable!(), +// } +// } +// } +// OwnedOperationData::Delete => { +// self.file_paths.remove(&id); +// } +// } +// } +// } +// _ => unreachable!(), +// }, +// CRDTOperationType::Relation(relation_op) => match relation_op.relation.as_str() +// { +// "TagOnObject" => match relation_op.data { +// RelationOperationData::Create => { +// self.tags_on_objects.insert( +// (relation_op.relation_item, relation_op.relation_group), +// TagOnObject { +// object_id: relation_op.relation_item, +// tag_id: relation_op.relation_group, +// }, +// ); +// } +// RelationOperationData::Update { field: _, value: _ } => { +// // match field.as_str() { +// // _ => unreachable!(), +// // } +// } +// RelationOperationData::Delete => { +// self.tags_on_objects +// .remove(&( +// relation_op.relation_item, +// relation_op.relation_group, +// )) +// .unwrap(); +// } +// }, +// _ => unreachable!(), +// }, +// } + +// self._operations.push(push_op) +// } +// } +// } diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml new file mode 100644 index 000000000000..825c16497371 --- /dev/null +++ b/crates/utils/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "sd-utils" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +uuid = { workspace = true } diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs new file mode 100644 index 000000000000..854304f5ec78 --- /dev/null +++ b/crates/utils/src/lib.rs @@ -0,0 +1,23 @@ +use uuid::Uuid; + +/// Combines an iterator of `T` and an iterator of `Option`, +/// removing any `None` values in the process +pub fn chain_optional_iter( + required: impl IntoIterator, + optional: impl IntoIterator>, +) -> Vec { + required + .into_iter() + .map(Some) + .chain(optional) + .flatten() + .collect() +} + +pub fn uuid_to_bytes(uuid: Uuid) -> Vec { + uuid.as_bytes().to_vec() +} + +pub fn from_bytes_to_uuid(bytes: &[u8]) -> Uuid { + Uuid::from_slice(bytes).expect("corrupted uuid in database") +} diff --git a/interface/app/$libraryId/Layout/Sidebar/Contents.tsx b/interface/app/$libraryId/Layout/Sidebar/Contents.tsx index 15e632ebdbd7..5612f820d36e 100644 --- a/interface/app/$libraryId/Layout/Sidebar/Contents.tsx +++ b/interface/app/$libraryId/Layout/Sidebar/Contents.tsx @@ -8,7 +8,7 @@ import { FilmStrip, Planet } from 'phosphor-react'; -import { LibraryContextProvider, useClientContext } from '@sd/client'; +import { LibraryContextProvider, useClientContext, useFeatureFlag } from '@sd/client'; import { SubtleButton } from '~/components/SubtleButton'; import Icon from './Icon'; import { LibrarySection } from './LibrarySection'; @@ -33,6 +33,12 @@ export default () => { Imports */} + {useFeatureFlag('syncRoute') && ( + + + Sync + + )} {library && ( diff --git a/interface/app/$libraryId/sync.tsx b/interface/app/$libraryId/sync.tsx index f3ab60987753..b8916fec5674 100644 --- a/interface/app/$libraryId/sync.tsx +++ b/interface/app/$libraryId/sync.tsx @@ -1,39 +1,27 @@ -import { CRDTOperation, useLibraryQuery, useLibrarySubscription } from '@sd/client'; -import { tw } from '@sd/ui'; - -const Label = tw.span`text-gray-300`; -const Pill = tw.span`rounded-full bg-gray-500 px-2 py-1`; -const Row = tw.p`overflow-hidden text-ellipsis space-x-1`; - -const OperationItem = ({ op }: { op: CRDTOperation }) => { - let contents = null; - - if ('model' in op.typ) { - let subContents = null; - - if (op.typ.data === 'd') subContents = 'Delete'; - else if (op.typ.data === 'c') subContents = 'Create'; - else subContents = `Update - ${op.typ.data.u.field}`; - - contents = ( - <> -
- {subContents} -
- - - {op.typ.model} - - - - {op.timestamp} - - - ); - } - - return
  • {contents}
    • ; -}; +import { useMemo } from 'react'; +import { stringify } from 'uuid'; +import { + CRDTOperation, + RelationOperation, + SharedOperation, + useLibraryQuery, + useLibrarySubscription +} from '@sd/client'; + +type MessageGroup = + | { + variant: 'Shared'; + model: string; + id: string; + messages: { op: SharedOperation; timestamp: number }[]; + } + | { + variant: 'Relation'; + relation: string; + item_id: string; + group_id: string; + messages: { op: RelationOperation; timestamp: number }[]; + }; export const Component = () => { const messages = useLibraryQuery(['sync.messages']); @@ -42,11 +30,156 @@ export const Component = () => { onData: () => messages.refetch() }); + const groups = useMemo(() => { + if (messages.data) return calculateGroups(messages.data); + }, [messages]); + return ( -
      - {messages.data?.map((op) => ( - +
        + {groups?.map((group, index) => ( + ))}
      ); }; + +const OperationGroup: React.FC<{ group: MessageGroup }> = ({ group }) => { + const [header, contents] = (() => { + switch (group.variant) { + case 'Shared': { + const header = ( +
      + {group.model} + {group.id} +
      + ); + const contents = ( +
        + {group.messages.map((message, index) => ( +
      • + {typeof message.op.data === 'string' ? ( +

        {message.op.data === 'c' ? 'Create' : 'Delete'}

        + ) : ( +

        Update - {message.op.data.u.field}

        + )} +

        {message.timestamp}

        +
        • + ))} +
      + ); + return [header, contents]; + } + case 'Relation': { + const header = ( +
      + {group.relation} + {group.item_id} + in + {group.group_id} +
      + ); + + const contents = ( +
        + {group.messages.map((message, index) => ( +
      • + {typeof message.op.data === 'string' ? ( +

        {message.op.data === 'c' ? 'Create' : 'Delete'}

        + ) : ( +

        Update - {message.op.data.u.field}

        + )} +

        {message.timestamp}

        +
        • + ))} +
      + ); + + return [header, contents]; + } + } + })(); + + return ( +
      + {header} + {contents} +
      + ); +}; + +function calculateGroups(messages: CRDTOperation[]) { + return messages.reduce((acc, curr) => { + const { typ } = curr; + + if ('model' in typ) { + const id = stringify(typ.record_id.pub_id); + + const latest = (() => { + const latest = acc[acc.length - 1]; + + if ( + !latest || + latest.variant !== 'Shared' || + latest.model !== typ.model || + latest.id !== id + ) { + const group: MessageGroup = { + variant: 'Shared', + model: typ.model, + id, + messages: [] + }; + + acc.push(group); + + return group; + } else { + return latest; + } + })(); + + latest.messages.push({ + op: typ, + timestamp: curr.timestamp + }); + } else { + const id = { + item: stringify(typ.relation_item.pub_id), + group: stringify(typ.relation_group.pub_id) + }; + + const latest = (() => { + const latest = acc[acc.length - 1]; + + if ( + !latest || + latest.variant !== 'Relation' || + latest.relation !== typ.relation || + latest.item_id !== id.item || + latest.group_id !== id.group + ) { + const group: MessageGroup = { + variant: 'Relation', + relation: typ.relation, + item_id: id.item, + group_id: id.group, + messages: [] + }; + + acc.push(group); + + return group; + } else { + return latest; + } + })(); + + latest.messages.push({ + op: typ, + timestamp: curr.timestamp + }); + } + + return acc; + }, []); +} diff --git a/interface/app/onboarding/creating-library.tsx b/interface/app/onboarding/creating-library.tsx index 4159ae121d35..5295f7b1c923 100644 --- a/interface/app/onboarding/creating-library.tsx +++ b/interface/app/onboarding/creating-library.tsx @@ -51,11 +51,10 @@ export default function OnboardingCreatingLibrary() { // it feels more fitting to configure it here (once) telemetryStore.shareTelemetry = obStore.shareTelemetry; + console.log('creating'); createLibrary.mutate({ name: obStore.newLibraryName }); - - return; }; const created = useRef(false); diff --git a/packages/client/src/core.ts b/packages/client/src/core.ts index f4b80f72c23e..a624b9933729 100644 --- a/packages/client/src/core.ts +++ b/packages/client/src/core.ts @@ -261,7 +261,7 @@ export type PairingStatus = { type: "EstablishingConnection" } | { type: "Pairin export type PeerId = string -export type PeerMetadata = { name: string; operating_system: OperatingSystem | null; version: string | null; email: string | null; img_url: string | null; instances: string[] } +export type PeerMetadata = { name: string; operating_system: OperatingSystem | null; version: string | null; email: string | null; img_url: string | null } export type RelationOperation = { relation_item: any; relation_group: any; relation: string; data: RelationOperationData } diff --git a/packages/client/src/hooks/useFeatureFlag.tsx b/packages/client/src/hooks/useFeatureFlag.tsx index ec48c100c8b4..e4a943f624d4 100644 --- a/packages/client/src/hooks/useFeatureFlag.tsx +++ b/packages/client/src/hooks/useFeatureFlag.tsx @@ -2,9 +2,9 @@ import { useEffect } from 'react'; import { subscribe, useSnapshot } from 'valtio'; import { valtioPersist } from '../lib/valito'; -export const features = ['spacedrop', 'p2pPairing'] as const; +export const features = ['spacedrop', 'p2pPairing', 'syncRoute'] as const; -export type FeatureFlag = (typeof features)[number]; +export type FeatureFlag = typeof features[number]; const featureFlagState = valtioPersist('sd-featureFlags', { enabled: [] as FeatureFlag[] diff --git a/packages/client/src/hooks/useNotifications.tsx b/packages/client/src/hooks/useNotifications.tsx index 252b62ae5574..4eb49aff1eeb 100644 --- a/packages/client/src/hooks/useNotifications.tsx +++ b/packages/client/src/hooks/useNotifications.tsx @@ -1,4 +1,4 @@ -import { PropsWithChildren, createContext, useContext, useState } from 'react'; +import { PropsWithChildren, createContext, useState } from 'react'; import { Notification } from '../core'; import { useBridgeSubscription } from '../rspc'; From 447069388b919e85d066eda9f7ac09403cd5b2c1 Mon Sep 17 00:00:00 2001 From: Brendan Allan Date: Tue, 25 Jul 2023 01:17:02 +0800 Subject: [PATCH 3/7] Push libraries into vec after loading (#1129) push libraries into vec after loading --- core/src/library/manager.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/library/manager.rs b/core/src/library/manager.rs index bd8f18ecaabe..8e906eea508e 100644 --- a/core/src/library/manager.rs +++ b/core/src/library/manager.rs @@ -220,8 +220,6 @@ impl LibraryManager { invalidate_query!(library, "library.list"); - debug!("Pushed library into manager '{id:?}'"); - Ok(LibraryConfigWrapped { uuid: id, config }) } @@ -419,6 +417,8 @@ impl LibraryManager { self.clone(), )); + self.libraries.write().await.push(library.clone()); + if should_seed { library.orphan_remover.invoke().await; indexer::rules::seed::new_or_existing_library(&library).await?; From 7e4ed2b5471a3c14595ff57c8da5dc6836a37649 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20Vasconcellos?= Date: Mon, 24 Jul 2023 14:36:00 -0300 Subject: [PATCH 4/7] [ENG-767, ENG-916] Improvements and fixes for `Open`/`Open With` (#1082) * Normalize PATH and XDG envvars on Linux * Fix normalize_xdg_environment * Replace custom Desktop file parsers with Glib * Fix AppImage env influencing external apps - Normalize GStream plugin path * Fix macos pulling linux deps * Attempt to fix gnome apps failing to launch - Fix incorrect logic in `normalize_pathlist` * Ensure empty envvars are not set * Revert AppImage ovewritten GTK specific variables - `normalize_pathlist` now prefers entries with less priority when dealing with repeated entries, this is not compatible with the default behavior, but it is a more sane approach IMHO * Remove 32-bit libs from release CI build host * Remove 32-bit from github runner 2 attempt - Remove deprecated vscode config * Remove libc6-i386 * [ENG-916] Implement `Open With` logic compatible with macOS < 12 * Add some missing gstreamer deps in Linux * Replace opener with Glib API * Fix reveal opening file instead of dir - Improve Open With logic in Windows - Expose functions to test if app is in a flatpak, snap or appimage --------- Co-authored-by: Brendan Allan --- .editorconfig | 2 +- .github/scripts/setup-system.sh | 6 +- .github/workflows/release.yml | 7 + .vscode/settings.json | 1 - Cargo.lock | 111 ++------ apps/cli/Cargo.toml | 2 - apps/desktop/crates/linux/Cargo.toml | 13 +- apps/desktop/crates/linux/src/app_info.rs | 140 ++++++++++ .../desktop/crates/linux/src/desktop_entry.rs | 180 ------------- apps/desktop/crates/linux/src/env.rs | 226 ++++++++++++++++ apps/desktop/crates/linux/src/handler.rs | 54 ---- apps/desktop/crates/linux/src/lib.rs | 30 +-- apps/desktop/crates/linux/src/system.rs | 63 ----- apps/desktop/crates/linux/tests/cmus.desktop | 12 - apps/desktop/crates/macos/Cargo.toml | 6 +- .../crates/macos/src-swift/files.swift | 133 +++++++--- apps/desktop/crates/macos/src/lib.rs | 8 +- apps/desktop/crates/windows/Cargo.toml | 4 +- apps/desktop/src-tauri/Cargo.toml | 1 + apps/desktop/src-tauri/src/file.rs | 243 ++++++++++-------- apps/desktop/src-tauri/src/main.rs | 20 +- apps/desktop/src/commands.ts | 4 +- crates/ffmpeg/Cargo.toml | 2 - crates/file-ext/Cargo.toml | 1 - crates/macos/Cargo.toml | 6 +- crates/prisma/Cargo.toml | 2 - crates/sync-generator/Cargo.toml | 2 - 27 files changed, 662 insertions(+), 617 deletions(-) create mode 100644 apps/desktop/crates/linux/src/app_info.rs delete mode 100644 apps/desktop/crates/linux/src/desktop_entry.rs create mode 100644 apps/desktop/crates/linux/src/env.rs delete mode 100644 apps/desktop/crates/linux/src/handler.rs delete mode 100644 apps/desktop/crates/linux/src/system.rs delete mode 100644 apps/desktop/crates/linux/tests/cmus.desktop diff --git a/.editorconfig b/.editorconfig index 8549c42f9685..462a3da63679 100644 --- a/.editorconfig +++ b/.editorconfig @@ -86,5 +86,5 @@ indent_style = space # Swift # https://github.com/apple/swift-format/blob/main/Documentation/Configuration.md#example [*.swift] -indent_size = 2 +indent_size = 4 indent_style = space diff --git a/.github/scripts/setup-system.sh b/.github/scripts/setup-system.sh index d3e5aea515da..e93c1f50d19f 100755 --- a/.github/scripts/setup-system.sh +++ b/.github/scripts/setup-system.sh @@ -150,7 +150,7 @@ if [ "$SYSNAME" = "Linux" ]; then DEBIAN_FFMPEG_DEPS="libheif-dev libavcodec-dev libavdevice-dev libavfilter-dev libavformat-dev libavutil-dev libswscale-dev libswresample-dev ffmpeg" # Webkit2gtk requires gstreamer plugins for video playback to work - DEBIAN_VIDEO_DEPS="gstreamer1.0-libav gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-plugins-ugly" + DEBIAN_VIDEO_DEPS="gstreamer1.0-alsa gstreamer1.0-gl gstreamer1.0-gtk3 gstreamer1.0-libav gstreamer1.0-pipewire gstreamer1.0-plugins-bad gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-ugly gstreamer1.0-pulseaudio gstreamer1.0-vaapi libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev libgstreamer-plugins-bad1.0-dev" # Bindgen dependencies - it's used by a dependency of Spacedrive DEBIAN_BINDGEN_DEPS="pkg-config clang" @@ -168,7 +168,7 @@ if [ "$SYSNAME" = "Linux" ]; then ARCH_TAURI_DEPS="webkit2gtk base-devel curl wget openssl appmenu-gtk-module gtk3 libappindicator-gtk3 librsvg libvips patchelf" # Webkit2gtk requires gstreamer plugins for video playback to work - ARCH_VIDEO_DEPS="gst-libav gst-plugins-base gst-plugins-good gst-plugins-bad gst-plugins-ugly" + ARCH_VIDEO_DEPS="gst-libav gst-plugins-bad gst-plugins-base gst-plugins-good gst-plugins-ugly gst-plugin-pipewire gstreamer-vaapi" # FFmpeg dependencies ARCH_FFMPEG_DEPS="libheif ffmpeg" @@ -202,7 +202,7 @@ if [ "$SYSNAME" = "Linux" ]; then FEDORA_FFMPEG_DEPS="libheif-devel ffmpeg ffmpeg-devel" # Webkit2gtk requires gstreamer plugins for video playback to work - FEDORA_VIDEO_DEPS="gstreamer1-plugin-libav gstreamer1-plugins-base gstreamer1-plugins-good gstreamer1-plugins-good-extras gstreamer1-plugins-bad-free gstreamer1-plugins-bad-free-extras gstreamer1-plugins-ugly-free" + FEDORA_VIDEO_DEPS="gstreamer1-devel gstreamer1-plugins-base-devel gstreamer1-plugins-good gstreamer1-plugins-good-gtk gstreamer1-plugins-good-extras gstreamer1-plugins-ugly-free gstreamer1-plugins-bad-free gstreamer1-plugins-bad-free-devel gstreamer1-plugins-bad-free-extras" # Bindgen dependencies - it's used by a dependency of Spacedrive FEDORA_BINDGEN_DEPS="clang" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4cace3e44170..9fd6e5e41247 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -39,6 +39,13 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 + - name: Remove 32-bit libs + if: ${{ runner.os == 'Linux' }} + run: | + dpkg -l | grep i386 + sudo apt-get purge --allow-remove-essential libc6-i386 ".*:i386" + sudo dpkg --remove-architecture i386 + - name: Install Apple API key if: ${{ runner.os == 'macOS' }} run: | diff --git a/.vscode/settings.json b/.vscode/settings.json index bc82ff68a767..41ac7fed5d2e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -65,7 +65,6 @@ "packages/config", "packages/ui" ], - "eslint.packageManager": "pnpm", "eslint.lintTask.enable": true, "explorer.fileNesting.enabled": true, "explorer.fileNesting.patterns": { diff --git a/Cargo.lock b/Cargo.lock index 2e8e3e3419ed..46e83631d7c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -271,12 +271,6 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" -[[package]] -name = "arrayvec" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" - [[package]] name = "arrayvec" version = "0.7.2" @@ -298,7 +292,7 @@ dependencies = [ "asn1-rs-derive 0.1.0", "asn1-rs-impl", "displaydoc", - "nom 7.1.3", + "nom", "num-traits", "rusticata-macros", "thiserror", @@ -314,7 +308,7 @@ dependencies = [ "asn1-rs-derive 0.4.0", "asn1-rs-impl", "displaydoc", - "nom 7.1.3", + "nom", "num-traits", "rusticata-macros", "thiserror", @@ -745,7 +739,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ae2468a89544a466886840aa467a25b766499f4f04bf7d9fcd10ecee9fccef" dependencies = [ "arrayref", - "arrayvec 0.7.2", + "arrayvec", "cc", "cfg-if", "constant_time_eq", @@ -1000,7 +994,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" dependencies = [ - "nom 7.1.3", + "nom", ] [[package]] @@ -1816,7 +1810,7 @@ checksum = "fe398ac75057914d7d07307bf67dc7f3f574a26783b4fc7805a20ffa9f506e82" dependencies = [ "asn1-rs 0.3.1", "displaydoc", - "nom 7.1.3", + "nom", "num-bigint", "num-traits", "rusticata-macros", @@ -1830,7 +1824,7 @@ checksum = "dbd676fbbab537128ef0278adb5576cf363cff6aa22a7b24effe97347cfab61e" dependencies = [ "asn1-rs 0.5.2", "displaydoc", - "nom 7.1.3", + "nom", "num-bigint", "num-traits", "rusticata-macros", @@ -2464,16 +2458,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92d73a076bdabd78c2f9045dba1b90664a655fa8372581c238596e1eb3a5e1b7" -[[package]] -name = "freedesktop_entry_parser" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db9c27b72f19a99a895f8ca89e2d26e4ef31013376e56fdafef697627306c3e4" -dependencies = [ - "nom 7.1.3", - "thiserror", -] - [[package]] name = "fsevent-sys" version = "4.1.0" @@ -3173,15 +3157,6 @@ dependencies = [ "digest 0.10.7", ] -[[package]] -name = "home" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" -dependencies = [ - "windows-sys 0.48.0", -] - [[package]] name = "hostname" version = "0.3.1" @@ -3829,24 +3804,11 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" -[[package]] -name = "lexical-core" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe" -dependencies = [ - "arrayvec 0.5.2", - "bitflags", - "cfg-if", - "ryu", - "static_assertions", -] - [[package]] name = "libc" -version = "0.2.146" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "libdbus-sys" @@ -4053,7 +4015,7 @@ version = "0.43.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39d5ef876a2b2323d63c258e63c2f8e36f205fe5a11f0b3095d59635650790ff" dependencies = [ - "arrayvec 0.7.2", + "arrayvec", "asynchronous-codec", "bytes", "either", @@ -4914,17 +4876,6 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" -[[package]] -name = "nom" -version = "5.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08959a387a676302eebf4ddbcbc611da04285579f76f88ee0506c63b1a61dd4b" -dependencies = [ - "lexical-core", - "memchr", - "version_check", -] - [[package]] name = "nom" version = "7.1.3" @@ -6875,7 +6826,7 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" dependencies = [ - "nom 7.1.3", + "nom", ] [[package]] @@ -7179,14 +7130,9 @@ dependencies = [ name = "sd-desktop-linux" version = "0.1.0" dependencies = [ - "aho-corasick 1.0.2", - "atty", - "freedesktop_entry_parser", - "mime", - "shlex", - "thiserror", - "xdg", - "xdg-mime", + "gtk", + "libc", + "tokio", ] [[package]] @@ -7335,7 +7281,7 @@ dependencies = [ name = "sd-sync-generator" version = "0.1.0" dependencies = [ - "nom 7.1.3", + "nom", "once_cell", "prisma-client-rust-sdk", "proc-macro2", @@ -7870,6 +7816,7 @@ name = "spacedrive" version = "0.1.0" dependencies = [ "axum", + "futures", "http", "httpz 0.0.3", "opener", @@ -8059,7 +8006,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c12bc9199d1db8234678b7051747c07f517cdcf019262d1847b94ec8b1aee3e" dependencies = [ "itertools", - "nom 7.1.3", + "nom", "unicode_categories", ] @@ -10402,7 +10349,7 @@ dependencies = [ "data-encoding", "der-parser 7.0.0", "lazy_static", - "nom 7.1.3", + "nom", "oid-registry 0.4.0", "ring", "rusticata-macros", @@ -10421,7 +10368,7 @@ dependencies = [ "data-encoding", "der-parser 8.2.0", "lazy_static", - "nom 7.1.3", + "nom", "oid-registry 0.6.1", "rusticata-macros", "thiserror", @@ -10437,28 +10384,6 @@ dependencies = [ "libc", ] -[[package]] -name = "xdg" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "688597db5a750e9cad4511cb94729a078e274308099a0382b5b8203bbc767fee" -dependencies = [ - "home", -] - -[[package]] -name = "xdg-mime" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87bf7b69bb50588d70a36e467be29d3df3e8c32580276d62eded9738c1a797aa" -dependencies = [ - "dirs-next", - "glob", - "mime", - "nom 5.1.3", - "unicase", -] - [[package]] name = "yasna" version = "0.5.2" diff --git a/apps/cli/Cargo.toml b/apps/cli/Cargo.toml index ebdcb92b5afc..b60a717b2181 100644 --- a/apps/cli/Cargo.toml +++ b/apps/cli/Cargo.toml @@ -5,8 +5,6 @@ license = { workspace = true } repository = { workspace = true } edition = { workspace = true } -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] indoc = "1.0.9" clap = { version = "4.3.0", features = ["derive"] } diff --git a/apps/desktop/crates/linux/Cargo.toml b/apps/desktop/crates/linux/Cargo.toml index febbc8a91348..ed82b3a3bd15 100644 --- a/apps/desktop/crates/linux/Cargo.toml +++ b/apps/desktop/crates/linux/Cargo.toml @@ -6,12 +6,9 @@ repository = { workspace = true } edition = { workspace = true } [dependencies] -aho-corasick = "1.0.2" -atty = "0.2.14" -freedesktop_entry_parser = "1.3.0" -mime = "0.3.17" -shlex = "1.1.0" -thiserror = "1.0.40" -xdg = "2.5.0" -xdg-mime = "0.3.3" +libc = "0.2" +tokio = { workspace = true, features = ["fs"] } +[target.'cfg(target_os = "linux")'.dependencies] +# WARNING: gtk should follow the same version used by tauri +gtk = "0.15" diff --git a/apps/desktop/crates/linux/src/app_info.rs b/apps/desktop/crates/linux/src/app_info.rs new file mode 100644 index 000000000000..4729196bdab9 --- /dev/null +++ b/apps/desktop/crates/linux/src/app_info.rs @@ -0,0 +1,140 @@ +use std::path::{Path, PathBuf}; + +use gtk::{ + gio::{ + content_type_guess, + prelude::AppInfoExt, + prelude::{AppLaunchContextExt, FileExt}, + AppInfo, AppLaunchContext, DesktopAppInfo, File as GioFile, ResourceError, + }, + glib::error::Error as GlibError, + prelude::IsA, +}; +use tokio::fs::File; +use tokio::io::AsyncReadExt; + +use crate::env::remove_prefix_from_pathlist; + +fn remove_prefix_from_env_in_ctx( + ctx: &impl IsA, + env_name: &str, + prefix: &impl AsRef, +) { + if let Some(value) = remove_prefix_from_pathlist(env_name, prefix) { + ctx.setenv(env_name, value); + } else { + ctx.unsetenv(env_name); + } +} + +thread_local! { + static LAUNCH_CTX: AppLaunchContext = { + // TODO: Display supports requires GDK, which can only run on the main thread + // let ctx = Display::default() + // .and_then(|display| display.app_launch_context()) + // .map(|display| display.to_value().get::().expect( + // "This is an Glib type conversion, it should never fail because GDKAppLaunchContext is a subclass of AppLaunchContext" + // )).unwrap_or_default(); + + let ctx = AppLaunchContext::default(); + + if let Some(appdir) = std::env::var_os("APPDIR").map(PathBuf::from) { + // Remove AppImage paths from environment variables to avoid external applications attempting to use the AppImage's libraries + // https://github.com/AppImage/AppImageKit/blob/701b711f42250584b65a88f6427006b1d160164d/src/AppRun.c#L168-L194 + ctx.unsetenv("PYTHONHOME"); + ctx.unsetenv("GTK_DATA_PREFIX"); + ctx.unsetenv("GTK_THEME"); + ctx.unsetenv("GDK_BACKEND"); + ctx.unsetenv("GTK_EXE_PREFIX"); + ctx.unsetenv("GTK_IM_MODULE_FILE"); + ctx.unsetenv("GDK_PIXBUF_MODULE_FILE"); + + remove_prefix_from_env_in_ctx(&ctx, "PATH", &appdir); + remove_prefix_from_env_in_ctx(&ctx, "LD_LIBRARY_PATH", &appdir); + remove_prefix_from_env_in_ctx(&ctx, "PYTHONPATH", &appdir); + remove_prefix_from_env_in_ctx(&ctx, "XDG_DATA_DIRS", &appdir); + remove_prefix_from_env_in_ctx(&ctx, "PERLLIB", &appdir); + remove_prefix_from_env_in_ctx(&ctx, "GSETTINGS_SCHEMA_DIR", &appdir); + remove_prefix_from_env_in_ctx(&ctx, "QT_PLUGIN_PATH", &appdir); + remove_prefix_from_env_in_ctx(&ctx, "GST_PLUGIN_SYSTEM_PATH", &appdir); + remove_prefix_from_env_in_ctx(&ctx, "GST_PLUGIN_SYSTEM_PATH_1_0", &appdir); + remove_prefix_from_env_in_ctx(&ctx, "GTK_PATH", &appdir); + remove_prefix_from_env_in_ctx(&ctx, "GIO_EXTRA_MODULES", &appdir); + } + + ctx + } +} + +pub struct App { + pub id: String, + pub name: String, + // pub icon: Option>, +} + +async fn recommended_for_type(file_path: impl AsRef) -> Vec { + let data = if let Ok(mut file) = File::open(&file_path).await { + let mut data = [0; 1024]; + if file.read_exact(&mut data).await.is_ok() { + Some(data) + } else { + None + } + } else { + None + }; + + let file_path = Some(file_path); + let (content_type, uncertain) = if let Some(data) = data { + content_type_guess(file_path, &data) + } else { + content_type_guess(file_path, &[]) + }; + + if uncertain { + vec![] + } else { + AppInfo::recommended_for_type(content_type.as_str()) + } +} + +pub async fn list_apps_associated_with_ext(file_path: impl AsRef) -> Vec { + recommended_for_type(file_path) + .await + .iter() + .flat_map(|app_info| { + app_info.id().map(|id| App { + id: id.to_string(), + name: app_info.name().to_string(), + // TODO: Icon supports requires GTK, which can only run on the main thread + // icon: app_info + // .icon() + // .and_then(|icon| { + // IconTheme::default().and_then(|icon_theme| { + // icon_theme.lookup_by_gicon(&icon, 128, IconLookupFlags::empty()) + // }) + // }) + // .and_then(|icon_info| icon_info.load_icon().ok()) + // .and_then(|pixbuf| pixbuf.save_to_bufferv("png", &[]).ok()), + }) + }) + .collect() +} + +pub fn open_files_path_with(file_paths: &[impl AsRef], id: &str) -> Result<(), GlibError> { + let Some(app) = DesktopAppInfo::new(id) else { + return Err(GlibError::new(ResourceError::NotFound, "App not found")) + }; + + LAUNCH_CTX.with(|ctx| { + app.launch( + &file_paths.iter().map(GioFile::for_path).collect::>(), + Some(ctx), + ) + }) +} + +pub fn open_file_path(file_path: &impl AsRef) -> Result<(), GlibError> { + let file_uri = GioFile::for_path(file_path).uri().to_string(); + LAUNCH_CTX.with(|ctx| AppInfo::launch_default_for_uri(&file_uri.to_string(), Some(ctx))) +} diff --git a/apps/desktop/crates/linux/src/desktop_entry.rs b/apps/desktop/crates/linux/src/desktop_entry.rs deleted file mode 100644 index 98b583932079..000000000000 --- a/apps/desktop/crates/linux/src/desktop_entry.rs +++ /dev/null @@ -1,180 +0,0 @@ -use std::{ - collections::HashMap, - convert::TryFrom, - ffi::OsString, - path::{Path, PathBuf}, - process::{Command, Stdio}, - str::FromStr, -}; - -use aho_corasick::AhoCorasick; -use mime::Mime; - -use crate::{Error, Result, SystemApps}; - -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct DesktopEntry { - pub name: String, - pub exec: String, - pub file_name: OsString, - pub terminal: bool, - pub mimes: Vec, - pub categories: HashMap, -} - -#[derive(PartialEq, Eq, Copy, Clone)] -pub enum Mode { - Launch, - Open, -} - -fn terminal() -> Result { - // TODO: Attemtp to read x-terminal-emulator bin (Debian/Ubuntu spec for setting default terminal) - SystemApps::get_entries() - .ok() - .and_then(|mut entries| { - entries - .find(|DesktopEntry { categories, .. }| categories.contains_key("TerminalEmulator")) - }) - .map(|e| e.exec) - .ok_or(Error::NoTerminal) -} - -impl DesktopEntry { - pub fn exec(&self, mode: Mode, arguments: &[&str]) -> Result<()> { - let supports_multiple = self.exec.contains("%F") || self.exec.contains("%U"); - if arguments.is_empty() { - self.exec_inner(&[])? - } else if supports_multiple || mode == Mode::Launch { - self.exec_inner(arguments)?; - } else { - for arg in arguments { - self.exec_inner(&[*arg])?; - } - }; - - Ok(()) - } - - fn exec_inner(&self, args: &[&str]) -> Result<()> { - let mut cmd = { - let (cmd, args) = self.get_cmd(args)?; - let mut cmd = Command::new(cmd); - cmd.args(args); - cmd - }; - - if self.terminal && atty::is(atty::Stream::Stdout) { - cmd.spawn()?.wait()?; - } else { - cmd.stdout(Stdio::null()).stderr(Stdio::null()).spawn()?; - } - - Ok(()) - } - - pub fn get_cmd(&self, args: &[&str]) -> Result<(String, Vec)> { - let special = AhoCorasick::new(["%f", "%F", "%u", "%U"]).expect("Failed to build pattern"); - - let mut exec = shlex::split(&self.exec).ok_or(Error::InvalidExec(self.exec.clone()))?; - - // The desktop entry doesn't contain arguments - we make best effort and append them at - // the end - if special.is_match(&self.exec) { - exec = exec - .into_iter() - .flat_map(|s| match s.as_str() { - "%f" | "%F" | "%u" | "%U" => { - args.iter().map(|arg| str::to_string(arg)).collect() - } - s if special.is_match(s) => vec![{ - let mut replaced = String::with_capacity(s.len() + args.len() * 2); - special.replace_all_with(s, &mut replaced, |_, _, dst| { - dst.push_str(args.join(" ").as_str()); - false - }); - replaced - }], - _ => vec![s], - }) - .collect() - } else { - exec.extend(args.iter().map(|arg| str::to_string(arg))); - } - - // If the entry expects a terminal (emulator), but this process is not running in one, we - // launch a new one. - if self.terminal && !atty::is(atty::Stream::Stdout) { - exec = shlex::split(&terminal()?) - .ok_or(Error::InvalidExec(self.exec.clone()))? - .into_iter() - .chain(["-e".to_owned()]) - .chain(exec) - .collect(); - } - - Ok((exec.remove(0), exec)) - } -} - -fn parse_file(path: &Path) -> Option { - let raw_entry = freedesktop_entry_parser::parse_entry(path).ok()?; - let section = raw_entry.section("Desktop Entry"); - - let mut entry = DesktopEntry { - file_name: path.file_name()?.to_owned(), - ..Default::default() - }; - - for attr in section.attrs().filter(|a| a.has_value()) { - match attr.name { - "Name" if entry.name.is_empty() => { - entry.name = attr.value?.into(); - } - "Exec" => entry.exec = attr.value?.into(), - "MimeType" => { - entry.mimes = attr - .value? - .split(';') - .filter_map(|m| Mime::from_str(m).ok()) - .collect::>(); - } - "Terminal" => entry.terminal = attr.value? == "true", - "Categories" => { - entry.categories = attr - .value? - .split(';') - .filter(|s| !s.is_empty()) - .map(|cat| (cat.to_owned(), ())) - .collect(); - } - _ => {} - } - } - - if !entry.name.is_empty() && !entry.exec.is_empty() { - Some(entry) - } else { - None - } -} - -impl TryFrom<&PathBuf> for DesktopEntry { - type Error = Error; - fn try_from(path: &PathBuf) -> Result { - parse_file(path).ok_or(Error::BadEntry(path.clone())) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn complex_exec() { - let entry = parse_file(Path::new("tests/cmus.desktop")).unwrap(); - assert_eq!(entry.mimes.len(), 2); - assert_eq!(entry.mimes[0].essence_str(), "audio/mp3"); - assert_eq!(entry.mimes[1].essence_str(), "audio/ogg"); - } -} diff --git a/apps/desktop/crates/linux/src/env.rs b/apps/desktop/crates/linux/src/env.rs new file mode 100644 index 000000000000..d280877decb6 --- /dev/null +++ b/apps/desktop/crates/linux/src/env.rs @@ -0,0 +1,226 @@ +use std::{ + collections::HashSet, + env, + ffi::{CStr, OsStr, OsString}, + mem, + os::unix::ffi::OsStrExt, + path::{Path, PathBuf}, + ptr, +}; + +pub fn get_current_user_home() -> Option { + use libc::{getpwuid_r, getuid, passwd, ERANGE}; + + if let Some(home) = env::var_os("HOME") { + let home = PathBuf::from(home); + if home.is_absolute() && home.is_dir() { + return Some(home); + } + } + + let uid = unsafe { getuid() }; + let mut buf = vec![0; 2048]; + let mut passwd = unsafe { mem::zeroed::() }; + let mut result = ptr::null_mut::(); + + loop { + let r = unsafe { getpwuid_r(uid, &mut passwd, buf.as_mut_ptr(), buf.len(), &mut result) }; + + if r != ERANGE { + break; + } + + let newsize = buf.len().checked_mul(2)?; + buf.resize(newsize, 0); + } + + if result.is_null() { + // There is no such user, or an error has occurred. + // errno gets set if there’s an error. + return None; + } + + if result != &mut passwd { + // The result of getpwuid_r should be its input passwd. + return None; + } + + let passwd: passwd = unsafe { result.read() }; + if passwd.pw_dir.is_null() { + return None; + } + + let home = PathBuf::from(OsStr::from_bytes( + unsafe { CStr::from_ptr(passwd.pw_dir) }.to_bytes(), + )); + if home.is_absolute() && home.is_dir() { + env::set_var("HOME", &home); + Some(home) + } else { + None + } +} + +fn normalize_pathlist( + env_name: &str, + default_dirs: &[PathBuf], +) -> Result, env::JoinPathsError> { + let dirs = if let Some(value) = env::var_os(env_name) { + let mut dirs = env::split_paths(&value) + .filter(|entry| !entry.as_os_str().is_empty()) + .collect::>(); + + let mut insert_index = dirs.len(); + for default_dir in default_dirs { + match dirs.iter().rev().position(|dir| dir == default_dir) { + Some(mut index) => { + index = dirs.len() - index - 1; + if index < insert_index { + insert_index = index + } + } + None => dirs.insert(insert_index, default_dir.to_path_buf()), + } + } + + dirs + } else { + default_dirs.into() + }; + + let mut unique = HashSet::new(); + let mut pathlist = dirs + .iter() + .rev() // Reverse order to remove duplicates from the end + .filter(|dir| unique.insert(*dir)) + .cloned() + .collect::>(); + + pathlist.reverse(); + + env::set_var(env_name, env::join_paths(&pathlist)?); + + Ok(pathlist) +} + +fn normalize_xdg_environment(name: &str, default_value: PathBuf) -> PathBuf { + if let Some(value) = env::var_os(name) { + if !value.is_empty() { + let path = PathBuf::from(value); + if path.is_absolute() && path.is_dir() { + return path; + } + } + } + + env::set_var(name, &default_value); + default_value +} + +pub fn normalize_environment() { + let home = get_current_user_home().expect("No user home directory found"); + + // Normalize user XDG dirs environment variables + // https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html + let data_home = normalize_xdg_environment("XDG_DATA_HOME", home.join(".local/share")); + normalize_xdg_environment("XDG_CACHE_HOME", home.join(".cache")); + normalize_xdg_environment("XDG_CONFIG_HOME", home.join(".config")); + + // Normalize system XDG dirs environment variables + // https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html + normalize_pathlist( + "XDG_DATA_DIRS", + &[ + PathBuf::from("/usr/share"), + PathBuf::from("/usr/local/share"), + PathBuf::from("/var/lib/flatpak/exports/share"), + data_home.join("flatpak/exports/share"), + ], + ) + .expect("XDG_DATA_DIRS must be successfully normalized"); + normalize_pathlist("XDG_CONFIG_DIRS", &[PathBuf::from("/etc/xdg")]) + .expect("XDG_CONFIG_DIRS must be successfully normalized"); + + // Normalize GStreamer plugin path + // https://gstreamer.freedesktop.org/documentation/gstreamer/gstregistry.html#gstregistry-page + normalize_pathlist( + "GST_PLUGIN_SYSTEM_PATH", + &[ + PathBuf::from("/usr/lib/gstreamer"), + data_home.join("gstreamer/plugins"), + ], + ) + .expect("GST_PLUGIN_SYSTEM_PATH must be successfully normalized"); + normalize_pathlist( + "GST_PLUGIN_SYSTEM_PATH_1_0", + &[ + PathBuf::from("/usr/lib/gstreamer-1.0"), + data_home.join("gstreamer-1.0/plugins"), + ], + ) + .expect("GST_PLUGIN_SYSTEM_PATH_1_0 must be successfully normalized"); + + // Normalize PATH + normalize_pathlist( + "PATH", + &[ + PathBuf::from("/sbin"), + PathBuf::from("/bin"), + PathBuf::from("/usr/sbin"), + PathBuf::from("/usr/bin"), + PathBuf::from("/usr/local/sbin"), + PathBuf::from("/usr/local/bin"), + PathBuf::from("/var/lib/flatpak/exports/bin"), + data_home.join("flatpak/exports/bin"), + ], + ) + .expect("PATH must be successfully normalized"); +} + +pub(crate) fn remove_prefix_from_pathlist( + env_name: &str, + prefix: &impl AsRef, +) -> Option { + env::var_os(env_name).and_then(|value| { + let mut dirs = env::split_paths(&value) + .filter(|dir| !(dir.as_os_str().is_empty() || dir.starts_with(prefix))) + .peekable(); + + if dirs.peek().is_none() { + None + } else { + Some(env::join_paths(dirs).expect("Should not fail because we are only filtering a pathlist retrieved from the environmnet")) + } + }) +} + +// Check if snap by looking if SNAP is set and not empty and that the SNAP directory exists +pub fn is_snap() -> bool { + if let Some(snap) = std::env::var_os("SNAP") { + if !snap.is_empty() && PathBuf::from(snap).is_dir() { + return true; + } + } + + false +} + +// Check if appimage by looking if APPDIR is set and is a valid directory +pub fn is_appimage() -> bool { + if let Some(appdir) = std::env::var_os("APPDIR").map(PathBuf::from) { + appdir.is_absolute() && appdir.is_dir() + } else { + false + } +} + +// Check if flatpak by looking if FLATPAK_ID is set and not empty and that the .flatpak-info file exists +pub fn is_flatpak() -> bool { + if let Some(flatpak_id) = std::env::var_os("FLATPAK_ID") { + if !flatpak_id.is_empty() && PathBuf::from("/.flatpak-info").is_file() { + return true; + } + } + + false +} diff --git a/apps/desktop/crates/linux/src/handler.rs b/apps/desktop/crates/linux/src/handler.rs deleted file mode 100644 index be0932d2db13..000000000000 --- a/apps/desktop/crates/linux/src/handler.rs +++ /dev/null @@ -1,54 +0,0 @@ -use std::{convert::TryFrom, ffi::OsString, fmt::Display, path::PathBuf, str::FromStr}; - -use mime::Mime; - -use crate::{DesktopEntry, Error, ExecMode, Result}; - -pub enum HandlerType { - Mime(Mime), - Ext(String), -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq, PartialOrd, Ord)] -pub struct Handler(OsString); - -impl Display for Handler { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(&self.0.to_string_lossy()) - } -} - -impl FromStr for Handler { - type Err = Error; - fn from_str(s: &str) -> Result { - let handler = Self::assume_valid(s.into()); - handler.get_entry()?; - Ok(handler) - } -} - -impl Handler { - pub fn assume_valid(name: OsString) -> Self { - Self(name) - } - - pub fn get_path(&self) -> Result { - let mut path = PathBuf::from("applications"); - path.push(&self.0); - xdg::BaseDirectories::new()? - .find_data_file(path) - .ok_or(Error::BadPath(self.0.to_string_lossy().to_string())) - } - - pub fn get_entry(&self) -> Result { - DesktopEntry::try_from(&self.get_path()?) - } - - pub fn launch(&self, args: &[&str]) -> Result<()> { - self.get_entry()?.exec(ExecMode::Launch, args) - } - - pub fn open(&self, args: &[&str]) -> Result<()> { - self.get_entry()?.exec(ExecMode::Open, args) - } -} diff --git a/apps/desktop/crates/linux/src/lib.rs b/apps/desktop/crates/linux/src/lib.rs index 617d0fc5aecc..14ddcb43e951 100644 --- a/apps/desktop/crates/linux/src/lib.rs +++ b/apps/desktop/crates/linux/src/lib.rs @@ -1,29 +1,7 @@ #![cfg(target_os = "linux")] -#[derive(Debug, thiserror::Error)] -pub enum Error { - #[error(transparent)] - Io(#[from] std::io::Error), - #[error(transparent)] - Xdg(#[from] xdg::BaseDirectoriesError), - #[error("no handlers found for '{0}'")] - NotFound(String), - #[error("bad Desktop Entry exec line: {0}")] - InvalidExec(String), - #[error("malformed desktop entry at {0}")] - BadEntry(std::path::PathBuf), - #[error("Please specify the default terminal with handlr set x-scheme-handler/terminal")] - NoTerminal, - #[error("Bad path: {0}")] - BadPath(String), -} +mod app_info; +mod env; -pub type Result = std::result::Result; - -mod desktop_entry; -mod handler; -mod system; - -pub use desktop_entry::{DesktopEntry, Mode as ExecMode}; -pub use handler::{Handler, HandlerType}; -pub use system::SystemApps; +pub use app_info::{list_apps_associated_with_ext, open_file_path, open_files_path_with}; +pub use env::{is_appimage, is_flatpak, is_snap, normalize_environment}; diff --git a/apps/desktop/crates/linux/src/system.rs b/apps/desktop/crates/linux/src/system.rs deleted file mode 100644 index 7ea6fd07dff5..000000000000 --- a/apps/desktop/crates/linux/src/system.rs +++ /dev/null @@ -1,63 +0,0 @@ -use std::{ - collections::{BTreeSet, HashMap}, - convert::TryFrom, - ffi::OsStr, -}; - -use mime::Mime; -use xdg_mime::SharedMimeInfo; - -use crate::{DesktopEntry, Handler, HandlerType, Result}; - -#[derive(Debug, Default, Clone)] -pub struct SystemApps(pub HashMap>); - -impl SystemApps { - pub fn get_handlers(&self, handler_type: HandlerType) -> impl Iterator { - let mimes = match handler_type { - HandlerType::Ext(ext) => { - SharedMimeInfo::new().get_mime_types_from_file_name(ext.as_str()) - } - HandlerType::Mime(mime) => vec![mime], - }; - - let mut handlers: BTreeSet<&Handler> = BTreeSet::new(); - for mime in mimes { - if let Some(mime_handlers) = self.0.get(&mime) { - handlers.extend(mime_handlers.iter()); - } - } - - handlers.into_iter() - } - - pub fn get_handler(&self, handler_type: HandlerType) -> Option<&Handler> { - self.get_handlers(handler_type).next() - } - - pub fn get_entries() -> Result> { - Ok(xdg::BaseDirectories::new()? - .list_data_files_once("applications") - .into_iter() - .filter(|p| p.extension().map_or(false, |x| x == OsStr::new("desktop"))) - .filter_map(|p| DesktopEntry::try_from(&p).ok())) - } - - pub fn populate() -> Result { - let mut map = HashMap::>::with_capacity(50); - - Self::get_entries()?.for_each( - |DesktopEntry { - mimes, file_name, .. - }| { - mimes.into_iter().for_each(|mime| { - map.entry(mime) - .or_default() - .insert(Handler::assume_valid(file_name.clone())); - }); - }, - ); - - Ok(Self(map)) - } -} diff --git a/apps/desktop/crates/linux/tests/cmus.desktop b/apps/desktop/crates/linux/tests/cmus.desktop deleted file mode 100644 index a26121d85c16..000000000000 --- a/apps/desktop/crates/linux/tests/cmus.desktop +++ /dev/null @@ -1,12 +0,0 @@ -[Desktop Entry] -Encoding=UTF-8 -Version=1.0 -Type=Application -Display=true -Exec=bash -c "(! pgrep cmus && tilix -e cmus && tilix -a session-add-down -e cava); sleep 0.1 && cmus-remote -q %f" -Terminal=false -Name=cmus-remote -Comment=Music player cmus-remote control -NoDisplay=true -Icon=cmus -MimeType=audio/mp3;audio/ogg; diff --git a/apps/desktop/crates/macos/Cargo.toml b/apps/desktop/crates/macos/Cargo.toml index dcff8a575bf2..e6d612df977c 100644 --- a/apps/desktop/crates/macos/Cargo.toml +++ b/apps/desktop/crates/macos/Cargo.toml @@ -6,8 +6,10 @@ repository = { workspace = true } edition = { workspace = true } [dependencies] -swift-rs = { workspace = true, features = ["serde"] } serde = { version = "1.0" } -[build-dependencies] +[target.'cfg(target_os = "macos")'.dependencies] +swift-rs = { workspace = true, features = ["serde"] } + +[target.'cfg(target_os = "macos")'.build-dependencies] swift-rs = { workspace = true, features = ["build"] } diff --git a/apps/desktop/crates/macos/src-swift/files.swift b/apps/desktop/crates/macos/src-swift/files.swift index 8a421201c751..4af38e41f50b 100644 --- a/apps/desktop/crates/macos/src-swift/files.swift +++ b/apps/desktop/crates/macos/src-swift/files.swift @@ -1,22 +1,35 @@ import AppKit import SwiftRs +extension NSBitmapImageRep { + var png: Data? { representation(using: .png, properties: [:]) } +} + +extension Data { + var bitmap: NSBitmapImageRep? { NSBitmapImageRep(data: self) } +} + +extension NSImage { + var png: Data? { tiffRepresentation?.bitmap?.png } +} + class OpenWithApplication: NSObject { - var name: SRString; - var id: SRString; - var url: SRString; + var name: SRString + var id: SRString + var url: SRString + var icon: SRData - init(name: SRString, id: SRString, url: SRString) { + init(name: SRString, id: SRString, url: SRString, icon: SRData) { self.name = name self.id = id self.url = url + self.icon = icon } } @_cdecl("get_open_with_applications") func getOpenWithApplications(urlString: SRString) -> SRObjectArray { - let url: URL; - + let url: URL if #available(macOS 13.0, *) { url = URL(filePath: urlString.toString()) } else { @@ -24,48 +37,84 @@ func getOpenWithApplications(urlString: SRString) -> SRObjectArray { url = URL(fileURLWithPath: urlString.toString()) } - + let appURLs: [URL] if #available(macOS 12.0, *) { - return SRObjectArray(NSWorkspace.shared.urlsForApplications(toOpen: url) - .compactMap { url in - Bundle(url: url)?.infoDictionary.map { ($0, url) } - } - .compactMap { (dict, url) -> NSObject? in - guard let name = (dict["CFBundleDisplayName"] ?? dict["CFBundleName"]) as? String else { - return nil - }; - - if !url.path.contains("/Applications/") { - return nil - } - - return OpenWithApplication( - name: SRString(name), - id: SRString(dict["CFBundleIdentifier"] as! String), - url: SRString(url.path) - ) - }) + appURLs = NSWorkspace.shared.urlsForApplications(toOpen: url) } else { - // Fallback on earlier versions - return SRObjectArray([]) + // Fallback for macOS versions prior to 12 + + // Get type identifier from file URL + let fileType: String + if #available(macOS 11.0, *) { + guard let _fileType = (try? url.resourceValues(forKeys: [.typeIdentifierKey]))?.typeIdentifier + else { + print("Failed to fetch file type for the specified file URL") + return SRObjectArray([]) + } + + fileType = _fileType + } else { + // Fallback for macOS versions prior to 11 + guard + let _fileType = UTTypeCreatePreferredIdentifierForTag( + kUTTagClassFilenameExtension, url.pathExtension as CFString, nil)?.takeRetainedValue() + else { + print("Failed to fetch file type for the specified file URL") + return SRObjectArray([]) + } + fileType = _fileType as String + } + + // Locates an array of bundle identifiers for apps capable of handling a specified content type with the specified roles. + guard + let bundleIds = LSCopyAllRoleHandlersForContentType(fileType as CFString, LSRolesMask.all)? + .takeRetainedValue() as? [String] + else { + print("Failed to fetch bundle IDs for the specified file type") + return SRObjectArray([]) + } + + // Retrieve all URLs for the app identified by a bundle id + appURLs = bundleIds.compactMap { bundleId -> URL? in + guard let retVal = LSCopyApplicationURLsForBundleIdentifier(bundleId as CFString, nil) else { + return nil + } + return retVal.takeRetainedValue() as? URL + } } + + return SRObjectArray( + appURLs.compactMap { url -> NSObject? in + guard !url.path.contains("/Applications/"), + let infoDict = Bundle(url: url)?.infoDictionary, + let name = (infoDict["CFBundleDisplayName"] ?? infoDict["CFBundleName"]) as? String, + let appId = infoDict["CFBundleIdentifier"] as? String + else { + return nil + } + + let icon = NSWorkspace.shared.icon(forFile: url.path) + + return OpenWithApplication( + name: SRString(name), + id: SRString(appId), + url: SRString(url.path), + icon: SRData([UInt8](icon.png ?? Data())) + ) + }) } @_cdecl("open_file_path_with") -func openFilePathWith(fileUrl: SRString, withUrl: SRString) { - let config = NSWorkspace.OpenConfiguration(); - - let at = URL(fileURLWithPath: withUrl.toString()); - print(at); +func openFilePathsWith(filePath: SRString, withUrl: SRString) { + let config = NSWorkspace.OpenConfiguration() + let at = URL(fileURLWithPath: withUrl.toString()) - NSWorkspace.shared.open( - [URL(fileURLWithPath: fileUrl.toString())], - withApplicationAt: at, - configuration: config - ) + // FIX-ME(HACK): The NULL split here is because I was not able to make this function accept a SRArray argument. + // So, considering these are file paths, and \0 is not a valid character for a file path, + // I am using it as a delimitor to allow the rust side to pass in an array of files paths to this function + let fileURLs = filePath.toString().split(separator: "\0").map { + filePath in URL(fileURLWithPath: String(filePath)) + } -// NSWorkspace.shared.openApplication(at: at, configuration: config) { (app, err) in -// print(app) -// print(err) -// } + NSWorkspace.shared.open(fileURLs, withApplicationAt: at, configuration: config) } diff --git a/apps/desktop/crates/macos/src/lib.rs b/apps/desktop/crates/macos/src/lib.rs index 4aceb304317f..7cbb1317e941 100644 --- a/apps/desktop/crates/macos/src/lib.rs +++ b/apps/desktop/crates/macos/src/lib.rs @@ -21,7 +21,13 @@ pub struct OpenWithApplication { pub name: SRString, pub id: SRString, pub url: SRString, + pub icon: SRData, } swift!(pub fn get_open_with_applications(url: &SRString) -> SRObjectArray); -swift!(pub fn open_file_path_with(file_url: &SRString, with_url: &SRString)); +swift!(pub(crate) fn open_file_path_with(file_url: &SRString, with_url: &SRString)); + +pub fn open_file_paths_with(file_urls: &[&str], with_url: &str) { + let file_url = file_urls.join("\0"); + unsafe { open_file_path_with(&file_url.as_str().into(), &with_url.into()) } +} diff --git a/apps/desktop/crates/windows/Cargo.toml b/apps/desktop/crates/windows/Cargo.toml index b09ecacb7b4a..8ca87856105b 100644 --- a/apps/desktop/crates/windows/Cargo.toml +++ b/apps/desktop/crates/windows/Cargo.toml @@ -8,8 +8,8 @@ edition = { workspace = true } [dependencies] thiserror = "1.0.40" normpath = "1.1.1" -libc = "0.2.146" +libc = "0.2" -[dependencies.windows] +[target.'cfg(target_os = "windows")'.dependencies.windows] version = "0.48" features = ["Win32_UI_Shell", "Win32_System_Com"] diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml index 9ecffc668af2..5cbdf0e27fcd 100644 --- a/apps/desktop/src-tauri/Cargo.toml +++ b/apps/desktop/src-tauri/Cargo.toml @@ -30,6 +30,7 @@ opener = { version = "0.6.1", features = ["reveal"] } specta = { workspace = true } tauri-specta = { workspace = true, features = ["typescript"] } uuid = { version = "1.3.3", features = ["serde"] } +futures = "0.3" prisma-client-rust = { workspace = true } sd-prisma = { path = "../../../crates/prisma" } diff --git a/apps/desktop/src-tauri/src/file.rs b/apps/desktop/src-tauri/src/file.rs index 91528d59e8fa..27f42490da90 100644 --- a/apps/desktop/src-tauri/src/file.rs +++ b/apps/desktop/src-tauri/src/file.rs @@ -1,5 +1,6 @@ use std::{ - collections::{BTreeSet, HashMap}, + collections::{BTreeSet, HashMap, HashSet}, + hash::{Hash, Hasher}, sync::Arc, }; @@ -38,10 +39,17 @@ pub async fn open_file_paths( .into_iter() .map(|(id, maybe_path)| { if let Some(path) = maybe_path { - opener::open(path) + #[cfg(target_os = "linux")] + let open_result = sd_desktop_linux::open_file_path(&path); + + #[cfg(not(target_os = "linux"))] + let open_result = opener::open(path); + + open_result .map(|_| OpenFilePathResult::AllGood(id)) - .unwrap_or_else(|e| { - OpenFilePathResult::OpenError(id, e.to_string()) + .unwrap_or_else(|err| { + error!("Failed to open logs dir: {err}"); + OpenFilePathResult::OpenError(id, err.to_string()) }) } else { OpenFilePathResult::NoFile(id) @@ -57,16 +65,26 @@ pub async fn open_file_paths( Ok(res) } -#[derive(Serialize, Type)] +#[derive(Serialize, Type, Debug, Clone)] pub struct OpenWithApplication { - id: i32, - name: String, - #[cfg(target_os = "linux")] - url: std::path::PathBuf, - #[cfg(not(target_os = "linux"))] url: String, + name: String, } +impl Hash for OpenWithApplication { + fn hash(&self, state: &mut H) { + self.url.hash(state); + } +} + +impl PartialEq for OpenWithApplication { + fn eq(&self, other: &Self) -> bool { + self.url == other.url + } +} + +impl Eq for OpenWithApplication {} + #[tauri::command(async)] #[specta::specta] pub async fn get_file_path_open_with_apps( @@ -87,125 +105,110 @@ pub async fn get_file_path_open_with_apps( }; #[cfg(target_os = "macos")] - return Ok(paths - .into_iter() - .flat_map(|(id, path)| { - let Some(path) = path - else { - error!("File not found in database"); - return vec![]; - }; + return { + Ok(paths + .into_values() + .flat_map(|path| { + let Some(path) = path.and_then(|path| path.into_os_string().into_string().ok()) + else { + error!("File not found in database"); + return None; + }; - unsafe { sd_desktop_macos::get_open_with_applications(&path.to_str().unwrap().into()) } - .as_slice() - .iter() - .map(|app| OpenWithApplication { - id, - name: app.name.to_string(), - url: app.url.to_string(), - }) - .collect::>() - }) - .collect()); + Some( + unsafe { sd_desktop_macos::get_open_with_applications(&path.as_str().into()) } + .as_slice() + .iter() + .map(|app| OpenWithApplication { + url: app.url.to_string(), + name: app.name.to_string(), + }) + .collect::>(), + ) + }) + .reduce(|intersection, set| intersection.intersection(&set).cloned().collect()) + .map(|set| set.into_iter().collect()) + .unwrap_or(vec![])) + }; #[cfg(target_os = "linux")] { - use sd_desktop_linux::{DesktopEntry, HandlerType, SystemApps}; + use futures::future; + use sd_desktop_linux::list_apps_associated_with_ext; - // TODO: cache this, and only update when the underlying XDG desktop apps changes - let Ok(system_apps) = SystemApps::populate() - .map_err(|e| { error!("{e:#?}"); }) - else { - return Ok(vec![]); - }; - - return Ok(paths - .into_iter() - .flat_map(|(id, path)| { - let Some(path) = path + let apps = future::join_all(paths.into_values().map(|path| async { + let Some(path) = path else { error!("File not found in database"); - return vec![]; + return None; }; - let Some(name) = path.file_name() - .and_then(|name| name.to_str()) - .map(|name| name.to_string()) - else { - error!("Failed to extract file name"); - return vec![]; - }; - - system_apps - .get_handlers(HandlerType::Ext(name)) - .map(|handler| { - handler - .get_path() - .map_err(|e| { - error!("{e:#?}"); - }) - .and_then(|path| { - DesktopEntry::try_from(&path) - // TODO: Ignore desktop entries that have commands that don't exist/aren't available in path - .map(|entry| OpenWithApplication { - id, - name: entry.name, - url: path, - }) - .map_err(|e| { - error!("{e:#?}"); - }) - }) + Some( + list_apps_associated_with_ext(&path) + .await + .into_iter() + .map(|app| OpenWithApplication { + url: app.id, + name: app.name, }) - .collect::, _>>() - .unwrap_or(vec![]) - }) - .collect()); + .collect::>(), + ) + })) + .await; + + return Ok(apps + .into_iter() + .flatten() + .reduce(|intersection, set| intersection.intersection(&set).cloned().collect()) + .map(|set| set.into_iter().collect()) + .unwrap_or(vec![])); } #[cfg(windows)] return Ok(paths - .into_iter() - .flat_map(|(id, path)| { + .into_values() + .filter_map(|path| { let Some(path) = path else { error!("File not found in database"); - return vec![]; + return None; }; let Some(ext) = path.extension() else { error!("Failed to extract file extension"); - return vec![]; + return None; }; sd_desktop_windows::list_apps_associated_with_ext(ext) .map_err(|e| { error!("{e:#?}"); }) - .map(|handlers| { - handlers - .iter() - .filter_map(|handler| { - let (Ok(name), Ok(url)) = ( - unsafe { handler.GetUIName() }.map_err(|e| { error!("{e:#?}");}) - .and_then(|name| unsafe { name.to_string() } - .map_err(|e| { error!("{e:#?}");})), - unsafe { handler.GetName() }.map_err(|e| { error!("{e:#?}");}) - .and_then(|name| unsafe { name.to_string() } - .map_err(|e| { error!("{e:#?}");})), - ) else { - error!("Failed to get handler info"); - return None - }; + .ok() + }) + .map(|handler| { + handler + .iter() + .filter_map(|handler| { + let (Ok(name), Ok(url)) = ( + unsafe { handler.GetUIName() }.map_err(|e| { error!("{e:#?}");}) + .and_then(|name| unsafe { name.to_string() } + .map_err(|e| { error!("{e:#?}");})), + unsafe { handler.GetName() }.map_err(|e| { error!("{e:#?}");}) + .and_then(|name| unsafe { name.to_string() } + .map_err(|e| { error!("{e:#?}");})), + ) else { + error!("Failed to get handler info"); + return None + }; - Some(OpenWithApplication { id, name, url }) - }) - .collect::>() + Some(OpenWithApplication { name, url }) }) - .unwrap_or(vec![]) + .collect::>() }) - .collect()); + .reduce(|intersection, set| intersection.intersection(&set).cloned().collect()) + .map(|set| set.into_iter().collect()) + .unwrap_or(vec![])); #[allow(unreachable_code)] Ok(vec![]) @@ -252,21 +255,14 @@ pub async fn open_file_path_with( #[cfg(target_os = "macos")] return { - unsafe { - sd_desktop_macos::open_file_path_with( - &path.into(), - &url.as_str().into(), - ) - }; + sd_desktop_macos::open_file_paths_with(&[path], url); Ok(()) }; #[cfg(target_os = "linux")] - return sd_desktop_linux::Handler::assume_valid(url.into()) - .open(&[path]) - .map_err(|e| { - error!("{e:#?}"); - }); + return sd_desktop_linux::open_files_path_with(&[path], url).map_err(|e| { + error!("{e:#?}"); + }); #[cfg(windows)] return sd_desktop_windows::open_file_path_with(path, url).map_err(|e| { @@ -344,7 +340,34 @@ pub async fn reveal_items( } for path in paths_to_open { - opener::reveal(path).ok(); + #[cfg(target_os = "linux")] + if sd_desktop_linux::is_appimage() { + // This is a workaround for the app, when package inside an AppImage, crashing when using opener::reveal. + sd_desktop_linux::open_file_path( + &(if path.is_file() { + path.parent().unwrap_or(&path) + } else { + &path + }), + ) + .map_err(|err| { + error!("Failed to open logs dir: {err}"); + }) + .ok() + } else { + opener::reveal(path) + .map_err(|err| { + error!("Failed to open logs dir: {err}"); + }) + .ok() + }; + + #[cfg(not(target_os = "linux"))] + opener::reveal(path) + .map_err(|err| { + error!("Failed to open logs dir: {err}"); + }) + .ok(); } Ok(()) diff --git a/apps/desktop/src-tauri/src/main.rs b/apps/desktop/src-tauri/src/main.rs index 58be59f65658..0b8238e60df4 100644 --- a/apps/desktop/src-tauri/src/main.rs +++ b/apps/desktop/src-tauri/src/main.rs @@ -9,7 +9,7 @@ use sd_core::{custom_uri::create_custom_uri_endpoint, Node, NodeError}; use tauri::{ api::path, async_runtime::block_on, ipc::RemoteDomainAccessScope, plugin::TauriPlugin, - AppHandle, Manager, RunEvent, Runtime, WindowEvent, + AppHandle, Manager, RunEvent, Runtime, }; use tokio::{task::block_in_place, time::sleep}; use tracing::{debug, error}; @@ -51,8 +51,17 @@ async fn reset_spacedrive(app_handle: AppHandle) { #[tauri::command(async)] #[specta::specta] async fn open_logs_dir(node: tauri::State<'_, Arc>) -> Result<(), ()> { - opener::open(node.data_dir.join("logs")).ok(); - Ok(()) + let logs_path = node.data_dir.join("logs"); + + #[cfg(target_os = "linux")] + let open_result = sd_desktop_linux::open_file_path(&logs_path); + + #[cfg(not(target_os = "linux"))] + let open_result = opener::open(logs_path); + + open_result.map_err(|err| { + error!("Failed to open logs dir: {err}"); + }) } pub fn tauri_error_plugin(err: NodeError) -> TauriPlugin { @@ -75,6 +84,9 @@ macro_rules! tauri_handlers { #[tokio::main] async fn main() -> tauri::Result<()> { + #[cfg(target_os = "linux")] + sd_desktop_linux::normalize_environment(); + #[cfg(target_os = "linux")] let (tx, rx) = tokio::sync::mpsc::channel(1); @@ -122,7 +134,7 @@ async fn main() -> tauri::Result<()> { // Instead, the window is hidden and the dock icon remains so that on user click it should show the window again. #[cfg(target_os = "macos")] let app = app.on_window_event(|event| { - if let WindowEvent::CloseRequested { api, .. } = event.event() { + if let tauri::WindowEvent::CloseRequested { api, .. } = event.event() { if event.window().label() == "main" { AppHandle::hide(&event.window().app_handle()).expect("Window should hide on macOS"); api.prevent_close(); diff --git a/apps/desktop/src/commands.ts b/apps/desktop/src/commands.ts index 3d136f3ef98f..696e03ad686b 100644 --- a/apps/desktop/src/commands.ts +++ b/apps/desktop/src/commands.ts @@ -42,7 +42,7 @@ export function lockAppTheme(themeType: AppThemeType) { return invoke()("lock_app_theme", { themeType }) } +export type AppThemeType = "Auto" | "Light" | "Dark" +export type OpenWithApplication = { url: string; name: string } export type OpenFilePathResult = { t: "NoLibrary" } | { t: "NoFile"; c: number } | { t: "OpenError"; c: [number, string] } | { t: "AllGood"; c: number } | { t: "Internal"; c: string } export type RevealItem = { Location: { id: number } } | { FilePath: { id: number } } -export type OpenWithApplication = { id: number; name: string; url: string } -export type AppThemeType = "Auto" | "Light" | "Dark" diff --git a/crates/ffmpeg/Cargo.toml b/crates/ffmpeg/Cargo.toml index e1efe2159f54..0099ae0934e9 100644 --- a/crates/ffmpeg/Cargo.toml +++ b/crates/ffmpeg/Cargo.toml @@ -9,8 +9,6 @@ license = { workspace = true } repository = { workspace = true } edition = { workspace = true } -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] ffmpeg-sys-next = "6.0.1" tracing = "0.1.37" diff --git a/crates/file-ext/Cargo.toml b/crates/file-ext/Cargo.toml index 4023b1a2868e..633a62ca569c 100644 --- a/crates/file-ext/Cargo.toml +++ b/crates/file-ext/Cargo.toml @@ -8,7 +8,6 @@ authors = [ license = { workspace = true } repository = { workspace = true } edition = { workspace = true } -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] serde = { version = "1.0.163", features = ["derive"] } diff --git a/crates/macos/Cargo.toml b/crates/macos/Cargo.toml index 157d4f6c80c3..ac866d9f4801 100644 --- a/crates/macos/Cargo.toml +++ b/crates/macos/Cargo.toml @@ -5,10 +5,8 @@ license = { workspace = true } repository = { workspace = true } edition = { workspace = true } -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] +[target.'cfg(target_os = "macos")'.dependencies] swift-rs = { workspace = true } -[build-dependencies] +[target.'cfg(target_os = "macos")'.build-dependencies] swift-rs = { workspace = true, features = ["build"] } diff --git a/crates/prisma/Cargo.toml b/crates/prisma/Cargo.toml index fca3f28af76e..5a6e7ab737a0 100644 --- a/crates/prisma/Cargo.toml +++ b/crates/prisma/Cargo.toml @@ -3,8 +3,6 @@ name = "sd-prisma" version = "0.1.0" edition = "2021" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] prisma-client-rust = { workspace = true } serde = "1.0" diff --git a/crates/sync-generator/Cargo.toml b/crates/sync-generator/Cargo.toml index 9fcc970314fa..f68b0d13f849 100644 --- a/crates/sync-generator/Cargo.toml +++ b/crates/sync-generator/Cargo.toml @@ -5,8 +5,6 @@ license = { workspace = true } repository = { workspace = true } edition = { workspace = true } -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] nom = "7.1.3" once_cell = "1.17.2" From 9283e74d8535b733e166f7ef0d3a83baed15fe3f Mon Sep 17 00:00:00 2001 From: "Ericson \"Fogo\" Soares" Date: Tue, 25 Jul 2023 09:47:56 -0300 Subject: [PATCH 5/7] [ENG-875] Re-index & Re-thumb should be smart (#1130) * New sub_path rescan rspc route * Changing some rescan call sites to use new route --- core/src/api/locations.rs | 31 +++++++++++++++++-- core/src/location/mod.rs | 1 - .../$libraryId/Explorer/ParentContextMenu.tsx | 4 +-- .../$libraryId/location/LocationOptions.tsx | 9 ++++-- packages/client/src/core.ts | 3 ++ 5 files changed, 41 insertions(+), 7 deletions(-) diff --git a/core/src/api/locations.rs b/core/src/api/locations.rs index f18fcb609af5..a828cc44b08f 100644 --- a/core/src/api/locations.rs +++ b/core/src/api/locations.rs @@ -2,8 +2,8 @@ use crate::{ invalidate_query, location::{ delete_location, find_location, indexer::rules::IndexerRuleCreateArgs, light_scan_location, - location_with_indexer_rules, relink_location, scan_location, LocationCreateArgs, - LocationError, LocationUpdateArgs, + location_with_indexer_rules, relink_location, scan_location, scan_location_sub_path, + LocationCreateArgs, LocationError, LocationUpdateArgs, }, prisma::{file_path, indexer_rule, indexer_rules_in_location, location, object, SortOrder}, util::AbortOnDrop, @@ -169,6 +169,33 @@ pub(crate) fn mount() -> AlphaRouter { }, ) }) + .procedure("subPathRescan", { + #[derive(Clone, Serialize, Deserialize, Type, Debug)] + pub struct RescanArgs { + pub location_id: location::id::Type, + pub sub_path: String, + } + + R.with2(library()).mutation( + |(_, library), + RescanArgs { + location_id, + sub_path, + }: RescanArgs| async move { + scan_location_sub_path( + &library, + find_location(&library, location_id) + .include(location_with_indexer_rules::include()) + .exec() + .await? + .ok_or(LocationError::IdNotFound(location_id))?, + sub_path, + ) + .await + .map_err(Into::into) + }, + ) + }) .procedure("quickRescan", { #[derive(Clone, Serialize, Deserialize, Type, Debug)] pub struct LightScanArgs { diff --git a/core/src/location/mod.rs b/core/src/location/mod.rs index 004e5b5da77d..ef309bb89684 100644 --- a/core/src/location/mod.rs +++ b/core/src/location/mod.rs @@ -402,7 +402,6 @@ pub async fn scan_location( .map_err(Into::into) } -#[cfg(feature = "location-watcher")] pub async fn scan_location_sub_path( library: &Arc, location: location_with_indexer_rules::Data, diff --git a/interface/app/$libraryId/Explorer/ParentContextMenu.tsx b/interface/app/$libraryId/Explorer/ParentContextMenu.tsx index 2d6999d097d5..50f522c63212 100644 --- a/interface/app/$libraryId/Explorer/ParentContextMenu.tsx +++ b/interface/app/$libraryId/Explorer/ParentContextMenu.tsx @@ -20,7 +20,7 @@ export default (props: PropsWithChildren) => { const generateThumbsForLocation = useLibraryMutation('jobs.generateThumbsForLocation'); const objectValidator = useLibraryMutation('jobs.objectValidator'); - const rescanLocation = useLibraryMutation('locations.fullRescan'); + const rescanLocation = useLibraryMutation('locations.subPathRescan'); const copyFiles = useLibraryMutation('files.copyFiles'); const cutFiles = useLibraryMutation('files.cutFiles'); @@ -110,7 +110,7 @@ export default (props: PropsWithChildren) => { try { await rescanLocation.mutateAsync({ location_id: parent.location.id, - reidentify_objects: false + sub_path: currentPath ?? '' }); } catch (error) { showAlertDialog({ diff --git a/interface/app/$libraryId/location/LocationOptions.tsx b/interface/app/$libraryId/location/LocationOptions.tsx index 7474df65e594..6e99da558cd0 100644 --- a/interface/app/$libraryId/location/LocationOptions.tsx +++ b/interface/app/$libraryId/location/LocationOptions.tsx @@ -19,7 +19,7 @@ const OptionButton = tw(TopBarButton)`w-full gap-1 !px-1.5 !py-1`; export default function LocationOptions({ location, path }: { location: Location; path: string }) { const navigate = useNavigate(); - const scanLocation = useLibraryMutation('locations.fullRescan'); + const scanLocationSubPath = useLibraryMutation('locations.subPathRescan'); const regenThumbs = useLibraryMutation('jobs.generateThumbsForLocation'); const archiveLocation = () => alert('Not implemented'); @@ -73,7 +73,12 @@ export default function LocationOptions({ location, path }: { location: Location - scanLocation.mutate({ location_id: location.id, reidentify_objects: false })}> + scanLocationSubPath.mutate( + { + location_id: location.id, + sub_path: path ?? '' + } + )}> Re-index diff --git a/packages/client/src/core.ts b/packages/client/src/core.ts index a624b9933729..df9e49657a0d 100644 --- a/packages/client/src/core.ts +++ b/packages/client/src/core.ts @@ -60,6 +60,7 @@ export type Procedures = { { key: "locations.indexer_rules.create", input: LibraryArgs, result: null } | { key: "locations.indexer_rules.delete", input: LibraryArgs, result: null } | { key: "locations.relink", input: LibraryArgs, result: null } | + { key: "locations.subPathRescan", input: LibraryArgs, result: null } | { key: "locations.update", input: LibraryArgs, result: null } | { key: "nodes.edit", input: ChangeNodeNameArgs, result: null } | { key: "notifications.test", input: never, result: null } | @@ -275,6 +276,8 @@ export type RenameMany = { from_pattern: FromPattern; to_pattern: string; from_f export type RenameOne = { from_file_path_id: number; to: string } +export type RescanArgs = { location_id: number; sub_path: string } + export type RuleKind = "AcceptFilesByGlob" | "RejectFilesByGlob" | "AcceptIfChildrenDirectoriesArePresent" | "RejectIfChildrenDirectoriesArePresent" export type SanitisedNodeConfig = { id: string; name: string; p2p_port: number | null; p2p_email: string | null; p2p_img_url: string | null } From 7215e40a3b678ef77420b46df193c3f4a6bbacb3 Mon Sep 17 00:00:00 2001 From: jake <77554505+brxken128@users.noreply.github.com> Date: Tue, 25 Jul 2023 15:21:50 +0100 Subject: [PATCH 6/7] [ENG-926] Prevent thumbnail destruction and fix the remover (#1127) * fix(core): thumbnail removal * chore(core): add todo * New actor on steroids * Improving thumbnail remover actor * Ignoring errors from files that doesn't exist --------- Co-authored-by: Ericson Soares --- Cargo.lock | 64 +++++ core/Cargo.toml | 3 + core/src/library/library.rs | 15 +- core/src/library/manager.rs | 23 +- core/src/location/file_path_helper/mod.rs | 17 +- core/src/location/indexer/indexer_job.rs | 12 +- core/src/location/indexer/mod.rs | 47 +++- core/src/location/indexer/shallow.rs | 11 +- core/src/location/indexer/walk.rs | 24 +- core/src/location/mod.rs | 2 +- core/src/object/fs/delete.rs | 1 - core/src/object/preview/thumbnail/mod.rs | 24 +- core/src/object/thumbnail_remover.rs | 310 +++++++++++++++++----- 13 files changed, 426 insertions(+), 127 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 46e83631d7c6..f53f24e31da1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -350,6 +350,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "async-channel" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" +dependencies = [ + "concurrent-queue", + "event-listener", + "futures-core", +] + [[package]] name = "async-io" version = "1.13.0" @@ -723,6 +734,18 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "blake2" version = "0.10.6" @@ -2473,6 +2496,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futf" version = "0.1.5" @@ -2508,6 +2537,17 @@ dependencies = [ "futures-sink", ] +[[package]] +name = "futures-concurrency" +version = "7.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b726119e6cd29cf120724495b2085e1ed3d17821ea17b86de54576d1aa565f5e" +dependencies = [ + "bitvec", + "futures-core", + "pin-project", +] + [[package]] name = "futures-core" version = "0.3.28" @@ -6288,6 +6328,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "radix_trie" version = "0.2.1" @@ -7024,6 +7070,7 @@ dependencies = [ name = "sd-core" version = "0.1.0" dependencies = [ + "async-channel", "async-stream", "async-trait", "base64 0.21.2", @@ -7033,6 +7080,7 @@ dependencies = [ "dashmap", "enumflags2 0.7.7", "futures", + "futures-concurrency", "globset", "hex", "hostname", @@ -7075,6 +7123,7 @@ dependencies = [ "thiserror", "tokio", "tokio-stream", + "tokio-util", "tracing 0.2.0", "tracing-appender", "tracing-subscriber 0.3.0", @@ -8304,6 +8353,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "tar" version = "0.4.38" @@ -10295,6 +10350,15 @@ dependencies = [ "windows-implement", ] +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "x11" version = "2.21.0" diff --git a/core/Cargo.toml b/core/Cargo.toml index b6a40d3e661c..f737ca3d2bc1 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -97,6 +97,9 @@ regex = "1.8.4" hex = "0.4.3" int-enum = "0.5.0" tokio-stream = "0.1.14" +futures-concurrency = "7.3" +async-channel = "1.9" +tokio-util = "0.7" [target.'cfg(target_os = "macos")'.dependencies] plist = "1" diff --git a/core/src/library/library.rs b/core/src/library/library.rs index 63aa7b444327..ed61c917daa1 100644 --- a/core/src/library/library.rs +++ b/core/src/library/library.rs @@ -9,9 +9,8 @@ use crate::{ }, node::NodeConfigManager, object::{ - orphan_remover::OrphanRemoverActor, - preview::{get_thumbnail_path, THUMBNAIL_CACHE_DIR_NAME}, - thumbnail_remover::ThumbnailRemoverActor, + orphan_remover::OrphanRemoverActor, preview::get_thumbnail_path, + thumbnail_remover::ThumbnailRemoverActorProxy, }, prisma::{file_path, location, PrismaClient}, util::{db::maybe_missing, error::FileIOError}, @@ -51,7 +50,7 @@ pub struct Library { /// p2p identity pub identity: Arc, pub orphan_remover: OrphanRemoverActor, - pub thumbnail_remover: ThumbnailRemoverActor, + pub thumbnail_remover_proxy: ThumbnailRemoverActorProxy, } impl Debug for Library { @@ -81,13 +80,7 @@ impl Library { let library = Self { orphan_remover: OrphanRemoverActor::spawn(db.clone()), - thumbnail_remover: ThumbnailRemoverActor::spawn( - db.clone(), - node_context - .config - .data_directory() - .join(THUMBNAIL_CACHE_DIR_NAME), - ), + thumbnail_remover_proxy: library_manager.thumbnail_remover_proxy(), id, db, config, diff --git a/core/src/library/manager.rs b/core/src/library/manager.rs index 8e906eea508e..baac310d5158 100644 --- a/core/src/library/manager.rs +++ b/core/src/library/manager.rs @@ -2,7 +2,11 @@ use crate::{ invalidate_query, location::{indexer, LocationManagerError}, node::{NodeConfig, Platform}, - object::tag, + object::{ + preview::get_thumbnails_directory, + tag, + thumbnail_remover::{ThumbnailRemoverActor, ThumbnailRemoverActorProxy}, + }, prisma::location, util::{ db::{self, MissingFieldError}, @@ -37,6 +41,8 @@ pub struct LibraryManager { libraries: RwLock>>, /// node_context holds the context for the node which this library manager is running on. pub node_context: Arc, + /// An actor that removes stale thumbnails from the file system + thumbnail_remover: ThumbnailRemoverActor, } #[derive(Error, Debug)] @@ -101,6 +107,7 @@ impl LibraryManager { let this = Arc::new(Self { libraries_dir: libraries_dir.clone(), libraries: Default::default(), + thumbnail_remover: ThumbnailRemoverActor::new(get_thumbnails_directory(&node_context)), node_context, }); @@ -124,7 +131,11 @@ impl LibraryManager { .file_stem() .and_then(|v| v.to_str().map(Uuid::from_str)) else { - warn!("Attempted to load library from path '{}' but it has an invalid filename. Skipping...", config_path.display()); + warn!( + "Attempted to load library from path '{}' \ + but it has an invalid filename. Skipping...", + config_path.display() + ); continue; }; @@ -149,6 +160,10 @@ impl LibraryManager { Ok(this) } + pub fn thumbnail_remover_proxy(&self) -> ThumbnailRemoverActorProxy { + self.thumbnail_remover.proxy() + } + /// create creates a new library with the given config and mounts it into the running [LibraryManager]. pub(crate) async fn create( self: &Arc, @@ -325,6 +340,7 @@ impl LibraryManager { invalidate_query!(library, "library.list"); + self.thumbnail_remover.remove_library(id).await; self.libraries.write().await.retain(|l| l.id != id); Ok(()) @@ -417,7 +433,8 @@ impl LibraryManager { self.clone(), )); - self.libraries.write().await.push(library.clone()); + self.thumbnail_remover.new_library(&library).await; + self.libraries.write().await.push(Arc::clone(&library)); if should_seed { library.orphan_remover.invoke().await; diff --git a/core/src/location/file_path_helper/mod.rs b/core/src/location/file_path_helper/mod.rs index 66c9f5ab4356..8e5775e9bcc2 100644 --- a/core/src/location/file_path_helper/mod.rs +++ b/core/src/location/file_path_helper/mod.rs @@ -5,7 +5,6 @@ use crate::{ use std::{ fs::Metadata, - hash::{Hash, Hasher}, path::{Path, PathBuf, MAIN_SEPARATOR_STR}, time::SystemTime, }; @@ -24,7 +23,7 @@ pub use isolated_file_path_data::{ }; // File Path selectables! -file_path::select!(file_path_just_pub_id { pub_id }); +file_path::select!(file_path_pub_and_cas_ids { pub_id cas_id }); file_path::select!(file_path_just_pub_id_materialized_path { pub_id materialized_path @@ -104,20 +103,6 @@ file_path::select!(file_path_to_full_path { // File Path includes! file_path::include!(file_path_with_object { object }); -impl Hash for file_path_just_pub_id::Data { - fn hash(&self, state: &mut H) { - self.pub_id.hash(state); - } -} - -impl PartialEq for file_path_just_pub_id::Data { - fn eq(&self, other: &Self) -> bool { - self.pub_id == other.pub_id - } -} - -impl Eq for file_path_just_pub_id::Data {} - #[derive(Clone, Copy, Debug, Serialize, Deserialize)] pub struct FilePathMetadata { pub inode: u64, diff --git a/core/src/location/indexer/indexer_job.rs b/core/src/location/indexer/indexer_job.rs index f13be22132bc..8c094a570652 100644 --- a/core/src/location/indexer/indexer_job.rs +++ b/core/src/location/indexer/indexer_job.rs @@ -196,6 +196,17 @@ impl StatefulJob for IndexerJobInit { ) .await?; let scan_read_time = scan_start.elapsed(); + let to_remove = to_remove.collect::>(); + + ctx.library + .thumbnail_remover_proxy + .remove_cas_ids( + to_remove + .iter() + .filter_map(|file_path| file_path.cas_id.clone()) + .collect::>(), + ) + .await; let db_delete_start = Instant::now(); // TODO pass these uuids to sync system @@ -451,7 +462,6 @@ impl StatefulJob for IndexerJobInit { if run_metadata.total_updated_paths > 0 { // Invoking orphan remover here as we probably have some orphans objects due to updates ctx.library.orphan_remover.invoke().await; - ctx.library.thumbnail_remover.invoke().await; } Ok(Some(json!({"init: ": init, "run_metadata": run_metadata}))) diff --git a/core/src/location/indexer/mod.rs b/core/src/location/indexer/mod.rs index 7a0497e7c277..2289f1d98503 100644 --- a/core/src/location/indexer/mod.rs +++ b/core/src/location/indexer/mod.rs @@ -19,7 +19,7 @@ use thiserror::Error; use tracing::trace; use super::{ - file_path_helper::{file_path_just_pub_id, FilePathError, IsolatedFilePathData}, + file_path_helper::{file_path_pub_and_cas_ids, FilePathError, IsolatedFilePathData}, location_with_indexer_rules, }; @@ -280,7 +280,7 @@ fn iso_file_path_factory( } async fn remove_non_existing_file_paths( - to_remove: impl IntoIterator, + to_remove: impl IntoIterator, db: &PrismaClient, ) -> Result { db.file_path() @@ -333,6 +333,25 @@ macro_rules! file_paths_db_fetcher_fn { macro_rules! to_remove_db_fetcher_fn { ($location_id:expr, $db:expr) => {{ |iso_file_path, unique_location_id_materialized_path_name_extension_params| async { + struct PubAndCasId { + pub_id: ::uuid::Uuid, + maybe_cas_id: Option, + } + + impl ::std::hash::Hash for PubAndCasId { + fn hash(&self, state: &mut H) { + self.pub_id.hash(state); + } + } + + impl ::std::cmp::PartialEq for PubAndCasId { + fn eq(&self, other: &Self) -> bool { + self.pub_id == other.pub_id + } + } + + impl ::std::cmp::Eq for PubAndCasId {} + let iso_file_path: $crate::location::file_path_helper::IsolatedFilePathData<'static> = iso_file_path; @@ -354,7 +373,9 @@ macro_rules! to_remove_db_fetcher_fn { ::prisma_client_rust::operator::or(unique_params.collect()), ]), ]) - .select($crate::location::file_path_helper::file_path_just_pub_id::select()) + .select( + $crate::location::file_path_helper::file_path_pub_and_cas_ids::select(), + ) }) .collect::<::std::vec::Vec<_>>(); @@ -367,9 +388,10 @@ macro_rules! to_remove_db_fetcher_fn { .map(|fetched_vec| { fetched_vec .into_iter() - .map(|fetched| { - ::uuid::Uuid::from_slice(&fetched.pub_id) - .expect("file_path.pub_id is invalid!") + .map(|fetched| PubAndCasId { + pub_id: ::uuid::Uuid::from_slice(&fetched.pub_id) + .expect("file_path.pub_id is invalid!"), + maybe_cas_id: fetched.cas_id, }) .collect::<::std::collections::HashSet<_>>() }) @@ -377,19 +399,20 @@ macro_rules! to_remove_db_fetcher_fn { let mut intersection = ::std::collections::HashSet::new(); while let Some(set) = sets.pop() { - for pub_id in set { + for pub_and_cas_ids in set { // Remove returns true if the element was present in the set - if sets.iter_mut().all(|set| set.remove(&pub_id)) { - intersection.insert(pub_id); + if sets.iter_mut().all(|set| set.remove(&pub_and_cas_ids)) { + intersection.insert(pub_and_cas_ids); } } } intersection .into_iter() - .map(|pub_id| { - $crate::location::file_path_helper::file_path_just_pub_id::Data { - pub_id: pub_id.as_bytes().to_vec(), + .map(|pub_and_cas_ids| { + $crate::location::file_path_helper::file_path_pub_and_cas_ids::Data { + pub_id: pub_and_cas_ids.pub_id.as_bytes().to_vec(), + cas_id: pub_and_cas_ids.maybe_cas_id, } }) .collect() diff --git a/core/src/location/indexer/shallow.rs b/core/src/location/indexer/shallow.rs index bfbe0f94ae47..003766d41495 100644 --- a/core/src/location/indexer/shallow.rs +++ b/core/src/location/indexer/shallow.rs @@ -80,6 +80,16 @@ pub async fn shallow( .await? }; + library + .thumbnail_remover_proxy + .remove_cas_ids( + to_remove + .iter() + .filter_map(|file_path| file_path.cas_id.clone()) + .collect::>(), + ) + .await; + errors.into_iter().for_each(|e| error!("{e}")); // TODO pass these uuids to sync system @@ -116,7 +126,6 @@ pub async fn shallow( invalidate_query!(library, "search.paths"); library.orphan_remover.invoke().await; - library.thumbnail_remover.invoke().await; Ok(()) } diff --git a/core/src/location/indexer/walk.rs b/core/src/location/indexer/walk.rs index 85b370055700..3f44588b1afe 100644 --- a/core/src/location/indexer/walk.rs +++ b/core/src/location/indexer/walk.rs @@ -1,6 +1,6 @@ use crate::{ location::file_path_helper::{ - file_path_just_pub_id, file_path_walker, FilePathMetadata, IsolatedFilePathData, + file_path_pub_and_cas_ids, file_path_walker, FilePathMetadata, IsolatedFilePathData, MetadataExt, }, prisma::file_path, @@ -109,7 +109,7 @@ pub struct WalkResult where Walked: Iterator, ToUpdate: Iterator, - ToRemove: Iterator, + ToRemove: Iterator, { pub walked: Walked, pub to_update: ToUpdate, @@ -136,13 +136,14 @@ pub(super) async fn walk( WalkResult< impl Iterator, impl Iterator, - impl Iterator, + impl Iterator, >, IndexerError, > where FilePathDBFetcherFut: Future, IndexerError>>, - ToRemoveDbFetcherFut: Future, IndexerError>>, + ToRemoveDbFetcherFut: + Future, IndexerError>>, { let root = root.as_ref(); @@ -204,13 +205,14 @@ pub(super) async fn keep_walking( WalkResult< impl Iterator, impl Iterator, - impl Iterator, + impl Iterator, >, IndexerError, > where FilePathDBFetcherFut: Future, IndexerError>>, - ToRemoveDbFetcherFut: Future, IndexerError>>, + ToRemoveDbFetcherFut: + Future, IndexerError>>, { let mut to_keep_walking = VecDeque::with_capacity(TO_WALK_QUEUE_INITIAL_CAPACITY); let mut indexed_paths = HashSet::with_capacity(WALK_SINGLE_DIR_PATHS_BUFFER_INITIAL_CAPACITY); @@ -259,14 +261,15 @@ pub(super) async fn walk_single_dir( ( impl Iterator, impl Iterator, - Vec, + Vec, Vec, ), IndexerError, > where FilePathDBFetcherFut: Future, IndexerError>>, - ToRemoveDbFetcherFut: Future, IndexerError>>, + ToRemoveDbFetcherFut: + Future, IndexerError>>, { let root = root.as_ref(); @@ -428,9 +431,10 @@ async fn inner_walk_single_dir( mut maybe_to_walk, errors, }: WorkingTable<'_>, -) -> Vec +) -> Vec where - ToRemoveDbFetcherFut: Future, IndexerError>>, + ToRemoveDbFetcherFut: + Future, IndexerError>>, { let Ok(iso_file_path_to_walk) = iso_file_path_factory(path, true).map_err(|e| errors.push(e)) else { diff --git a/core/src/location/mod.rs b/core/src/location/mod.rs index ef309bb89684..965c4e891880 100644 --- a/core/src/location/mod.rs +++ b/core/src/location/mod.rs @@ -713,7 +713,7 @@ pub async fn delete_directory( db.file_path().delete_many(children_params).exec().await?; library.orphan_remover.invoke().await; - library.thumbnail_remover.invoke().await; + invalidate_query!(library, "search.paths"); Ok(()) diff --git a/core/src/object/fs/delete.rs b/core/src/object/fs/delete.rs index e35767ea1a9b..2aa0c338c429 100644 --- a/core/src/object/fs/delete.rs +++ b/core/src/object/fs/delete.rs @@ -99,7 +99,6 @@ impl StatefulJob for FileDeleterJobInit { invalidate_query!(ctx.library, "search.paths"); ctx.library.orphan_remover.invoke().await; - ctx.library.thumbnail_remover.invoke().await; Ok(Some(json!({ "init": init }))) } diff --git a/core/src/object/preview/thumbnail/mod.rs b/core/src/object/preview/thumbnail/mod.rs index 10e676cc52ba..3fcb0ba99035 100644 --- a/core/src/object/preview/thumbnail/mod.rs +++ b/core/src/object/preview/thumbnail/mod.rs @@ -5,6 +5,7 @@ use crate::{ location::file_path_helper::{file_path_for_thumbnailer, FilePathError, IsolatedFilePathData}, prisma::location, util::{db::maybe_missing, error::FileIOError, version_manager::VersionManagerError}, + NodeContext, }; use std::{ @@ -41,13 +42,22 @@ pub const THUMBNAIL_CACHE_DIR_NAME: &str = "thumbnails"; /// This does not check if a thumbnail exists, it just returns the path that it would exist at pub fn get_thumbnail_path(library: &Library, cas_id: &str) -> PathBuf { - library - .config() - .data_directory() - .join(THUMBNAIL_CACHE_DIR_NAME) - .join(get_shard_hex(cas_id)) - .join(cas_id) - .with_extension("webp") + let mut thumb_path = library.config().data_directory(); + + thumb_path.push(THUMBNAIL_CACHE_DIR_NAME); + thumb_path.push(get_shard_hex(cas_id)); + thumb_path.push(cas_id); + thumb_path.set_extension("webp"); + + thumb_path +} + +pub fn get_thumbnails_directory(node_ctx: &NodeContext) -> PathBuf { + let mut thumb_path = node_ctx.config.data_directory(); + + thumb_path.push(THUMBNAIL_CACHE_DIR_NAME); + + thumb_path } // this is used to pass the relevant data to the frontend so it can request the thumbnail diff --git a/core/src/object/thumbnail_remover.rs b/core/src/object/thumbnail_remover.rs index e46a47e5e36d..830a95406d6c 100644 --- a/core/src/object/thumbnail_remover.rs +++ b/core/src/object/thumbnail_remover.rs @@ -1,21 +1,32 @@ use crate::{ + library::Library, prisma::{file_path, PrismaClient}, util::error::{FileIOError, NonUtf8PathError}, }; -use std::{collections::HashSet, ffi::OsStr, path::Path, sync::Arc, time::Duration}; +use std::{ + collections::{HashMap, HashSet}, + ffi::OsStr, + path::{Path, PathBuf}, + pin::pin, + sync::Arc, + time::Duration, +}; -use futures::future::try_join_all; +use async_channel as chan; +use futures::{stream::FuturesUnordered, FutureExt}; +use futures_concurrency::{future::TryJoin, stream::Merge}; use thiserror::Error; use tokio::{ - fs, select, - sync::mpsc, - time::{interval_at, Instant, MissedTickBehavior}, + fs, io, + time::{interval, MissedTickBehavior}, }; -use tracing::error; +use tokio_stream::{wrappers::IntervalStream, StreamExt}; +use tokio_util::sync::{CancellationToken, DropGuard}; +use tracing::{debug, error, trace}; +use uuid::Uuid; -const TEN_SECONDS: Duration = Duration::from_secs(10); -const FIVE_MINUTES: Duration = Duration::from_secs(5 * 60); +const HALF_HOUR: Duration = Duration::from_secs(30 * 60); #[derive(Error, Debug)] enum ThumbnailRemoverActorError { @@ -30,53 +41,221 @@ enum ThumbnailRemoverActorError { } #[derive(Clone)] +pub struct ThumbnailRemoverActorProxy { + cas_ids_to_delete_tx: chan::Sender>, + non_indexed_thumbnails_cas_ids_tx: chan::Sender, +} + +impl ThumbnailRemoverActorProxy { + pub async fn new_non_indexed_thumbnail(&self, cas_id: String) { + if self + .non_indexed_thumbnails_cas_ids_tx + .send(cas_id) + .await + .is_err() + { + error!("Thumbnail remover actor is dead"); + } + } + + pub async fn remove_cas_ids(&self, cas_ids: Vec) { + if self.cas_ids_to_delete_tx.send(cas_ids).await.is_err() { + error!("Thumbnail remover actor is dead"); + } + } +} + +enum DatabaseMessage { + Add(Uuid, Arc), + Remove(Uuid), +} + pub struct ThumbnailRemoverActor { - tx: mpsc::Sender<()>, + databases_tx: chan::Sender, + cas_ids_to_delete_tx: chan::Sender>, + non_indexed_thumbnails_cas_ids_tx: chan::Sender, + _cancel_loop: DropGuard, } impl ThumbnailRemoverActor { - pub fn spawn(db: Arc, thumbnails_directory: impl AsRef) -> Self { - let (tx, mut rx) = mpsc::channel(4); + pub fn new(thumbnails_directory: impl AsRef) -> Self { let thumbnails_directory = thumbnails_directory.as_ref().to_path_buf(); + let (databases_tx, databases_rx) = chan::bounded(4); + let (non_indexed_thumbnails_cas_ids_tx, non_indexed_thumbnails_cas_ids_rx) = + chan::unbounded(); + let (cas_ids_to_delete_tx, cas_ids_to_delete_rx) = chan::bounded(16); + let cancel_token = CancellationToken::new(); + let inner_cancel_token = cancel_token.child_token(); tokio::spawn(async move { - let mut last_checked = Instant::now(); + loop { + if let Err(e) = tokio::spawn(Self::worker( + thumbnails_directory.clone(), + databases_rx.clone(), + cas_ids_to_delete_rx.clone(), + non_indexed_thumbnails_cas_ids_rx.clone(), + inner_cancel_token.child_token(), + )) + .await + { + error!( + "Error on Thumbnail Remover Actor; \ + Error: {e}; \ + Restarting the worker loop...", + ); + } + if inner_cancel_token.is_cancelled() { + break; + } + } + }); - let mut check_interval = interval_at(Instant::now() + FIVE_MINUTES, FIVE_MINUTES); - check_interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + Self { + databases_tx, + cas_ids_to_delete_tx, + non_indexed_thumbnails_cas_ids_tx, + _cancel_loop: cancel_token.drop_guard(), + } + } - loop { - // Here we wait for a signal or for the tick interval to be reached - select! { - _ = check_interval.tick() => {} - signal = rx.recv() => { - if signal.is_none() { - break; + pub async fn new_library(&self, Library { id, db, .. }: &Library) { + if self + .databases_tx + .send(DatabaseMessage::Add(*id, Arc::clone(db))) + .await + .is_err() + { + error!("Thumbnail remover actor is dead") + } + } + + pub async fn remove_library(&self, library_id: Uuid) { + if self + .databases_tx + .send(DatabaseMessage::Remove(library_id)) + .await + .is_err() + { + error!("Thumbnail remover actor is dead") + } + } + + pub fn proxy(&self) -> ThumbnailRemoverActorProxy { + ThumbnailRemoverActorProxy { + cas_ids_to_delete_tx: self.cas_ids_to_delete_tx.clone(), + non_indexed_thumbnails_cas_ids_tx: self.non_indexed_thumbnails_cas_ids_tx.clone(), + } + } + + async fn worker( + thumbnails_directory: PathBuf, + databases_rx: chan::Receiver, + cas_ids_to_delete_rx: chan::Receiver>, + non_indexed_thumbnails_cas_ids_rx: chan::Receiver, + cancel_token: CancellationToken, + ) { + let mut check_interval = interval(HALF_HOUR); + check_interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + + let mut databases = HashMap::new(); + let mut non_indexed_thumbnails_cas_ids = HashSet::new(); + + enum StreamMessage { + Run, + ToDelete(Vec), + Database(DatabaseMessage), + NonIndexedThumbnail(String), + Stop, + } + + let cancel = pin!(cancel_token.cancelled()); + + let mut msg_stream = ( + databases_rx.map(StreamMessage::Database), + cas_ids_to_delete_rx.map(StreamMessage::ToDelete), + non_indexed_thumbnails_cas_ids_rx.map(StreamMessage::NonIndexedThumbnail), + IntervalStream::new(check_interval).map(|_| StreamMessage::Run), + cancel.into_stream().map(|()| StreamMessage::Stop), + ) + .merge(); + + while let Some(msg) = msg_stream.next().await { + match msg { + StreamMessage::Run => { + // For any of them we process a clean up if a time since the last one already passed + if !databases.is_empty() { + if let Err(e) = Self::process_clean_up( + &thumbnails_directory, + databases.values(), + &non_indexed_thumbnails_cas_ids, + ) + .await + { + error!("Got an error when trying to clean stale thumbnails: {e:#?}"); } } } - - // For any of them we process a clean up if a time since the last one already passed - if last_checked.elapsed() > TEN_SECONDS { - if let Err(e) = Self::process_clean_up(&db, &thumbnails_directory).await { - error!("Got an error when trying to clean stale thumbnails: {e:#?}"); + StreamMessage::ToDelete(cas_ids) => { + if let Err(e) = Self::remove_by_cas_ids(&thumbnails_directory, cas_ids).await { + error!("Got an error when trying to remove thumbnails: {e:#?}"); } - last_checked = Instant::now(); } - } - }); - Self { tx } + StreamMessage::Database(DatabaseMessage::Add(id, db)) => { + databases.insert(id, db); + } + StreamMessage::Database(DatabaseMessage::Remove(id)) => { + databases.remove(&id); + } + StreamMessage::NonIndexedThumbnail(cas_id) => { + non_indexed_thumbnails_cas_ids.insert(cas_id); + } + StreamMessage::Stop => { + debug!("Thumbnail remover actor is stopping"); + break; + } + } + } } - pub async fn invoke(&self) { - self.tx.send(()).await.ok(); + async fn remove_by_cas_ids( + thumbnails_directory: &Path, + cas_ids: Vec, + ) -> Result<(), ThumbnailRemoverActorError> { + cas_ids + .into_iter() + .map(|cas_id| async move { + let thumbnail_path = + thumbnails_directory.join(format!("{}/{}.webp", &cas_id[0..2], &cas_id[2..])); + + trace!("Removing thumbnail: {}", thumbnail_path.display()); + + match fs::remove_file(&thumbnail_path).await { + Ok(()) => Ok(()), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(FileIOError::from((thumbnail_path, e))), + } + }) + .collect::>() + .try_join() + .await?; + + Ok(()) } async fn process_clean_up( - db: &PrismaClient, thumbnails_directory: &Path, + databases: impl Iterator>, + non_indexed_thumbnails_cas_ids: &HashSet, ) -> Result<(), ThumbnailRemoverActorError> { + let databases = databases.collect::>(); + + // Thumbnails directory have the following structure: + // thumbnails/ + // ├── version.txt + //└── [0..2]/ # sharding + // └── [2..].webp + let mut read_dir = fs::read_dir(thumbnails_directory) .await .map_err(|e| FileIOError::from((thumbnails_directory, e)))?; @@ -104,7 +283,7 @@ impl ThumbnailRemoverActor { .to_str() .ok_or_else(|| NonUtf8PathError(entry.path().into_boxed_path()))?; - let mut thumbnails_paths_by_cas_id = Vec::new(); + let mut thumbnails_paths_by_cas_id = HashMap::new(); let mut entry_read_dir = fs::read_dir(&entry_path) .await @@ -134,7 +313,7 @@ impl ThumbnailRemoverActor { .ok_or_else(|| NonUtf8PathError(entry.path().into_boxed_path()))?; thumbnails_paths_by_cas_id - .push((format!("{}{}", entry_path_name, thumbnail_name), thumb_path)); + .insert(format!("{}{}", entry_path_name, thumbnail_name), thumb_path); } if thumbnails_paths_by_cas_id.is_empty() { @@ -145,38 +324,41 @@ impl ThumbnailRemoverActor { continue; } - let thumbs_in_db = db - .file_path() - .find_many(vec![file_path::cas_id::in_vec( - thumbnails_paths_by_cas_id - .iter() - .map(|(cas_id, _)| cas_id) - .cloned() - .collect(), - )]) - .select(file_path::select!({ cas_id })) - .exec() - .await? - .into_iter() - .map(|file_path| { - file_path - .cas_id - .expect("only file paths with a cas_id were queried") + let mut thumbs_in_db_futs = databases + .iter() + .map(|db| { + db.file_path() + .find_many(vec![file_path::cas_id::in_vec( + thumbnails_paths_by_cas_id.keys().cloned().collect(), + )]) + .select(file_path::select!({ cas_id })) + .exec() }) - .collect::>(); + .collect::>(); - try_join_all( - thumbnails_paths_by_cas_id + while let Some(maybe_thumbs_in_db) = thumbs_in_db_futs.next().await { + maybe_thumbs_in_db? .into_iter() - .filter_map(|(cas_id, path)| { - (!thumbs_in_db.contains(&cas_id)).then_some(async move { - fs::remove_file(&path) - .await - .map_err(|e| FileIOError::from((path, e))) - }) - }), - ) - .await?; + .filter_map(|file_path| file_path.cas_id) + .for_each(|cas_id| { + thumbnails_paths_by_cas_id.remove(&cas_id); + }); + } + + thumbnails_paths_by_cas_id + .retain(|cas_id, _| !non_indexed_thumbnails_cas_ids.contains(cas_id)); + + thumbnails_paths_by_cas_id + .into_values() + .map(|path| async move { + trace!("Removing stale thumbnail: {}", path.display()); + fs::remove_file(&path) + .await + .map_err(|e| FileIOError::from((path, e))) + }) + .collect::>() + .try_join() + .await?; } Ok(()) From 528657bd1949d3198fba672323786a05394215a2 Mon Sep 17 00:00:00 2001 From: Utku <74243531+utkubakir@users.noreply.github.com> Date: Wed, 26 Jul 2023 11:59:35 +0300 Subject: [PATCH 7/7] [MOB-31] Separate analytics for mobile (#1132) * 1 line change pr damn * more lines --- apps/mobile/src/main.tsx | 27 ++++++++++++++++++++++ packages/client/src/hooks/usePlausible.tsx | 2 ++ 2 files changed, 29 insertions(+) diff --git a/apps/mobile/src/main.tsx b/apps/mobile/src/main.tsx index fe2960fbc8f4..6d44ec8c73b2 100644 --- a/apps/mobile/src/main.tsx +++ b/apps/mobile/src/main.tsx @@ -3,6 +3,7 @@ import 'event-target-polyfill'; import * as SplashScreen from 'expo-splash-screen'; import { Suspense, lazy } from 'react'; import { Platform } from 'react-native'; +import { Dimensions } from 'react-native'; import { reactNativeLink } from './lib/rspcReactNativeTransport'; // Enable the splash screen @@ -61,6 +62,32 @@ globalThis.rspcLinks = [ reactNativeLink() ]; +// Polyfill for Plausible to work properly (@sd/client/hooks/usePlausible) + +window.location = { + // @ts-ignore + ancestorOrigins: {}, + href: 'https://spacedrive.com', + origin: 'https://spacedrive.com', + protocol: 'https:', + host: 'spacedrive.com', + hostname: 'spacedrive.com', + port: '', + pathname: '/', + search: '', + hash: '' +}; +// @ts-ignore +window.document = {}; + +const { width, height } = Dimensions.get('window'); + +//@ts-ignore +window.screen = { + width, + height +}; + /* https://github.com/facebook/hermes/issues/23 diff --git a/packages/client/src/hooks/usePlausible.tsx b/packages/client/src/hooks/usePlausible.tsx index 6109bfdb928a..b38ac13ba8c7 100644 --- a/packages/client/src/hooks/usePlausible.tsx +++ b/packages/client/src/hooks/usePlausible.tsx @@ -8,6 +8,7 @@ import { PlausiblePlatformType, telemetryStore, useTelemetryState } from './useT */ const VERSION = '0.1.0'; const DOMAIN = 'app.spacedrive.com'; +const MOBILE_DOMAIN = 'mobile.spacedrive.com'; const PlausibleProvider = Plausible({ trackLocalhost: true, @@ -190,6 +191,7 @@ const submitPlausibleEvent = async ({ event, debugState, ...props }: SubmitEvent debug: debugState.enabled }, options: { + domain: props.platformType === 'mobile' ? MOBILE_DOMAIN : DOMAIN, deviceWidth: props.screenWidth ?? window.screen.width, referrer: '', ...('plausibleOptions' in event ? event.plausibleOptions : undefined)