fix: more work on syncing

Author: ErikBjare

aw-sync/src/lib.rs

@@ -7,6 +7,7 @@ extern crate serde_json;
 mod sync;
 pub use sync::sync_datastores;
 pub use sync::sync_run;
+pub use sync::SyncSpec;
 
 mod accessmethod;
 pub use accessmethod::AccessMethod;

aw-sync/src/main.rs

@@ -60,7 +60,7 @@ enum Commands {
         /// Specify buckets to sync
         /// If not specified, all buckets will be synced
         #[clap(long)]
-        buckets: Vec<String>,
+        buckets: Option<Vec<String>>,
     },
     /// List buckets and their sync status
     List {},
@@ -100,8 +100,13 @@ fn main() -> std::io::Result<()> {
                     .unwrap()
                     .with_timezone(&chrono::Utc)
             });
+            let sync_spec = sync::SyncSpec {
+                path: sync_directory.to_path_buf(),
+                buckets: buckets.clone(),
+                start,
+            };
 
-            sync::sync_run(sync_directory, client, buckets, start).map_err(|e| {
+            sync::sync_run(client, &sync_spec).map_err(|e| {
                 println!("Error: {}", e);
                 std::io::Error::new(std::io::ErrorKind::Other, e)
             })?;

aw-sync/src/sync.rs

@@ -20,49 +20,49 @@ use aw_models::{Bucket, Event};
 
 use crate::accessmethod::AccessMethod;
 
-fn setup_local_remote(client: &AwClient, sync_directory: &Path) -> Result<Datastore, String> {
-    // FIXME: Don't run twice if already exists
-    fs::create_dir_all(sync_directory).unwrap();
-
-    let info = client.get_info().unwrap();
-    let remotedir = sync_directory.join(info.device_id.as_str());
-    fs::create_dir_all(&remotedir).unwrap();
-
-    let dbfile = remotedir
-        .join("test.db")
-        .into_os_string()
-        .into_string()
-        .unwrap();
-
-    let ds_localremote = Datastore::new(dbfile, false);
-    info!("Set up remote for local device");
+pub struct SyncSpec {
+    /// Path of sync folder
+    pub path: PathBuf,
+    /// Bucket IDs to sync
+    pub buckets: Option<Vec<String>>,
+    /// Start of time range to sync
+    pub start: Option<DateTime<Utc>>,
+}
 
-    Ok(ds_localremote)
+impl Default for SyncSpec {
+    fn default() -> Self {
+        // TODO: Better default path
+        let path = Path::new("/tmp/aw-sync").to_path_buf();
+        SyncSpec {
+            path,
+            buckets: None,
+            start: None,
+        }
+    }
 }
 
 /// Performs a single sync pass
-pub fn sync_run(
-    sync_directory: &Path,
-    client: AwClient,
-    buckets: &Vec<String>,
-    start: Option<DateTime<Utc>>,
-) -> Result<(), String> {
-    let ds_localremote = setup_local_remote(&client, sync_directory)?;
+pub fn sync_run(client: AwClient, sync_spec: &SyncSpec) -> Result<(), String> {
+    let ds_localremote = setup_local_remote(&client, sync_spec.path.as_path())?;
 
     //let ds_remotes = setup_test(sync_directory).unwrap();
     //info!("Set up remotes for testing");
 
     let info = client.get_info().unwrap();
-    let remote_dbfiles = find_remotes_nonlocal(sync_directory, info.device_id.as_str());
+    let remote_dbfiles = find_remotes_nonlocal(sync_spec.path.as_path(), info.device_id.as_str());
     info!("Found remotes: {:?}", remote_dbfiles);
 
     // TODO: Check for compatible remote db version before opening
-    let ds_remotes: Vec<Datastore> = remote_dbfiles.iter().map(create_datastore).collect();
+    let ds_remotes: Vec<Datastore> = remote_dbfiles
+        .iter()
+        .map(|p| p.as_path())
+        .map(create_datastore)
+        .collect();
 
     // Pull
     info!("Pulling...");
     for ds_from in &ds_remotes {
-        sync_datastores(ds_from, &client, false, None, buckets);
+        sync_datastores(ds_from, &client, false, None, sync_spec);
     }
 
     // Push local server buckets to sync folder
@@ -72,10 +72,10 @@ pub fn sync_run(
         &ds_localremote,
         true,
         Some(info.device_id.as_str()),
-        buckets,
+        sync_spec,
     );
 
-    list_buckets(&client, sync_directory);
+    list_buckets(&client, sync_spec.path.as_path());
 
     Ok(())
 }
@@ -88,7 +88,11 @@ pub fn list_buckets(client: &AwClient, sync_directory: &Path) {
     info!("Found remotes: {:?}", remote_dbfiles);
 
     // TODO: Check for compatible remote db version before opening
-    let ds_remotes: Vec<Datastore> = remote_dbfiles.iter().map(create_datastore).collect();
+    let ds_remotes: Vec<Datastore> = remote_dbfiles
+        .iter()
+        .map(|p| p.as_path())
+        .map(create_datastore)
+        .collect();
 
     log_buckets(client);
     log_buckets(&ds_localremote);
@@ -97,6 +101,26 @@ pub fn list_buckets(client: &AwClient, sync_directory: &Path) {
     }
 }
 
+fn setup_local_remote(client: &AwClient, path: &Path) -> Result<Datastore, String> {
+    // FIXME: Don't run twice if already exists
+    fs::create_dir_all(path).unwrap();
+
+    let info = client.get_info().unwrap();
+    let remotedir = path.join(info.device_id.as_str());
+    fs::create_dir_all(&remotedir).unwrap();
+
+    let dbfile = remotedir
+        .join("test.db")
+        .into_os_string()
+        .into_string()
+        .unwrap();
+
+    let ds_localremote = Datastore::new(dbfile, false);
+    info!("Set up remote for local device");
+
+    Ok(ds_localremote)
+}
+
 /// Returns a list of all remote dbs
 fn find_remotes(sync_directory: &Path) -> std::io::Result<Vec<PathBuf>> {
     println!("{}", sync_directory.display());
@@ -130,9 +154,9 @@ fn find_remotes_nonlocal(sync_directory: &Path, device_id: &str) -> Vec<PathBuf>
         .collect()
 }
 
-fn create_datastore(dspath: &PathBuf) -> Datastore {
-    let pathstr = dspath.clone().into_os_string().into_string().unwrap();
-    Datastore::new(pathstr, false)
+fn create_datastore(path: &Path) -> Datastore {
+    let pathstr = path.as_os_str().to_str().unwrap();
+    Datastore::new(pathstr.to_string(), false)
 }
 
 // TODO: Move into tests
@@ -239,7 +263,7 @@ pub fn sync_datastores(
     ds_to: &dyn AccessMethod,
     is_push: bool,
     src_did: Option<&str>,
-    buckets: &[String],
+    sync_spec: &SyncSpec,
 ) {
     // FIXME: "-synced" should only be appended when synced to the local database, not to the
     // staging area for local buckets.
@@ -259,7 +283,7 @@ pub fn sync_datastores(
         })
         // If buckets vec isn't empty, filter out buckets not in the buckets vec
         .filter(|bucket| {
-            if buckets.is_empty() {
+            if let Some(buckets) = &sync_spec.buckets {
                 buckets.iter().any(|b_id| b_id == &bucket.id)
             } else {
                 true

aw-sync/tests/sync.rs

@@ -10,6 +10,7 @@ mod sync_tests {
     use aw_datastore::{Datastore, DatastoreError};
     use aw_models::{Bucket, Event};
     use aw_sync;
+    use aw_sync::SyncSpec;
 
     struct TestState {
         ds_src: Datastore,
@@ -101,7 +102,13 @@ mod sync_tests {
         let state = init_teststate();
         create_bucket(&state.ds_src, 0);
 
-        aw_sync::sync_datastores(&state.ds_src, &state.ds_dest, false, None, &[]);
+        aw_sync::sync_datastores(
+            &state.ds_src,
+            &state.ds_dest,
+            false,
+            None,
+            &SyncSpec::default(),
+        );
 
         let buckets_src: HashMap<String, Bucket> = state.ds_src.get_buckets().unwrap();
         let buckets_dest: HashMap<String, Bucket> = state.ds_dest.get_buckets().unwrap();
@@ -136,7 +143,13 @@ mod sync_tests {
             .heartbeat(bucket_id.as_str(), create_event("1"), 1.0)
             .unwrap();
 
-        aw_sync::sync_datastores(&state.ds_src, &state.ds_dest, false, None, &[]);
+        aw_sync::sync_datastores(
+            &state.ds_src,
+            &state.ds_dest,
+            false,
+            None,
+            &SyncSpec::default(),
+        );
 
         let all_datastores: Vec<&Datastore> =
             [&state.ds_src, &state.ds_dest].iter().cloned().collect();
@@ -150,7 +163,13 @@ mod sync_tests {
             .ds_src
             .heartbeat(bucket_id.as_str(), create_event("1"), 1.0)
             .unwrap();
-        aw_sync::sync_datastores(&state.ds_src, &state.ds_dest, false, None, &[]);
+        aw_sync::sync_datastores(
+            &state.ds_src,
+            &state.ds_dest,
+            false,
+            None,
+            &SyncSpec::default(),
+        );
 
         // Check again that new events were indeed synced
         check_synced_buckets_equal_to_src(&all_buckets_map);
@@ -163,7 +182,13 @@ mod sync_tests {
         let bucket_id = create_bucket(&state.ds_src, 0);
         create_events(&state.ds_src, bucket_id.as_str(), 10);
 
-        aw_sync::sync_datastores(&state.ds_src, &state.ds_dest, false, None, &[]);
+        aw_sync::sync_datastores(
+            &state.ds_src,
+            &state.ds_dest,
+            false,
+            None,
+            &SyncSpec::default(),
+        );
 
         let all_datastores: Vec<&Datastore> =
             [&state.ds_src, &state.ds_dest].iter().cloned().collect();
@@ -174,7 +199,13 @@ mod sync_tests {
 
         // Add some more events
         create_events(&state.ds_src, bucket_id.as_str(), 10);
-        aw_sync::sync_datastores(&state.ds_src, &state.ds_dest, false, None, &[]);
+        aw_sync::sync_datastores(
+            &state.ds_src,
+            &state.ds_dest,
+            false,
+            None,
+            &SyncSpec::default(),
+        );
 
         // Check again that new events were indeed synced
         check_synced_buckets_equal_to_src(&all_buckets_map);