From 72993e7c4f5b829928aa53e999c46751abfc0ae4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Fri, 12 May 2023 22:54:18 +0200 Subject: [PATCH] feat: allow specifying exactly which db file to sync with --- aw-datastore/src/worker.rs | 4 ++-- aw-sync/src/main.rs | 24 +++++++++++++++++++++++- aw-sync/src/sync.rs | 33 ++++++++++++++++++++++++++++----- aw-sync/test-sync-pull.sh | 23 ++++++++++++++++++++--- 4 files changed, 73 insertions(+), 11 deletions(-) diff --git a/aw-datastore/src/worker.rs b/aw-datastore/src/worker.rs index 86f82ae7..2cec48f5 100644 --- a/aw-datastore/src/worker.rs +++ b/aw-datastore/src/worker.rs @@ -171,7 +171,7 @@ impl DatastoreWorker { Ok((req, res_sender)) => (req, res_sender), Err(_) => { // All references to responder is gone, quit - info!("DB worker quitting"); + debug!("DB worker quitting"); self.quit = true; break; } @@ -205,7 +205,7 @@ impl DatastoreWorker { break; }; } - info!("DB Worker thread finished"); + debug!("DB Worker thread finished"); } fn handle_request( diff --git a/aw-sync/src/main.rs b/aw-sync/src/main.rs index efa47a8e..cc63d066 100644 --- a/aw-sync/src/main.rs +++ b/aw-sync/src/main.rs @@ -15,6 +15,7 @@ extern crate serde_json; use std::error::Error; use std::path::Path; +use std::path::PathBuf; use chrono::{DateTime, Datelike, TimeZone, Utc}; use clap::{Parser, Subcommand}; @@ -41,10 +42,15 @@ struct Opts { /// Convenience option for using the default testing host and port. #[clap(long)] testing: bool, - /// Path to sync directory. + /// Full path to sync directory. /// If not specified, exit. #[clap(long)] sync_dir: String, + /// Full path to sync db file + /// Useful for syncing buckets from a specific db file in the sync directory. + /// Must be a valid absolute path to a file in the sync directory. + #[clap(long)] + sync_db: Option, } #[derive(Subcommand)] @@ -90,6 +96,10 @@ fn main() -> Result<(), Box> { }; info!("Using sync dir: {}", sync_directory.display()); + if let Some(sync_db) = &opts.sync_db { + info!("Using sync db: {}", sync_db); + } + let port = if opts.testing && opts.port == DEFAULT_PORT { "5666" } else { @@ -121,8 +131,20 @@ fn main() -> Result<(), Box> { .as_ref() .map(|b| b.split(',').map(|s| s.to_string()).collect()); + let sync_db: Option = opts.sync_db.as_ref().map(|db| { + let db_path = Path::new(db); + if !db_path.is_absolute() { + panic!("Sync db path must be absolute"); + } + if !db_path.starts_with(sync_directory) { + panic!("Sync db path must be in sync directory"); + } + db_path.to_path_buf() + }); + let sync_spec = sync::SyncSpec { path: sync_directory.to_path_buf(), + path_db: sync_db, buckets: buckets_vec, start, }; diff --git a/aw-sync/src/sync.rs b/aw-sync/src/sync.rs index 7471118b..bbf26f6b 100644 --- a/aw-sync/src/sync.rs +++ b/aw-sync/src/sync.rs @@ -31,6 +31,9 @@ pub enum SyncMode { pub struct SyncSpec { /// Path of sync folder pub path: PathBuf, + /// Path of sync db + /// If None, will use all + pub path_db: Option, /// Bucket IDs to sync pub buckets: Option>, /// Start of time range to sync @@ -43,6 +46,7 @@ impl Default for SyncSpec { let path = Path::new("/tmp/aw-sync").to_path_buf(); SyncSpec { path, + path_db: None, buckets: None, start: None, } @@ -60,7 +64,11 @@ pub fn sync_run(client: AwClient, sync_spec: &SyncSpec, mode: SyncMode) -> Resul // FIXME: Bad device_id assumption? let ds_localremote = setup_local_remote(sync_spec.path.as_path(), device_id)?; - let remote_dbfiles = find_remotes_nonlocal(sync_spec.path.as_path(), device_id); + let remote_dbfiles = find_remotes_nonlocal( + sync_spec.path.as_path(), + device_id, + sync_spec.path_db.as_ref(), + ); // Log if remotes found // TODO: Only log remotes of interest @@ -127,7 +135,7 @@ pub fn list_buckets(client: &AwClient, sync_directory: &Path) -> Result<(), Stri let device_id = info.device_id.as_str(); let ds_localremote = setup_local_remote(sync_directory, device_id)?; - let remote_dbfiles = find_remotes_nonlocal(sync_directory, device_id); + let remote_dbfiles = find_remotes_nonlocal(sync_directory, device_id, None); info!("Found remotes: {:?}", remote_dbfiles); // TODO: Check for compatible remote db version before opening @@ -177,11 +185,15 @@ fn find_remotes(sync_directory: &Path) -> std::io::Result> { } /// Returns a list of all remotes, excluding local ones -fn find_remotes_nonlocal(sync_directory: &Path, device_id: &str) -> Vec { +fn find_remotes_nonlocal( + sync_directory: &Path, + device_id: &str, + sync_db: Option<&PathBuf>, +) -> Vec { let remotes_all = find_remotes(sync_directory).unwrap(); - // Filter out own remote remotes_all .into_iter() + // Filter out own remote .filter(|path| { !(path .clone() @@ -190,6 +202,14 @@ fn find_remotes_nonlocal(sync_directory: &Path, device_id: &str) -> Vec .unwrap() .contains(device_id)) }) + // If sync_db is Some, return only remotes in that path + .filter(|path| { + if let Some(sync_db) = sync_db { + path.starts_with(sync_db) + } else { + true + } + }) .collect() } @@ -231,7 +251,10 @@ fn get_or_create_sync_bucket( serde_json::json!(bucket_from.hostname), ); ds_to.create_bucket(&bucket_new).unwrap(); - ds_to.get_bucket(new_id.as_str()).unwrap() + match ds_to.get_bucket(new_id.as_str()) { + Ok(bucket) => bucket, + Err(e) => panic!("{e:?}"), + } } Err(e) => panic!("{e:?}"), } diff --git a/aw-sync/test-sync-pull.sh b/aw-sync/test-sync-pull.sh index b37c9870..89f4235a 100755 --- a/aw-sync/test-sync-pull.sh +++ b/aw-sync/test-sync-pull.sh @@ -27,8 +27,14 @@ SYNCROOTDIR="$HOME/ActivityWatchSync" function sync_host() { host=$1 SYNCDIR="$SYNCROOTDIR/$host" - for db in $(ls $SYNCDIR/*/*.db); do - AWSYNCPARAMS="--port $PORT --sync-dir $SYNCDIR" + dbs=$(find $SYNCDIR -name "*.db") + for db in $dbs; do + # workaround to avoid trying to sync empty database files (size 45056) + if [ "$(stat -c%s $db)" -lt 50000 ]; then + continue + fi + + AWSYNCPARAMS="--port $PORT --sync-dir $SYNCDIR --sync-db $db" BUCKETS="aw-watcher-window_$host,aw-watcher-afk_$host" echo "Syncing $db to $host" @@ -45,7 +51,18 @@ if [ -z "$host" ]; then echo "Syncing all hosts" sleep 0.5 # For each host in the sync directory, pull the data from each database file using aw-sync - for host in $(ls $SYNCROOTDIR); do + # Use `find` to get all directories in the sync directory + hostnames=$(find $SYNCROOTDIR -maxdepth 1 -type d -exec basename {} \;) + # filter out "erb-m2.local" + hostnames=$(echo $hostnames | tr ' ' '\n' | grep -v "erb-m2.local") + # filter out folder not containing subfolders with .db files + for host in $hostnames; do + if [ ! "$(find $SYNCROOTDIR/$host -name "*.db")" ]; then + hostnames=$(echo $hostnames | tr ' ' '\n' | grep -v $host) + fi + done + # Sync each host, file-by-file + for host in $hostnames; do sync_host $host done else