From e8c895c9798af6cca009b98e9f23cc68b44bba69 Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Thu, 18 Dec 2025 05:56:31 -0600 Subject: [PATCH 1/3] New push option to force pushing rewrites --- Parallel.Cli/Commands/PushCommand.cs | 2 +- Parallel.Core/IO/Syncing/BaseSyncManager.cs | 2 +- Parallel.Core/IO/Syncing/DeltaSyncManager.cs | 32 -------------------- Parallel.Core/IO/Syncing/FileSyncManager.cs | 4 +-- Parallel.Core/IO/Syncing/ISyncManager.cs | 4 +-- 5 files changed, 6 insertions(+), 38 deletions(-) delete mode 100644 Parallel.Core/IO/Syncing/DeltaSyncManager.cs diff --git a/Parallel.Cli/Commands/PushCommand.cs b/Parallel.Cli/Commands/PushCommand.cs index d128448..946a690 100644 --- a/Parallel.Cli/Commands/PushCommand.cs +++ b/Parallel.Cli/Commands/PushCommand.cs @@ -121,7 +121,7 @@ private async Task PushPathInternalAsync(ISyncManager syncManager, string path, } CommandLine.WriteLine(syncManager.RemoteVault, $"Backing up {files.Length:N0} files...", ConsoleColor.DarkGray); - int pushedFiles = await syncManager.PushFilesAsync(files, new ProgressReport(syncManager.RemoteVault, successFiles)); + int pushedFiles = await syncManager.PushFilesAsync(files, new ProgressReport(syncManager.RemoteVault, successFiles), force); CommandLine.WriteLine(syncManager.RemoteVault, $"Successfully pushed {pushedFiles:N0} files in {_sw.Elapsed}.", ConsoleColor.Green); } } diff --git a/Parallel.Core/IO/Syncing/BaseSyncManager.cs b/Parallel.Core/IO/Syncing/BaseSyncManager.cs index db87873..c49f693 100644 --- a/Parallel.Core/IO/Syncing/BaseSyncManager.cs +++ b/Parallel.Core/IO/Syncing/BaseSyncManager.cs @@ -106,7 +106,7 @@ public async Task DisconnectAsync() } /// - public abstract Task PushFilesAsync(SystemFile[] files, IProgressReporter progress); + public abstract Task PushFilesAsync(SystemFile[] files, IProgressReporter progress, bool overwrite); /// public abstract Task PullFilesAsync(SystemFile[] files, IProgressReporter progress); diff --git a/Parallel.Core/IO/Syncing/DeltaSyncManager.cs b/Parallel.Core/IO/Syncing/DeltaSyncManager.cs deleted file mode 100644 index af5c1cb..0000000 --- a/Parallel.Core/IO/Syncing/DeltaSyncManager.cs +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2025 Kyle Ebbinga - -using Parallel.Core.Diagnostics; -using Parallel.Core.Models; -using Parallel.Core.Settings; - -namespace Parallel.Core.IO.Syncing -{ - /// - /// Represents the way to sync files to an associated file system using file deltas. - /// - public class DeltaSyncManager : BaseSyncManager - { - /// - /// Initializes a new instance of the class. - /// - /// - public DeltaSyncManager(RemoteVaultConfig remoteVaultConfig) : base(remoteVaultConfig) { } - - /// - public override Task PushFilesAsync(SystemFile[] files, IProgressReporter progress) - { - throw new NotImplementedException(); - } - - /// - public override Task PullFilesAsync(SystemFile[] files, IProgressReporter progress) - { - throw new NotImplementedException(); - } - } -} \ No newline at end of file diff --git a/Parallel.Core/IO/Syncing/FileSyncManager.cs b/Parallel.Core/IO/Syncing/FileSyncManager.cs index 3e544cd..11a90ab 100644 --- a/Parallel.Core/IO/Syncing/FileSyncManager.cs +++ b/Parallel.Core/IO/Syncing/FileSyncManager.cs @@ -23,7 +23,7 @@ public class FileSyncManager : BaseSyncManager public FileSyncManager(LocalVaultConfig localVault) : base(localVault) { } /// - public override async Task PushFilesAsync(SystemFile[] files, IProgressReporter progress) + public override async Task PushFilesAsync(SystemFile[] files, IProgressReporter progress, bool overwrite) { if (files.Length == 0) return 0; int queued = 0, completed = 0, total = 0; @@ -45,7 +45,7 @@ public override async Task PushFilesAsync(SystemFile[] files, IProgressRepo { Log.Debug($"Pushing -> {file.LocalPath}"); file.RemotePath = PathBuilder.GetObjectPath(RemoteVault, file.CheckSum!); - long result = await StorageProvider.UploadFileAsync(file, false, ct); + long result = await StorageProvider.UploadFileAsync(file, overwrite, ct); if (result <= 0) { progress.Failed(new InvalidOperationException(), file); diff --git a/Parallel.Core/IO/Syncing/ISyncManager.cs b/Parallel.Core/IO/Syncing/ISyncManager.cs index f276537..4859228 100644 --- a/Parallel.Core/IO/Syncing/ISyncManager.cs +++ b/Parallel.Core/IO/Syncing/ISyncManager.cs @@ -48,9 +48,9 @@ public interface ISyncManager /// Pushes an array of files to a vault. /// /// - /// /// - Task PushFilesAsync(SystemFile[] files, IProgressReporter progress); + /// + Task PushFilesAsync(SystemFile[] files, IProgressReporter progress, bool overwrite); /// /// Pulls an array of files from a vault. From f371a05fbc10a0d6922b710a2813b97bf76ec6d2 Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Thu, 18 Dec 2025 06:27:57 -0600 Subject: [PATCH 2/3] Various log cleanups and optimizations --- Parallel.Cli/Commands/PullCommand.cs | 4 +-- Parallel.Cli/Commands/PushCommand.cs | 2 +- Parallel.Core/IO/Scanning/FileScanner.cs | 36 +++++---------------- Parallel.Core/IO/Syncing/FileSyncManager.cs | 6 ++-- Parallel.Core/Models/SystemFile.cs | 4 +-- 5 files changed, 16 insertions(+), 36 deletions(-) diff --git a/Parallel.Cli/Commands/PullCommand.cs b/Parallel.Cli/Commands/PullCommand.cs index 861eaf0..e7246ce 100644 --- a/Parallel.Cli/Commands/PullCommand.cs +++ b/Parallel.Cli/Commands/PullCommand.cs @@ -72,7 +72,7 @@ private async Task PullPathAsync(LocalVaultConfig vault, string path, DateTime t if (!File.Exists(file.LocalPath) || FileScanner.HasChanged(file, new SystemFile(file.LocalPath)) || force) pullFiles.Add(file); }); - Log.Debug($"Pulling {pullFiles.Count} files..."); + CommandLine.WriteLine(syncManager.RemoteVault, $"Pulling {pullFiles.Count:N0} files...", ConsoleColor.DarkGray); int pulledFiles = await syncManager.PullFilesAsync(pullFiles.ToArray(), new ProgressReport(vault, files.Count())); CommandLine.WriteLine(vault, $"Successfully pulled {pulledFiles:N0} files from '{vault.Credentials.RootDirectory}'.", ConsoleColor.Green); await syncManager.DisconnectAsync(); @@ -95,7 +95,7 @@ private async Task PullFileAsync(ISyncManager syncManager, string fullPath, bool return; } - Log.Debug($"Pulling '{fullPath}'"); + CommandLine.WriteLine(syncManager.RemoteVault, $"Pulling 1 file...", ConsoleColor.DarkGray); int pulledFiles = await syncManager.PullFilesAsync([remoteFile], new ProgressReport(syncManager.RemoteVault, 1)); CommandLine.WriteLine(syncManager.RemoteVault, $"Successfully pulled {pulledFiles:N0} file from '{syncManager.RemoteVault.Credentials.RootDirectory}'.", ConsoleColor.Green); await syncManager.DisconnectAsync(); diff --git a/Parallel.Cli/Commands/PushCommand.cs b/Parallel.Cli/Commands/PushCommand.cs index 946a690..5aa6b49 100644 --- a/Parallel.Cli/Commands/PushCommand.cs +++ b/Parallel.Cli/Commands/PushCommand.cs @@ -120,7 +120,7 @@ private async Task PushPathInternalAsync(ISyncManager syncManager, string path, return; } - CommandLine.WriteLine(syncManager.RemoteVault, $"Backing up {files.Length:N0} files...", ConsoleColor.DarkGray); + CommandLine.WriteLine(syncManager.RemoteVault, $"Pushing {files.Length:N0} files...", ConsoleColor.DarkGray); int pushedFiles = await syncManager.PushFilesAsync(files, new ProgressReport(syncManager.RemoteVault, successFiles), force); CommandLine.WriteLine(syncManager.RemoteVault, $"Successfully pushed {pushedFiles:N0} files in {_sw.Elapsed}.", ConsoleColor.Green); } diff --git a/Parallel.Core/IO/Scanning/FileScanner.cs b/Parallel.Core/IO/Scanning/FileScanner.cs index 41af071..0159224 100644 --- a/Parallel.Core/IO/Scanning/FileScanner.cs +++ b/Parallel.Core/IO/Scanning/FileScanner.cs @@ -45,10 +45,7 @@ public async Task GetFileChangesAsync(string path, string[] ignore ConcurrentBag scannedFiles = new(); ConcurrentBag changedFiles = new(); - Log.Debug("Getting system files..."); HashSet localFiles = FileScanner.GetFiles(path, ignoreFolders, ".").ToHashSet(); - - Log.Debug("Getting database files..."); IEnumerable remoteFiles = _db is null ? [] : await _db.GetLatestFilesAsync(path, false); System.Threading.Tasks.Parallel.ForEach(remoteFiles, ParallelConfig.Options, (remoteFile, ct) => { @@ -94,16 +91,16 @@ public async Task GetFileChangesAsync(string path, string[] ignore /// /// Gets if a file has changed. /// - /// The source file to compare. - /// The target file to compare to. + /// The source file to compare. + /// The target file to compare to. /// True is success, otherwise false. - public static bool HasChanged(SystemFile sourcePath, SystemFile? targetPath) + public static bool HasChanged(SystemFile source, SystemFile? target) { - if (string.IsNullOrEmpty(sourcePath.CheckSum)) sourcePath.TryGenerateCheckSum(); - return targetPath == null || (sourcePath.LastWrite.TotalMilliseconds > targetPath.LastWrite.TotalMilliseconds && Convert.ToBoolean(!sourcePath.CheckSum?.Equals(targetPath.CheckSum))); + if (target is null || !source.TryGenerateCheckSum()) return false; + if (source.LastWrite.TotalMilliseconds <= target.LastWrite.TotalMilliseconds) return false; + return source.CheckSum != target.CheckSum; } - /// /// Gets the total size, in bytes, of a directory. /// @@ -222,12 +219,11 @@ public static IEnumerable GetFiles(string root, string[] exempt, string IEnumerable files; try { - Log.Debug($"Scanning -> {current}"); files = Directory.EnumerateFiles(current, searchPattern); } catch { - Log.Debug($"No file access -> {current}"); + Log.Warning($"No file access -> {current}"); continue; } @@ -250,7 +246,7 @@ public static IEnumerable GetFiles(string root, string[] exempt, string } catch { - Log.Debug($"No directory access -> {current}"); + Log.Warning($"No directory access -> {current}"); continue; } @@ -273,8 +269,6 @@ public static Dictionary GetDuplicateFiles(string path) SystemFile entry = new(file); dict.AddOrUpdate(entry.Name, _ => [entry], (k, v) => { - Log.Debug($"Checking: {entry.LocalPath}"); - lock (v) { SystemFile? key = v.FirstOrDefault(); @@ -287,20 +281,6 @@ public static Dictionary GetDuplicateFiles(string path) return v; }); - - /*if (dict.TryGetValue(entry.Name, out List? value)) - { - SystemFile? key = value.FirstOrDefault(); - if (!HasChanged(entry, key)) - { - Log.Debug($"HasChanged: {entry.LocalPath}"); - value.Add(entry); - } - } - else - { - dict[entry.Name] = new List { entry }; - }*/ }); return dict.Where(kv => kv.Value.Count > 1).OrderByDescending(kv => kv.Value.Count).ToDictionary(k => k.Key, v => v.Value.OrderBy(l => l.LastWrite.TotalMilliseconds).ToArray()); diff --git a/Parallel.Core/IO/Syncing/FileSyncManager.cs b/Parallel.Core/IO/Syncing/FileSyncManager.cs index 11a90ab..5177550 100644 --- a/Parallel.Core/IO/Syncing/FileSyncManager.cs +++ b/Parallel.Core/IO/Syncing/FileSyncManager.cs @@ -37,7 +37,8 @@ public override async Task PushFilesAsync(SystemFile[] files, IProgressRepo Task worker = System.Threading.Tasks.Parallel.ForEachAsync(uploadFiles, ParallelConfig.Options, async (file, ct) => { Interlocked.Increment(ref queued); - if (string.IsNullOrEmpty(file.CheckSum) && !file.TryGenerateCheckSum()) return; + if (!file.TryGenerateCheckSum()) return; + SemaphoreSlim lockedThread = threadPool.GetOrAdd(file.CheckSum!, _ => new SemaphoreSlim(1, 1)); await lockedThread.WaitAsync(ct); @@ -103,7 +104,8 @@ public override async Task PullFilesAsync(SystemFile[] files, IProgressRepo Task worker = System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options, async (file, ct) => { Interlocked.Increment(ref queued); - if (string.IsNullOrEmpty(file.CheckSum) && !file.TryGenerateCheckSum()) return; + if (!file.TryGenerateCheckSum()) return; + SemaphoreSlim lockedThread = threadPool.GetOrAdd(file.CheckSum!, _ => new SemaphoreSlim(1, 1)); await lockedThread.WaitAsync(ct); diff --git a/Parallel.Core/Models/SystemFile.cs b/Parallel.Core/Models/SystemFile.cs index a9ec1a8..f9e7272 100644 --- a/Parallel.Core/Models/SystemFile.cs +++ b/Parallel.Core/Models/SystemFile.cs @@ -174,9 +174,7 @@ public bool TryGenerateCheckSum() try { - Log.Debug($"Generating checksum -> {LocalPath}"); if (!File.Exists(LocalPath)) return false; - using SHA256 sha256 = SHA256.Create(); using FileStream fs = File.OpenRead(LocalPath); CheckSum = Convert.ToHexStringLower(sha256.ComputeHash(fs)); @@ -184,7 +182,7 @@ public bool TryGenerateCheckSum() } catch (Exception ex) { - Log.Error(ex, $"Checksum generation failed -> {LocalPath}"); + Log.Error(ex.GetBaseException().ToString()); return false; } } From 7d18cf2afc711113e3513141356d8a964d42c8ab Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Thu, 18 Dec 2025 06:35:50 -0600 Subject: [PATCH 3/3] Fixed potential issue with ignoring files --- Parallel.Core/IO/Scanning/FileScanner.cs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Parallel.Core/IO/Scanning/FileScanner.cs b/Parallel.Core/IO/Scanning/FileScanner.cs index 0159224..be0dd9f 100644 --- a/Parallel.Core/IO/Scanning/FileScanner.cs +++ b/Parallel.Core/IO/Scanning/FileScanner.cs @@ -229,11 +229,12 @@ public static IEnumerable GetFiles(string root, string[] exempt, string foreach (string file in files) { - if (IsIgnored(file, exempt)) - { - Log.Debug($"Ignored -> {file}"); - continue; - } + // This is a better system. However, if a user adds something to ignore, with this code it will never mark it for deletion. + // if (IsIgnored(file, exempt)) + // { + // Log.Debug($"Ignored -> {file}"); + // continue; + // } yield return file; }