Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Implemented test for reproducing issue #5196 #5201

Merged
merged 1 commit into from
May 27, 2024
Merged

Implemented test for reproducing issue #5196 #5201

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 51 additions & 53 deletions Duplicati/Library/Main/Operation/BackupHandler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,23 +1,23 @@
// Copyright (C) 2024, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
// Copyright (C) 2024, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.

using System;
using System.Collections.Generic;
Expand All @@ -33,8 +33,6 @@
using Duplicati.Library.Common.IO;
using Duplicati.Library.Common;
using Duplicati.Library.Logging;
using Duplicati.Library.Main.Operation.Backup;
using Duplicati.Library.Main.Operation.Common;

namespace Duplicati.Library.Main.Operation
{
Expand All @@ -44,7 +42,7 @@ namespace Duplicati.Library.Main.Operation
/// to the chosen destination
/// </summary>
internal class BackupHandler : IDisposable
{
{
/// <summary>
/// The tag used for logging
/// </summary>
Expand All @@ -68,11 +66,11 @@ public BackupHandler(string backendurl, Options options, BackupResults results)
m_options = options;
m_result = results;
m_backendurl = backendurl;

if (options.AllowPassphraseChange)
throw new UserInformationException(Strings.Common.PassphraseChangeUnsupported, "PassphraseChangeUnsupported");
}

public static Snapshots.ISnapshotService GetSnapshot(string[] sources, Options options)
{
try
Expand Down Expand Up @@ -146,7 +144,7 @@ private UsnJournalService GetJournalService(IEnumerable<string> sources, ISnapsh
private void PreBackupVerify(BackendManager backend, string protectedfile)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PreBackupVerify);
using(new Logging.Timer(LOGTAG, "PreBackupVerify", "PreBackupVerify"))
using (new Logging.Timer(LOGTAG, "PreBackupVerify", "PreBackupVerify"))
{
try
{
Expand Down Expand Up @@ -275,23 +273,23 @@ private void CompactIfRequired(BackendManager backend, long lastVolumeSize)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Delete);
m_result.DeleteResults = new DeleteResults(m_result);
using(var db = new LocalDeleteDatabase(m_database))
using (var db = new LocalDeleteDatabase(m_database))
new DeleteHandler(backend.BackendUrl, m_options, (DeleteResults)m_result.DeleteResults).DoRun(db, ref m_transaction, true, currentIsSmall, backend);

}
else if (currentIsSmall && !m_options.NoAutoCompact)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Compact);
m_result.CompactResults = new CompactResults(m_result);
using(var db = new LocalDeleteDatabase(m_database))
using (var db = new LocalDeleteDatabase(m_database))
new CompactHandler(backend.BackendUrl, m_options, (CompactResults)m_result.CompactResults).DoCompact(db, true, ref m_transaction, backend);
}
}

private void PostBackupVerification(string currentFilelistVolume)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupVerify);
using(var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, m_database))
using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, m_database))
{
using (new Logging.Timer(LOGTAG, "AfterBackupVerify", "AfterBackupVerify"))
FilelistProcessor.VerifyRemoteList(backend, m_options, m_database, m_result.BackendWriter, new string[] { currentFilelistVolume });
Expand All @@ -305,8 +303,8 @@ private void PostBackupVerification(string currentFilelistVolume)
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupTest);
m_result.TestResults = new TestResults(m_result);

using(var testdb = new LocalTestDatabase(m_database))
using(var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, testdb))
using (var testdb = new LocalTestDatabase(m_database))
using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, testdb))
new TestHandler(m_backendurl, m_options, (TestResults)m_result.TestResults)
.DoRun(samplesToTest, testdb, backend);
}
Expand Down Expand Up @@ -341,7 +339,7 @@ private void UpdateStorageStatsFromDatabase()
}
}
}

m_result.BackendWriter.AssignedQuotaSpace = m_options.QuotaSize;
}
}
Expand All @@ -359,7 +357,7 @@ private static Exception BuildException(Exception source, params Task[] tasks)
var ex = new List<Exception>();
ex.Add(source);

foreach(var t in tasks)
foreach (var t in tasks)
if (t != null)
{
if (!t.IsCompleted && !t.IsFaulted && !t.IsCanceled)
Expand Down Expand Up @@ -401,26 +399,26 @@ private static async Task<long> FlushBackend(BackupResults result, IWriteChannel

private async Task RunAsync(string[] sources, Library.Utility.IFilter filter, CancellationToken token)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Begin);
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Begin);

// New isolated scope for each operation
using(new IsolatedChannelScope())
using(m_database = new LocalBackupDatabase(m_options.Dbpath, m_options))
using (new IsolatedChannelScope())
using (m_database = new LocalBackupDatabase(m_options.Dbpath, m_options))
{
m_result.SetDatabase(m_database);
m_result.Dryrun = m_options.Dryrun;

// Check the database integrity
Utility.UpdateOptionsFromDb(m_database, m_options);
Utility.VerifyParameters(m_database, m_options);
Utility.VerifyOptionsAndUpdateDatabase(m_database, m_options);

var probe_path = m_database.GetFirstPath();
if (probe_path != null && Util.GuessDirSeparator(probe_path) != Util.DirectorySeparatorString)
throw new UserInformationException(string.Format("The backup contains files that belong to another operating system. Proceeding with a backup would cause the database to contain paths from two different operation systems, which is not supported. To proceed without losing remote data, delete all filesets and make sure the --{0} option is set, then run the backup again to re-use the existing data on the remote store.", "no-auto-compact"), "CrossOsDatabaseReuseNotSupported");

if (m_database.PartiallyRecreated)
throw new UserInformationException("The database was only partially recreated. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsPartiallyRecreated");

if (m_database.RepairInProgress)
throw new UserInformationException("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the backup process cannot continue. You may delete the local database and attempt to repair it again.", "DatabaseRepairInProgress");

Expand All @@ -434,12 +432,12 @@ private async Task RunAsync(string[] sources, Library.Utility.IFilter filter, Ca
try
{
// Setup runners and instances here
using(var db = new Backup.BackupDatabase(m_database, m_options))
using(var backendManager = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, m_database))
using(var filesetvolume = new FilesetVolumeWriter(m_options, m_database.OperationTimestamp))
using(var stats = new Backup.BackupStatsCollector(m_result))
using (var db = new Backup.BackupDatabase(m_database, m_options))
using (var backendManager = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, m_database))
using (var filesetvolume = new FilesetVolumeWriter(m_options, m_database.OperationTimestamp))
using (var stats = new Backup.BackupStatsCollector(m_result))
// Keep a reference to these channels to avoid shutdown
using(var uploadtarget = ChannelManager.GetChannel(Backup.Channels.BackendRequest.ForWrite))
using (var uploadtarget = ChannelManager.GetChannel(Backup.Channels.BackendRequest.ForWrite))
{
long filesetid;
var counterToken = new CancellationTokenSource();
Expand Down Expand Up @@ -478,7 +476,7 @@ private async Task RunAsync(string[] sources, Library.Utility.IFilter filter, Ca
var prevfileset = m_database.FilesetTimes.FirstOrDefault();
if (prevfileset.Value.ToUniversalTime() > m_database.OperationTimestamp.ToUniversalTime())
throw new Exception(string.Format("The previous backup has time {0}, but this backup has time {1}. Something is wrong with the clock.", prevfileset.Value.ToLocalTime(), m_database.OperationTimestamp.ToLocalTime()));

var lastfilesetid = prevfileset.Value.Ticks == 0 ? -1 : prevfileset.Key;

// Rebuild any index files that are missing
Expand All @@ -488,7 +486,7 @@ private async Task RunAsync(string[] sources, Library.Utility.IFilter filter, Ca
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_ProcessingFiles);

var repcnt = 0;
while(repcnt < 100 && await db.GetRemoteVolumeIDAsync(filesetvolume.RemoteFilename) >= 0)
while (repcnt < 100 && await db.GetRemoteVolumeIDAsync(filesetvolume.RemoteFilename) >= 0)
filesetvolume.ResetRemoteFilename(m_options, m_database.OperationTimestamp.AddSeconds(repcnt++));

if (await db.GetRemoteVolumeIDAsync(filesetvolume.RemoteFilename) >= 0)
Expand Down Expand Up @@ -544,7 +542,7 @@ private async Task RunAsync(string[] sources, Library.Utility.IFilter filter, Ca

// TODO: Remove this later
m_transaction = m_database.BeginTransaction();

if (await m_result.TaskReader.ProgressAsync)
CompactIfRequired(backendManager, lastVolumeSize);

Expand All @@ -561,9 +559,9 @@ private async Task RunAsync(string[] sources, Library.Utility.IFilter filter, Ca
}
else
{
using(new Logging.Timer(LOGTAG, "CommitFinalizingBackup", "CommitFinalizingBackup"))
using (new Logging.Timer(LOGTAG, "CommitFinalizingBackup", "CommitFinalizingBackup"))
m_transaction.Commit();

m_transaction = null;

if (m_result.TaskControlRendevouz() != TaskControlState.Abort)
Expand All @@ -574,8 +572,8 @@ private async Task RunAsync(string[] sources, Library.Utility.IFilter filter, Ca
PostBackupVerification(filesetvolume.RemoteFilename);
}
}
m_database.WriteResults();

m_database.WriteResults();
m_database.PurgeLogData(m_options.LogRetention);
m_database.PurgeDeletedVolumes(DateTime.UtcNow);

Expand All @@ -594,7 +592,7 @@ private async Task RunAsync(string[] sources, Library.Utility.IFilter filter, Ca
Logging.Log.WriteErrorMessage(LOGTAG, "FatalError", ex, "Fatal error");
if (aex == ex)
throw;

throw aex;
}
finally
Expand All @@ -605,7 +603,7 @@ private async Task RunAsync(string[] sources, Library.Utility.IFilter filter, Ca
// TODO: We want to commit? always?
if (m_transaction != null)
try { m_transaction.Rollback(); }
catch (Exception ex) { Logging.Log.WriteErrorMessage(LOGTAG, "RollbackError", ex, "Rollback error: {0}", ex.Message); }
catch (Exception ex) { Logging.Log.WriteErrorMessage(LOGTAG, "RollbackError", ex, "Rollback error: {0}", ex.Message); }
}
}
}
Expand Down
Loading
Loading