Attempt repair content on failed content installation

Assets/PatchKit Patcher/Scripts/AppData/Local/IUnarchiver.cs

@@ -7,5 +7,11 @@ public interface IUnarchiver
         event UnarchiveProgressChangedHandler UnarchiveProgressChanged;
 
         void Unarchive(CancellationToken cancellationToken);
+
+        // set to true to continue unpacking on error. Check HasErrors later to see if there are any
+        bool ContinueOnError { set; }
+
+        // After Unarchive() if set to true, there were unpacking errors.
+        bool HasErrors { get; }
     }
 }

Assets/PatchKit Patcher/Scripts/AppData/Local/Pack1Unarchiver.cs

@@ -34,6 +34,7 @@ public class Pack1Unarchiver : IUnarchiver
         private readonly string _suffix;
         private readonly byte[] _key;
         private readonly byte[] _iv;
+        private int _processedFiles = 0; // debugging
 
         /// <summary>
         /// The range (in bytes) of the partial pack1 source file
@@ -42,6 +43,12 @@ public class Pack1Unarchiver : IUnarchiver
 
         public event UnarchiveProgressChangedHandler UnarchiveProgressChanged;
 
+        // set to true to continue unpacking on error. Check HasErrors later to see if there are any
+        public bool ContinueOnError { private get; set; }
+
+        // After Unarchive() finishes if this set to true, there were unpacking errors.
+        public bool HasErrors { get; private set; }
+
         public Pack1Unarchiver(string packagePath, Pack1Meta metaData, string destinationDirPath, string key, string suffix = "")
             : this(packagePath, metaData, destinationDirPath, Encoding.ASCII.GetBytes(key), suffix, new BytesRange(0, -1))
         {
@@ -87,6 +94,7 @@ private Pack1Unarchiver(string packagePath, Pack1Meta metaData, string destinati
         public void Unarchive(CancellationToken cancellationToken)
         {
             int entry = 1;
+            HasErrors = false;
 
             DebugLogger.Log("Unpacking " + _metaData.Files.Length + " files...");
             foreach (var file in _metaData.Files)
@@ -149,7 +157,28 @@ private void Unpack(Pack1Meta.FileEntry file, Action<double> progress, Cancellat
             switch (file.Type)
             {
                 case Pack1Meta.RegularFileType:
-                    UnpackRegularFile(file, progress, cancellationToken, destinationDirPath);
+                    try
+                    {
+                        UnpackRegularFile(file, progress, cancellationToken, destinationDirPath);
+                    }
+                    catch (Ionic.Zlib.ZlibException e)
+                    {
+                        if (ContinueOnError)
+                        {
+                            DebugLogger.LogWarning("ZlibException caught. The process will continue, but I will try to repair it later.");
+                            DebugLogger.LogException(e);
+                            HasErrors = true;
+                        }
+                        else
+                        {
+                            throw;
+                        }
+                    }
+                    finally
+                    {
+                        _processedFiles += 1;
+                    }
+
                     break;
                 case Pack1Meta.DirectoryFileType:
                     progress(0.0);
@@ -246,6 +275,7 @@ private void UnpackRegularFile(Pack1Meta.FileEntry file, Action<double> onProgre
                         using (var target = new FileStream(destPath, FileMode.Create))
                         {
                             ExtractFileFromStream(limitedStream, target, file.Size.Value, decryptor, decompressorCreator, onProgress, cancellationToken);
+                            DebugTestCorruption(target);
                         }
                     }
 
@@ -259,6 +289,30 @@ private void UnpackRegularFile(Pack1Meta.FileEntry file, Action<double> onProgre
             DebugLogger.Log("File " + file.Name + " unpacked successfully!");
         }
 
+        // allows to test corruption if valid environment variable is set
+        private void DebugTestCorruption(FileStream target)
+        {
+            if (_processedFiles == 0)
+            {
+                // do not corrupt the first file
+                return;
+            }
+
+            if (
+                _processedFiles % 10 == 0 && EnvironmentInfo.GetEnvironmentVariable(EnvironmentVariables.CorruptFilesOnUnpack10, "") == "1"
+                ||
+                _processedFiles % 50 == 0 && EnvironmentInfo.GetEnvironmentVariable(EnvironmentVariables.CorruptFilesOnUnpack50, "") == "1"
+                ||
+                _processedFiles % 300 == 0 && EnvironmentInfo.GetEnvironmentVariable(EnvironmentVariables.CorruptFilesOnUnpack300, "") == "1"
+                )
+            {
+                DebugLogger.LogWarning("DEBUG: Writing extra byte and triggering zlibexception");
+
+                target.Write(new byte[] { 1 }, 0, 1);
+                throw new Ionic.Zlib.ZlibException();
+            }
+        }
+
         private Stream CreateXzDecompressor(Stream source)
         {
             if (source.CanSeek)

Assets/PatchKit Patcher/Scripts/AppData/Local/ZipUnarchiver.cs

@@ -19,6 +19,12 @@ public class ZipUnarchiver : IUnarchiver
 
         public event UnarchiveProgressChangedHandler UnarchiveProgressChanged;
 
+        // not used
+        public bool ContinueOnError { private get; set; }
+
+        // not used
+        public bool HasErrors { get; private set; }
+
         public ZipUnarchiver(string packagePath, string destinationDirPath, string password = null)
         {
             Checks.ArgumentFileExists(packagePath, "packagePath");

Assets/PatchKit Patcher/Scripts/AppUpdater/AppRepairer.cs

@@ -0,0 +1,177 @@
+using System;
+using System.IO;
+using System.Linq;
+using System.Linq.Expressions;
+using System.Threading;
+using System.Collections.Generic;
+using PatchKit.Unity.Patcher.Cancellation;
+using PatchKit.Unity.Patcher.Debug;
+using PatchKit.Unity.Patcher.AppUpdater.Commands;
+using PatchKit.Unity.Patcher.AppUpdater.Status;
+using PatchKit.Api.Models.Main;
+
+namespace PatchKit.Unity.Patcher.AppUpdater
+{
+    public class AppRepairer
+    {
+        private static readonly DebugLogger DebugLogger = new DebugLogger(typeof(AppRepairer));
+
+        public readonly AppUpdaterContext Context;
+
+        // set to true if you wish to check file hashes
+        public bool CheckHashes = false;
+
+        // how many times process will repeat until it ultimately fails
+        public int RepeatCount = 3;
+
+        private UpdaterStatus _status;
+
+        private AppUpdaterStrategyResolver _strategyResolver;
+
+        private AppUpdaterCommandFactory _commandFactory;
+
+        private int _lowestVersionWithContentId;
+
+
+        public AppRepairer(AppUpdaterContext context, UpdaterStatus status)
+        {
+            DebugLogger.LogConstructor();
+
+            Checks.ArgumentNotNull(context, "context");
+
+            Context = context;
+            _status = status;
+
+            _strategyResolver = new AppUpdaterStrategyResolver(_status);
+            _commandFactory = new AppUpdaterCommandFactory();
+        }
+
+        // returns true if data is valid (was valid from the start or successfull repair was performed)
+        public bool Perform(PatchKit.Unity.Patcher.Cancellation.CancellationToken cancellationToken)
+        {
+            _lowestVersionWithContentId = Context.App.GetLowestVersionWithContentId(cancellationToken);
+
+            for(int attempt = 1; attempt <= RepeatCount; ++attempt)
+            {
+                DebugLogger.Log("Running integrity check, attempt " + attempt + " of " + RepeatCount);
+
+                if (PerformInternal(cancellationToken))
+                {
+                    return true;
+                }
+            }
+
+            // retry count reached, let's check for the last time if data is ok, but without repairing
+            int installedVersionId = Context.App.GetInstalledVersionId();
+            VersionIntegrity results = CheckIntegrity(cancellationToken, installedVersionId);
+            var filesNeedFixing = FilesNeedFixing(results);
+
+            if (filesNeedFixing.Count() == 0)
+            {
+                DebugLogger.Log("No missing or invalid size files.");
+                return true;
+            }
+
+
+            DebugLogger.LogError("Still have corrupted files after all fixing attempts");
+            return false;
+        }
+
+        // returns true if there was no integrity errors, false if there was and repair was performed
+        private bool PerformInternal(PatchKit.Unity.Patcher.Cancellation.CancellationToken cancellationToken)
+        {
+            int installedVersionId = Context.App.GetInstalledVersionId();
+
+            VersionIntegrity results = CheckIntegrity(cancellationToken, installedVersionId);
+            var filesNeedFixing = FilesNeedFixing(results);
+
+            if (filesNeedFixing.Count() == 0)
+            {
+                DebugLogger.Log("No missing or invalid size files.");
+                return true;
+            }
+
+            // need to collect some data about the application to calculate the repair cost and make decisions
+
+            int latestVersionId = Context.App.GetLatestVersionId(true, cancellationToken);
+
+            AppContentSummary installedVersionContentSummary
+                = Context.App.RemoteMetaData.GetContentSummary(installedVersionId, cancellationToken);
+
+            AppContentSummary latestVersionContentSummary
+                = Context.App.RemoteMetaData.GetContentSummary(latestVersionId, cancellationToken);
+
+            bool isNewVersionAvailable = installedVersionId < latestVersionId;
+
+            long contentSize = isNewVersionAvailable
+                ? latestVersionContentSummary.Size
+                : installedVersionContentSummary.Size;
+
+            double repairCost = CalculateRepairCost(installedVersionContentSummary, filesNeedFixing);
+
+
+            if (_lowestVersionWithContentId > installedVersionId)
+            {
+                DebugLogger.Log(
+                    "Repair is impossible because lowest version with content id is "
+                    + _lowestVersionWithContentId +
+                    " and currently installed version id is "
+                    + installedVersionId +
+                    ". Uninstalling to prepare for content strategy.");
+
+                IUninstallCommand uninstall = _commandFactory.CreateUninstallCommand(Context);
+                uninstall.Prepare(_status, cancellationToken);
+                uninstall.Execute(cancellationToken);
+            }
+            else if (repairCost < contentSize)
+            {
+                DebugLogger.Log(string.Format("Repair cost {0} is smaller than content cost {1}, repairing...", repairCost, contentSize));
+                IAppUpdaterStrategy repairStrategy = _strategyResolver.Create(StrategyType.Repair, Context);
+                repairStrategy.Update(cancellationToken);
+            }
+            else
+            {
+                DebugLogger.Log("Content cost is smaller than repair. Uninstalling to prepare for content strategy.");
+                IUninstallCommand uninstall = _commandFactory.CreateUninstallCommand(Context);
+                uninstall.Prepare(_status, cancellationToken);
+                uninstall.Execute(cancellationToken);
+            }
+
+            return false;
+        }
+
+        private VersionIntegrity CheckIntegrity(
+            PatchKit.Unity.Patcher.Cancellation.CancellationToken cancellationToken,
+            int installedVersionId
+            )
+        {
+            ICheckVersionIntegrityCommand checkIntegrity = _commandFactory
+                .CreateCheckVersionIntegrityCommand(
+                    versionId: installedVersionId,
+                    context: Context,
+                    isCheckingHash: CheckHashes,
+                    isCheckingSize: true,
+                    cancellationToken: cancellationToken);
+
+            checkIntegrity.Prepare(_status, cancellationToken);
+            checkIntegrity.Execute(cancellationToken);
+
+            return checkIntegrity.Results;
+        }
+
+        private IEnumerable<FileIntegrity> FilesNeedFixing(VersionIntegrity results)
+        {
+            var missingFiles = results.Files.Where(f => f.Status == FileIntegrityStatus.MissingData);
+            var invalidSizeFiles = results.Files.Where(f => f.Status == FileIntegrityStatus.InvalidSize);
+
+            return missingFiles.Concat(invalidSizeFiles);
+        }
+
+        private long CalculateRepairCost(AppContentSummary contentSummary, IEnumerable<FileIntegrity> filesToRepair)
+        {
+            return filesToRepair
+                .Select(f => contentSummary.Files.FirstOrDefault(e => e.Path == f.FileName))
+                .Sum(f => f.Size);
+        }
+    }
+}