Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Optimization for big packages #1123

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Optimization for big packages #1123

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 88 additions & 0 deletions src/Squirrel/DeltaPackage.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,94 @@ public ReleasePackage ApplyDeltaPackage(ReleasePackage basePackage, ReleasePacka
return new ReleasePackage(outputFile);
}

public ReleasePackage ApplyDeltaPackage(ReleasePackage basePackage, IEnumerable<ReleasePackage> deltaPackages, string outputFile)
{
Contract.Requires(deltaPackages != null);
Contract.Requires(!String.IsNullOrEmpty(outputFile) && !File.Exists(outputFile));

string workingPath;
string deltaPath;
var opts = new ExtractionOptions()
{
ExtractFullPath = true,
Overwrite = true,
PreserveFileTime = true
};

using (Utility.WithTempDirectory(out workingPath, localAppDirectory))
{
using (var za = ZipArchive.Open(basePackage.InputPackageFile))
using (var reader = za.ExtractAllEntries())
{
reader.WriteAllToDirectory(workingPath, opts);
}

foreach (var deltaPackage in deltaPackages)
{
using (Utility.WithTempDirectory(out deltaPath, localAppDirectory))
{
using (var za = ZipArchive.Open(deltaPackage.InputPackageFile))
using (var reader = za.ExtractAllEntries())
{
reader.WriteAllToDirectory(deltaPath, opts);
}

var pathsVisited = new List<string>();

var deltaPathRelativePaths = new DirectoryInfo(deltaPath).GetAllFilesRecursively()
.Select(x => x.FullName.Replace(deltaPath + Path.DirectorySeparatorChar, ""))
.ToArray();

// Apply all of the .diff files
deltaPathRelativePaths
.Where(x => x.StartsWith("lib", StringComparison.InvariantCultureIgnoreCase))
.Where(x => !x.EndsWith(".shasum", StringComparison.InvariantCultureIgnoreCase))
.Where(x => !x.EndsWith(".diff", StringComparison.InvariantCultureIgnoreCase) ||
!deltaPathRelativePaths.Contains(x.Replace(".diff", ".bsdiff")))
.ForEach(file =>
{
pathsVisited.Add(Regex.Replace(file, @"\.(bs)?diff$", "").ToLowerInvariant());
applyDiffToFile(deltaPath, file, workingPath);
});

// Delete all of the files that were in the old package but
// not in the new one.
new DirectoryInfo(workingPath).GetAllFilesRecursively()
.Select(x => x.FullName.Replace(workingPath + Path.DirectorySeparatorChar, "")
.ToLowerInvariant())
.Where(x => x.StartsWith("lib", StringComparison.InvariantCultureIgnoreCase) &&
!pathsVisited.Contains(x))
.ForEach(x =>
{
this.Log().Info("{0} was in old package but not in new one, deleting", x);
File.Delete(Path.Combine(workingPath, x));
});

// Update all the files that aren't in 'lib' with the delta
// package's versions (i.e. the nuspec file, etc etc).
deltaPathRelativePaths
.Where(x => !x.StartsWith("lib", StringComparison.InvariantCultureIgnoreCase))
.ForEach(x =>
{
this.Log().Info("Updating metadata file: {0}", x);
File.Copy(Path.Combine(deltaPath, x), Path.Combine(workingPath, x), true);
});
}

}
this.Log().Info("Repacking into full package: {0}", outputFile);
using (var za = ZipArchive.Create())
using (var tgt = File.OpenWrite(outputFile))
{
za.DeflateCompressionLevel = CompressionLevel.BestSpeed;
za.AddAllFromDirectory(workingPath);
za.SaveTo(tgt);
}
}

return new ReleasePackage(outputFile);
}

void createDeltaForSingleFile(FileInfo targetFile, DirectoryInfo workingDirectory, Dictionary<string, string> baseFileListing)
{
// NB: There are three cases here that we'll handle:
Expand Down
26 changes: 11 additions & 15 deletions src/Squirrel/UpdateManager.ApplyReleases.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -308,39 +308,35 @@ async Task<ReleaseEntry> createFullPackagesFromDeltas(IEnumerable<ReleaseEntry>
Contract.Requires(releasesToApply != null);

// If there are no remote releases at all, bail
if (!releasesToApply.Any()) {
if (!releasesToApply.Any())
{
return null;
}

// If there are no deltas in our list, we're already done
if (releasesToApply.All(x => !x.IsDelta)) {
if (releasesToApply.All(x => !x.IsDelta))
{
return releasesToApply.MaxBy(x => x.Version).FirstOrDefault();
}

if (!releasesToApply.All(x => x.IsDelta)) {
if (!releasesToApply.All(x => x.IsDelta))
{
throw new Exception("Cannot apply combinations of delta and full packages");
}

// Smash together our base full package and the nearest delta
var ret = await Task.Run(() => {
var ret = await Task.Run(() =>
{
var basePkg = new ReleasePackage(Path.Combine(rootAppDirectory, "packages", currentVersion.Filename));
var deltaPkg = new ReleasePackage(Path.Combine(rootAppDirectory, "packages", releasesToApply.First().Filename));
var deltaPkg = releasesToApply.Select(x => new ReleasePackage(Path.Combine(rootAppDirectory, "packages", x.Filename)));

var deltaBuilder = new DeltaPackageBuilder(Directory.GetParent(this.rootAppDirectory).FullName);

return deltaBuilder.ApplyDeltaPackage(basePkg, deltaPkg,
Regex.Replace(deltaPkg.InputPackageFile, @"-delta.nupkg$", ".nupkg", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant));
Regex.Replace(deltaPkg.Last().InputPackageFile, @"-delta.nupkg$", ".nupkg", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant));
});

if (releasesToApply.Count() == 1) {
return ReleaseEntry.GenerateFromFile(ret.InputPackageFile);
}

var fi = new FileInfo(ret.InputPackageFile);
var entry = ReleaseEntry.GenerateFromFile(fi.OpenRead(), fi.Name);

// Recursively combine the rest of them
return await createFullPackagesFromDeltas(releasesToApply.Skip(1), entry);
return ReleaseEntry.GenerateFromFile(ret.InputPackageFile);
}

void executeSelfUpdate(SemanticVersion currentVersion)
Expand Down