Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Merge pull request #1533 from NuGet/tilovell/WorkerShouldRecordLastFa…
…ilureApplyingEdit

Tilovell/worker should record last failure applying edit
  • Loading branch information
Tim Lovell-Smith committed Aug 27, 2013
2 parents c2c08e1 + 5aafc7b commit dc1df22
Show file tree
Hide file tree
Showing 7 changed files with 304 additions and 91 deletions.
4 changes: 2 additions & 2 deletions Changelog.md
Expand Up @@ -12,7 +12,7 @@ Other changes include minor fixes in stats page, GetUpdates() API and email vali

### Nuget.org deployed on Azure Websites

The nuget.org website is now deployed on Azure web sites instead of Azure cloud services. Expect a detaiked blog post from the NuGet team on the steps involed in migration and key take aways.
The nuget.org website is now deployed on Azure web sites instead of Azure cloud services. Expect a detailed blog post from the NuGet team on the steps involed in migration and key take aways.
A couple of bug fixes were made to enable this migration( to be compatible with Azure web sites).

- Canonical domain name for nuget.org : nuget.org will now re-direct to www.nuget.org.
Expand All @@ -32,7 +32,7 @@ Other changes include [updated terms of use and privacy policy](https://www.nuge

### Accessiblity bug fixes

A bunch of accessiblity issue like sorting, highlighting and WCAG level A HTMl 5 errors in the website are fixed.
A bunch of accessiblity issue like sorting, highlighting and WCAG level A HTML 5 errors in the website are fixed.

### Other bug fixes

Expand Down
1 change: 1 addition & 0 deletions src/NuGetGallery.Core/Entities/PackageEdit.cs
Expand Up @@ -28,6 +28,7 @@ public class PackageEdit : IEntity
/// Count so that the worker role can tell itself not to retry processing this edit forever if it gets stuck.
/// </summary>
public int TriedCount { get; set; }
public string LastError { get; set; }

//////////////// The rest are same as on Package ////////////

Expand Down
215 changes: 126 additions & 89 deletions src/NuGetGallery.Operations/Tasks/HandleQueuedPackageEditsTask.cs
@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Data;
using System.Data.Entity;
using System.Data.Entity.Infrastructure;
using System.Data.EntityClient;
Expand Down Expand Up @@ -28,67 +29,43 @@ public override void ExecuteCommand()
var connectionString = ConnectionString.ConnectionString;
var storageAccount = StorageAccount;

var entitiesContext = new EntitiesContext(connectionString, readOnly: false);
// We group edits together by their package key and process them together - this is a read-only operation
var entitiesContext = new EntitiesContext(connectionString, readOnly: true);
var editsPerPackage = entitiesContext.Set<PackageEdit>()
.Where(pe => pe.TriedCount < 3)
.Include(pe => pe.Package)
.Include(pe => pe.Package.PackageRegistration)
.Include(pe => pe.Package.User)
.ToList()
.GroupBy(pe => pe.PackageKey);

// Do edit with a 'most recent edit to this package wins - other edits are deleted' strategy.
// Not doing editing in parallel because
// Now that we have our list of packages with pending edits, we'll process the pending edits for each
// Note that we're not doing editing in parallel because
// a) any particular blob may use a large amount of memory to process. Let's not multiply that!
// b) we don't want multithreaded usage of the entitiesContext (and its implied transactions)!
foreach (IGrouping<int, PackageEdit> editsGroup in editsPerPackage)
{
ProcessPackageEdits(editsGroup, entitiesContext);
ProcessPackageEdits(editsGroup.Key);
}
}

/// <summary>
/// Creates an archived copy of the original package blob if it doesn't already exist.
/// </summary>
private void ArchiveOriginalPackageBlob(CloudBlockBlob originalPackageBlob, CloudBlockBlob latestPackageBlob)
private void ProcessPackageEdits(int packageKey)
{
// Copy the blob to backup only if it isn't already successfully copied
if ((!originalPackageBlob.Exists()) || (originalPackageBlob.CopyState != null && originalPackageBlob.CopyState.Status != CopyStatus.Success))
{
if (!WhatIf)
{
Log.Info("Backing up blob: {0} to {1}", latestPackageBlob.Name, originalPackageBlob.Name);
originalPackageBlob.StartCopyFromBlob(latestPackageBlob);
CopyState state = originalPackageBlob.CopyState;

for (int i = 0; (state == null || state.Status == CopyStatus.Pending) && i < SleepTimes.Length; i++)
{
Log.Info("(sleeping for a copy completion)");
Thread.Sleep(SleepTimes[i]);
originalPackageBlob.FetchAttributes(); // To get a refreshed CopyState

//refresh state
state = originalPackageBlob.CopyState;
}
// Create a fresh entities context so that we work in isolation
var entitiesContext = new EntitiesContext(ConnectionString.ConnectionString, readOnly: false);

if (state.Status != CopyStatus.Success)
{
string msg = string.Format("Blob copy failed: CopyState={0}", state.StatusDescription);
Log.Error("(error) " + msg);
throw new BlobBackupFailedException(msg);
}
}
}
}
// Get the list of edits for this package
// Do edit with a 'most recent edit to this package wins - other edits are deleted' strategy.
var editsForThisPackage = entitiesContext.Set<PackageEdit>()
.Where(pe => pe.PackageKey == packageKey && pe.TriedCount < 3)
.Include(pe => pe.Package)
.Include(pe => pe.Package.PackageRegistration)
.Include(pe => pe.User)
.OrderByDescending(pe => pe.Timestamp)
.ToList();

private void ProcessPackageEdits(IEnumerable<PackageEdit> editsForThisPackage, EntitiesContext entitiesContext)
{
// List of Work to do:
// 1) Backup old blob, if the original has not been backed up yet
// 2) Downloads blob, create new NUPKG locally
// 3) Upload blob
// 4) Update the database
PackageEdit edit = editsForThisPackage.OrderByDescending(pe => pe.Timestamp).First();
PackageEdit edit = editsForThisPackage.First();

var blobClient = StorageAccount.CreateCloudBlobClient();
var packagesContainer = Util.GetPackagesBlobContainer(blobClient);
Expand All @@ -115,11 +92,12 @@ private void ProcessPackageEdits(IEnumerable<PackageEdit> editsForThisPackage, E
};

Log.Info(
"Processing Edit Key={0}, PackageId={1}, Version={2}",
"Processing Edit Key={0}, PackageId={1}, Version={2}, User={3}",
edit.Key,
edit.Package.PackageRegistration.Id,
edit.Package.Version);

edit.Package.Version,
edit.User.Username);

if (!WhatIf)
{
edit.TriedCount += 1;
Expand All @@ -131,90 +109,149 @@ private void ProcessPackageEdits(IEnumerable<PackageEdit> editsForThisPackage, E
}
}

ArchiveOriginalPackageBlob(originalPackageBackupBlob, latestPackageBlob);
using (var readWriteStream = new MemoryStream())
try
{
// Download to memory
CloudBlockBlob downloadSourceBlob = WhatIf ? latestPackageBlob : originalPackageBackupBlob;
Log.Info("Downloading original package blob to memory {0}", downloadSourceBlob.Name);
downloadSourceBlob.DownloadToStream(readWriteStream);

// Rewrite in memory
Log.Info("Rewriting nupkg package in memory", downloadSourceBlob.Name);
NupkgRewriter.RewriteNupkgManifest(readWriteStream, edits);

// Get updated hash code, and file size
Log.Info("Computing updated hash code of memory stream");
var newPackageFileSize = readWriteStream.Length;
var hashAlgorithm = HashAlgorithm.Create("SHA512");
byte[] hashBytes = hashAlgorithm.ComputeHash(readWriteStream.GetBuffer());
var newHash = Convert.ToBase64String(hashBytes);

if (!WhatIf)
ArchiveOriginalPackageBlob(originalPackageBackupBlob, latestPackageBlob);
using (var readWriteStream = new MemoryStream())
{
// Snapshot the blob
var blobSnapshot = latestPackageBlob.CreateSnapshot();

// Start Transaction: Complete the edit in the gallery DB.
// Use explicit SQL transactions instead of EF operation-grouping
// so that we can manually roll the transaction back on a blob related failure.
ObjectContext objectContext = (entitiesContext as IObjectContextAdapter).ObjectContext;
((objectContext.Connection) as EntityConnection).Open(); // must open in order to begin transaction
using (EntityTransaction transaction = ((objectContext.Connection) as EntityConnection).BeginTransaction())
// Download to memory
CloudBlockBlob downloadSourceBlob = WhatIf ? latestPackageBlob : originalPackageBackupBlob;
Log.Info("Downloading original package blob to memory {0}", downloadSourceBlob.Name);
downloadSourceBlob.DownloadToStream(readWriteStream);

// Rewrite in memory
Log.Info("Rewriting nupkg package in memory", downloadSourceBlob.Name);
NupkgRewriter.RewriteNupkgManifest(readWriteStream, edits);

// Get updated hash code, and file size
Log.Info("Computing updated hash code of memory stream");
var newPackageFileSize = readWriteStream.Length;
var hashAlgorithm = HashAlgorithm.Create("SHA512");
byte[] hashBytes = hashAlgorithm.ComputeHash(readWriteStream.GetBuffer());
var newHash = Convert.ToBase64String(hashBytes);

if (!WhatIf)
{
edit.Apply(hashAlgorithm: "SHA512", hash: newHash, packageFileSize: newPackageFileSize);
// Snapshot the blob
var blobSnapshot = latestPackageBlob.CreateSnapshot();

// Add to transaction: delete all the pending edits of this package.
// Build up the changes in the entities context
edit.Apply(hashAlgorithm: "SHA512", hash: newHash, packageFileSize: newPackageFileSize);
foreach (var eachEdit in editsForThisPackage)
{
entitiesContext.DeleteOnCommit(eachEdit);
}

entitiesContext.SaveChanges(); // (transaction is still not committed, but do some EF legwork up-front of modifying the blob)
// Upload the blob before doing SaveChanges(). If blob update fails, we won't do SaveChanges() and the edit can be retried.
// If SaveChanges() fails we can undo the blob upload.
try
{
// Reupload blob
Log.Info("Uploading blob from memory {0}", latestPackageBlob.Name);
readWriteStream.Position = 0;
latestPackageBlob.UploadFromStream(readWriteStream);
}
catch (Exception e)
{
// Uploading the updated nupkg failed.
// Rollback the transaction, which restores the Edit to PackageEdits so it can be attempted again.
Log.Error("(error) - package edit blob update failed. Rolling back the DB transaction.");
Log.ErrorException("(exception", e);
Log.Error("(error) - package edit blob update failed.");
Log.ErrorException("(exception)", e);
Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri);
transaction.Rollback();
return;
throw; // To handler block that will record error in DB
}

try
{
transaction.Commit();
// SaveChanges tries to commit changes to DB
entitiesContext.SaveChanges();
}
catch (Exception e)
{
// Commit changes to DB failed.
// Commit changes to DB probably failed.
// Since our blob update wasn't part of the transaction (and doesn't AFAIK have a 'commit()' operator we can utilize for the type of blobs we are using)
// try, (single attempt) to roll back the blob update by restoring the previous snapshot.
Log.Error("(error) - package edit DB update failed. Trying to roll back the blob to its previous snapshot.");
Log.ErrorException("(exception", e);
Log.ErrorException("(exception)", e);
Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri);
try
{
latestPackageBlob.StartCopyFromBlob(blobSnapshot);
}
catch (Exception e2)
{
// In this case it may not be the end of the world - the package metadata mismatches the edit now,
// but there's still an edit in the queue, waiting to be rerun and put everything back in synch.
// If blob rollback fails it is not be the end of the world
// - the package metadata mismatches the edit now,
// but there should still an edit in the queue, waiting to be rerun and put everything back in synch.
Log.Error("(error) - rolling back the package blob to its previous snapshot failed.");
Log.ErrorException("(exception", e2);
Log.ErrorException("(exception)", e2);
Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri);
}

throw; // To handler block that will record error in DB
}
}
}
}
catch (Exception e)
{
if (!WhatIf)
{
try
{
Log.Info("Storing the error on package edit with key {0}", edit.Key);

// Try to record the error into the PackageEdit database record
// so that we can actually diagnose failures.
// This must be done on a fresh context to ensure no conflicts.
var errorContext = new EntitiesContext(ConnectionString.ConnectionString, readOnly: false);
var errorEdit = errorContext.Set<PackageEdit>().Where(pe => pe.Key == edit.Key).FirstOrDefault();

if (errorEdit != null)
{
errorEdit.LastError = string.Format("{0} : {1}", e.GetType(), e);
errorContext.SaveChanges();
}
else
{
Log.Info("The package edit with key {0} couldn't be found. It was likely canceled and deleted.", edit.Key);
}
}
catch (Exception errorException)
{
Log.ErrorException("(error) - couldn't save the last error on the edit that was being applied.", errorException);
}
}
}
}

/// <summary>
/// Creates an archived copy of the original package blob if it doesn't already exist.
/// </summary>
private void ArchiveOriginalPackageBlob(CloudBlockBlob originalPackageBlob, CloudBlockBlob latestPackageBlob)
{
// Copy the blob to backup only if it isn't already successfully copied
if ((!originalPackageBlob.Exists()) || (originalPackageBlob.CopyState != null && originalPackageBlob.CopyState.Status != CopyStatus.Success))
{
if (!WhatIf)
{
Log.Info("Backing up blob: {0} to {1}", latestPackageBlob.Name, originalPackageBlob.Name);
originalPackageBlob.StartCopyFromBlob(latestPackageBlob);
CopyState state = originalPackageBlob.CopyState;

for (int i = 0; (state == null || state.Status == CopyStatus.Pending) && i < SleepTimes.Length; i++)
{
Log.Info("(sleeping for a copy completion)");
Thread.Sleep(SleepTimes[i]);
originalPackageBlob.FetchAttributes(); // To get a refreshed CopyState

//refresh state
state = originalPackageBlob.CopyState;
}

if (state.Status != CopyStatus.Success)
{
string msg = string.Format("Blob copy failed: CopyState={0}", state.StatusDescription);
Log.Error("(error) " + msg);
throw new BlobBackupFailedException(msg);
}
}
}
}
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

@@ -0,0 +1,18 @@
namespace NuGetGallery.Migrations
{
using System;
using System.Data.Entity.Migrations;

public partial class AddPackageEditLastErrorColumn : DbMigration
{
public override void Up()
{
AddColumn("dbo.PackageEdits", "LastError", c => c.String());
}

public override void Down()
{
DropColumn("dbo.PackageEdits", "LastError");
}
}
}

0 comments on commit dc1df22

Please sign in to comment.