[Revalidation] Add Admin Panel (#6172)

This change adds two features:

* A revalidation state service - This service allows parallel updates to the state. This is used by the Gallery, and later, the revalidation job for things like the killswitch and the desired revalidation rate.
* An admin UI to stop/start the revalidation job.

Addresses https://github.com/NuGet/Engineering/issues/1437

src/NuGetGallery.Core/CoreConstants.cs

@@ -30,5 +30,6 @@ public static class CoreConstants
         public const string PackagesFolderName = "packages";
         public const string UploadsFolderName = "uploads";
         public const string ValidationFolderName = "validation";
+        public const string RevalidationFolderName = "revalidation";
     }
 }

src/NuGetGallery.Core/Extensions/StorageExceptionExtensions.cs

@@ -0,0 +1,21 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Net;
+using Microsoft.WindowsAzure.Storage;
+
+namespace NuGetGallery
+{
+    public static class StorageExceptionExtensions
+    {
+        public static bool IsFileAlreadyExistsException(this StorageException e)
+        {
+            return e?.RequestInformation?.HttpStatusCode == (int?)HttpStatusCode.Conflict;
+        }
+
+        public static bool IsPreconditionFailedException(this StorageException e)
+        {
+            return e?.RequestInformation?.HttpStatusCode == (int?)HttpStatusCode.PreconditionFailed;
+        }
+    }
+}

src/NuGetGallery.Core/NuGetGallery.Core.csproj

@@ -148,6 +148,7 @@
     <Compile Include="Extensions\EntitiesContextExtensions.cs" />
     <Compile Include="Extensions\PackageRegistrationExtensions.cs" />
     <Compile Include="Extensions\PackageValidationSetExtensions.cs" />
+    <Compile Include="Extensions\StorageExceptionExtensions.cs" />
     <Compile Include="Extensions\UserExtensionsCore.cs" />
     <Compile Include="ICloudStorageStatusDependency.cs" />
     <Compile Include="Infrastructure\AzureEntityList.cs" />
@@ -190,8 +191,11 @@
     <Compile Include="Services\IFileReference.cs" />
     <Compile Include="Services\ICoreFileStorageService.cs" />
     <Compile Include="Services\IFileMetadataService.cs" />
+    <Compile Include="Services\IRevalidationStateService.cs" />
     <Compile Include="Services\ISimpleCloudBlob.cs" />
     <Compile Include="Services\PackageFileServiceMetadata.cs" />
+    <Compile Include="Services\RevalidationState.cs" />
+    <Compile Include="Services\RevalidationStateService.cs" />
     <Compile Include="Services\TestableStorageClientException.cs" />
     <Compile Include="StreamExtensions.cs" />
     <Compile Include="CoreStrings.Designer.cs">

src/NuGetGallery.Core/Services/CloudBlobCoreFileStorageService.cs

@@ -38,7 +38,8 @@ public class CloudBlobCoreFileStorageService : ICoreFileStorageService
             CoreConstants.UploadsFolderName,
             CoreConstants.PackageReadMesFolderName,
             CoreConstants.ValidationFolderName,
-            CoreConstants.UserCertificatesFolderName
+            CoreConstants.UserCertificatesFolderName,
+            CoreConstants.RevalidationFolderName,
         };
 
         protected readonly ICloudBlobClient _client;
@@ -240,7 +241,7 @@ public async Task<IFileReference> GetFileReferenceAsync(string folderName, strin
                     srcAccessCondition,
                     mappedDestAccessCondition);
             }
-            catch (StorageException ex) when (ex.RequestInformation?.HttpStatusCode == (int?)HttpStatusCode.Conflict)
+            catch (StorageException ex) when (ex.IsFileAlreadyExistsException())
             {
                 throw new FileAlreadyExistsException(
                     String.Format(
@@ -293,16 +294,48 @@ private static string Log(AccessCondition accessCondition)
             return "(none)";
         }
 
-        public async Task SaveFileAsync(string folderName, string fileName, Stream packageFile, bool overwrite = true)
+        public async Task SaveFileAsync(string folderName, string fileName, Stream file, bool overwrite = true)
         {
             ICloudBlobContainer container = await GetContainerAsync(folderName);
             var blob = container.GetBlobReference(fileName);
 
             try
             {
-                await blob.UploadFromStreamAsync(packageFile, overwrite);
+                await blob.UploadFromStreamAsync(file, overwrite);
             }
-            catch (StorageException ex) when (ex.RequestInformation?.HttpStatusCode == (int?)HttpStatusCode.Conflict)
+            catch (StorageException ex) when (ex.IsFileAlreadyExistsException())
+            {
+                throw new FileAlreadyExistsException(
+                    String.Format(
+                        CultureInfo.CurrentCulture,
+                        "There is already a blob with name {0} in container {1}.",
+                        fileName,
+                        folderName),
+                    ex);
+            }
+
+            blob.Properties.ContentType = GetContentType(folderName);
+            await blob.SetPropertiesAsync();
+        }
+
+        public async Task SaveFileAsync(string folderName, string fileName, Stream file, IAccessCondition accessConditions)
+        {
+            ICloudBlobContainer container = await GetContainerAsync(folderName);
+            var blob = container.GetBlobReference(fileName);
+
+            accessConditions = accessConditions ?? AccessConditionWrapper.GenerateIfNotExistsCondition();
+
+            var mappedAccessCondition = new AccessCondition
+            {
+                IfNoneMatchETag = accessConditions.IfNoneMatchETag,
+                IfMatchETag = accessConditions.IfMatchETag,
+            };
+
+            try
+            {
+                await blob.UploadFromStreamAsync(file, mappedAccessCondition);
+            }
+            catch (StorageException ex) when (ex.IsFileAlreadyExistsException())
             {
                 throw new FileAlreadyExistsException(
                     String.Format(
@@ -509,6 +542,7 @@ private static string GetContentType(string folderName)
                     return CoreConstants.OctetStreamContentType;
 
                 case CoreConstants.PackageReadMesFolderName:
+                case CoreConstants.RevalidationFolderName:
                     return CoreConstants.TextContentType;
 
                 case CoreConstants.UserCertificatesFolderName:
@@ -535,16 +569,13 @@ private async Task<ICloudBlobContainer> PrepareContainer(string folderName, bool
 
         private struct StorageResult
         {
-            private HttpStatusCode _statusCode;
-            private Stream _data;
-
-            public HttpStatusCode StatusCode { get { return _statusCode; } }
-            public Stream Data { get { return _data; } }
+            public HttpStatusCode StatusCode { get; }
+            public Stream Data { get; }
 
             public StorageResult(HttpStatusCode statusCode, Stream data)
             {
-                _statusCode = statusCode;
-                _data = data;
+                StatusCode = statusCode;
+                Data = data;
             }
         }
     }

src/NuGetGallery.Core/Services/CloudBlobWrapper.cs

@@ -71,22 +71,27 @@ public async Task SetMetadataAsync(AccessCondition accessCondition)
             await _blob.SetMetadataAsync(accessCondition, options: null, operationContext: null);
         }
 
-        public async Task UploadFromStreamAsync(Stream packageFile, bool overwrite)
+        public async Task UploadFromStreamAsync(Stream source, bool overwrite)
         {
             if (overwrite)
             {
-                await _blob.UploadFromStreamAsync(packageFile);
+                await _blob.UploadFromStreamAsync(source);
             }
             else
             {
-                await _blob.UploadFromStreamAsync(
-                    packageFile,
-                    AccessCondition.GenerateIfNoneMatchCondition("*"),
-                    new BlobRequestOptions(),
-                    new OperationContext());
+                await UploadFromStreamAsync(source, AccessCondition.GenerateIfNoneMatchCondition("*"));
             }
         }
 
+        public async Task UploadFromStreamAsync(Stream source, AccessCondition accessCondition)
+        {
+            await _blob.UploadFromStreamAsync(
+                source,
+                accessCondition,
+                options: null,
+                operationContext: null);
+        }
+
         public async Task FetchAttributesAsync()
         {
             await _blob.FetchAttributesAsync();

src/NuGetGallery.Core/Services/CloudFileReference.cs

@@ -1,26 +1,18 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
-using System;
-using System.Collections.Generic;
 using System.IO;
-using System.Linq;
-using System.Text;
 
 namespace NuGetGallery
 {
     public class CloudFileReference : IFileReference
     {
         private Stream _stream;
-        private string _contentId;
 
-        public string ContentId
-        {
-            get { return _contentId; }
-        }
+        public string ContentId { get; }
 
         private CloudFileReference(Stream stream, string contentId)
         {
-            _contentId = contentId;
+            ContentId = contentId;
             _stream = stream;
         }
 

src/NuGetGallery.Core/Services/ICoreFileStorageService.cs

@@ -51,7 +51,17 @@ public interface ICoreFileStorageService
             FileUriPermissions permissions,
             DateTimeOffset endOfAccess);
 
-        Task SaveFileAsync(string folderName, string fileName, Stream packageFile, bool overwrite = true);
+        Task SaveFileAsync(string folderName, string fileName, Stream file, bool overwrite = true);
+
+        /// <summary>
+        /// Saves the file. An exception should be thrown if the access condition is not met.
+        /// </summary>
+        /// <param name="folderName">The folder that contains the file.</param>
+        /// <param name="fileName">The name of file or relative file path.</param>
+        /// <param name="file">The content that should be saved to the file.</param>
+        /// <param name="accessCondition">The condition used to determine whether to persist the save operation.</param>
+        /// <returns>A task that completes once the file is saved.</returns>
+        Task SaveFileAsync(string folderName, string fileName, Stream file, IAccessCondition accessCondition);
 
         /// <summary>
         /// Copies the source URI to the destination file. If the destination already exists and the content

src/NuGetGallery.Core/Services/IFileReference.cs

@@ -1,10 +1,6 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
-using System;
-using System.Collections.Generic;
 using System.IO;
-using System.Linq;
-using System.Text;
 
 namespace NuGetGallery
 {

src/NuGetGallery.Core/Services/IRevalidationStateService.cs

@@ -0,0 +1,31 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Threading.Tasks;
+
+namespace NuGetGallery
+{
+    public interface IRevalidationStateService
+    {
+        /// <summary>
+        /// Get the latest state. Throws if the state blob cannot be found.
+        /// </summary>
+        /// <returns>The latest state.</returns>
+        Task<RevalidationState> GetStateAsync();
+
+        /// <summary>
+        /// Attempt to update the state atomically. Throws if the update fails.
+        /// </summary>
+        /// <param name="updateAction">The action used to update the state.</param>
+        /// <returns>A task that completes once the state has been updated.</returns>
+        Task UpdateStateAsync(Action<RevalidationState> updateAction);
+
+        /// <summary>
+        /// Attempt to update the state atomically. Throws is the update fails.
+        /// </summary>
+        /// <param name="updateAction">The callback that updates the state. Changes are only persisted if the callback returns true</param>
+        /// <returns>The updated state.</returns>
+        Task<RevalidationState> MaybeUpdateStateAsync(Func<RevalidationState, bool> updateAction);
+    }
+}

src/NuGetGallery.Core/Services/ISimpleCloudBlob.cs

@@ -27,7 +27,8 @@ public interface ISimpleCloudBlob
         Task<bool> ExistsAsync();
         Task SetPropertiesAsync();
         Task SetMetadataAsync(AccessCondition accessCondition);
-        Task UploadFromStreamAsync(Stream packageFile, bool overwrite);
+        Task UploadFromStreamAsync(Stream source, bool overwrite);
+        Task UploadFromStreamAsync(Stream source, AccessCondition accessCondition);
 
         Task FetchAttributesAsync();
 

src/NuGetGallery.Core/Services/RevalidationState.cs

@@ -0,0 +1,27 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace NuGetGallery
+{
+    /// <summary>
+    /// The settings for the revalidation job.
+    /// </summary>
+    public class RevalidationState
+    {
+        /// <summary>
+        /// Whether the revalidation job has been deactivated.
+        /// </summary>
+        public bool IsKillswitchActive { get; set; } = false;
+
+        /// <summary>
+        /// Whether the revalidation job's state has been initialized.
+        /// </summary>
+        public bool IsInitialized { get; set; } = false;
+
+        /// <summary>
+        /// The desired number maximal number of package events (pushes, lists, unlists, revalidations)
+        /// per hour.
+        /// </summary>
+        public int DesiredPackageEventRate { get; set; }
+    }
+}