From 606ca272986b9eef0ef1c5d20fbda745408bece8 Mon Sep 17 00:00:00 2001 From: Damon Tivel Date: Wed, 22 Aug 2018 12:04:39 -0700 Subject: [PATCH] Catalog2Dnx: improve throughput (#335) Progress on https://github.com/NuGet/NuGetGallery/issues/6267. --- src/Catalog/Dnx/DnxCatalogCollector.cs | 224 ++++++++++++--- src/Catalog/Dnx/DnxConstants.cs | 20 ++ src/Catalog/Dnx/DnxMaker.cs | 85 +++++- src/Catalog/Helpers/PackageUtility.cs | 4 +- .../NuGet.Services.Metadata.Catalog.csproj | 6 + src/Catalog/PackageCatalogItemCreator.cs | 33 ++- src/Catalog/Persistence/AggregateStorage.cs | 34 ++- .../Persistence/AggregateStorageFactory.cs | 19 +- .../Persistence/AzureCloudBlockBlob.cs | 10 +- src/Catalog/Persistence/AzureStorage.cs | 188 ++++++++++--- .../Persistence/AzureStorageFactory.cs | 39 ++- src/Catalog/Persistence/FileStorage.cs | 40 +-- src/Catalog/Persistence/FileStorageFactory.cs | 12 +- src/Catalog/Persistence/IAzureStorage.cs | 4 +- src/Catalog/Persistence/ICloudBlockBlob.cs | 1 + src/Catalog/Persistence/IStorage.cs | 9 + .../OptimisticConcurrencyControlToken.cs | 65 +++++ src/Catalog/Persistence/Storage.cs | 88 +++--- src/Catalog/Persistence/StorageConstants.cs | 11 + src/Catalog/Persistence/StorageFactory.cs | 3 +- .../FlatContainerPackagePathProvider.cs | 9 +- .../PackagesFolderPackagePathProvider.cs | 5 +- src/Catalog/Registration/RecordingStorage.cs | 17 ++ src/Catalog/Telemetry/TelemetryConstants.cs | 1 + src/Catalog/app.config | 4 + src/Catalog/packages.config | 1 + src/Ng/Arguments.cs | 11 +- src/Ng/CommandHelpers.cs | 35 ++- src/Ng/Jobs/Catalog2DnxJob.cs | 31 ++- src/Ng/Scripts/Catalog2DnxV3.cmd | 5 + src/Ng/Scripts/Catalog2DnxV3China.cmd | 5 + ...ervices.Metadata.Catalog.Monitoring.csproj | 20 +- .../app.config | 4 + .../packages.config | 14 +- src/V3PerPackage/EnqueueCommand.cs | 5 +- src/V3PerPackage/PerBatchProcessor.cs | 8 +- tests/CatalogTests/CatalogTests.csproj | 4 + .../Helpers/PackageUtilityTests.cs | 14 +- .../PackageCatalogItemCreatorTests.cs | 50 +++- .../Persistence/AggregateStorageTests.cs | 47 ++++ .../Persistence/AzureCloudBlockBlobTests.cs | 15 +- .../Persistence/FileStorageTests.cs | 43 +++ .../OptimisticConcurrencyControlTokenTests.cs | 59 ++++ .../Registration/RecordingStorageTests.cs | 43 +++ tests/NgTests/AggregateStorageTests.cs | 17 +- tests/NgTests/Data/Catalogs.cs | 19 ++ .../Data/TestCatalogEntries.Designer.cs | 50 ++++ tests/NgTests/Data/TestCatalogEntries.resx | 91 +++++++ tests/NgTests/DnxCatalogCollectorTests.cs | 256 +++++++++++++++--- tests/NgTests/DnxMakerTests.cs | 240 ++++++++++++++-- tests/NgTests/Infrastructure/MemoryStorage.cs | 24 +- 51 files changed, 1701 insertions(+), 341 deletions(-) create mode 100644 src/Catalog/Dnx/DnxConstants.cs create mode 100644 src/Catalog/Persistence/OptimisticConcurrencyControlToken.cs create mode 100644 src/Catalog/Persistence/StorageConstants.cs create mode 100644 tests/CatalogTests/Persistence/AggregateStorageTests.cs create mode 100644 tests/CatalogTests/Persistence/FileStorageTests.cs create mode 100644 tests/CatalogTests/Persistence/OptimisticConcurrencyControlTokenTests.cs create mode 100644 tests/CatalogTests/Registration/RecordingStorageTests.cs diff --git a/src/Catalog/Dnx/DnxCatalogCollector.cs b/src/Catalog/Dnx/DnxCatalogCollector.cs index 27975c287..7c28e974a 100644 --- a/src/Catalog/Dnx/DnxCatalogCollector.cs +++ b/src/Catalog/Dnx/DnxCatalogCollector.cs @@ -21,13 +21,17 @@ namespace NuGet.Services.Metadata.Catalog.Dnx public class DnxCatalogCollector : CommitCollector { private readonly StorageFactory _storageFactory; + private readonly IAzureStorage _sourceStorage; private readonly DnxMaker _dnxMaker; private readonly ILogger _logger; private readonly int _maxDegreeOfParallelism; + private readonly Uri _contentBaseAddress; public DnxCatalogCollector( Uri index, StorageFactory storageFactory, + IAzureStorage preferredPackageSourceStorage, + Uri contentBaseAddress, ITelemetryService telemetryService, ILogger logger, int maxDegreeOfParallelism, @@ -36,6 +40,8 @@ public class DnxCatalogCollector : CommitCollector : base(index, telemetryService, handlerFunc, httpClientTimeout) { _storageFactory = storageFactory ?? throw new ArgumentNullException(nameof(storageFactory)); + _sourceStorage = preferredPackageSourceStorage; + _contentBaseAddress = contentBaseAddress; _dnxMaker = new DnxMaker(storageFactory); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -49,8 +55,6 @@ public class DnxCatalogCollector : CommitCollector _maxDegreeOfParallelism = maxDegreeOfParallelism; } - public Uri ContentBaseAddress { get; set; } - protected override async Task OnProcessBatch( CollectorHttpClient client, IEnumerable items, @@ -62,7 +66,7 @@ public class DnxCatalogCollector : CommitCollector var catalogEntries = items.Select( item => new CatalogEntry( item["nuget:id"].ToString().ToLowerInvariant(), - NuGetVersionUtility.NormalizeVersion(item["nuget:version"].ToString().ToLowerInvariant()), + NuGetVersionUtility.NormalizeVersion(item["nuget:version"].ToString()).ToLowerInvariant(), item["@type"].ToString().Replace("nuget:", Schema.Prefixes.NuGet), item)) .ToList(); @@ -79,35 +83,56 @@ public class DnxCatalogCollector : CommitCollector return true; } - private async Task> ProcessCatalogEntriesAsync(CollectorHttpClient client, IEnumerable catalogEntries, CancellationToken cancellationToken) + private async Task> ProcessCatalogEntriesAsync( + CollectorHttpClient client, + IEnumerable catalogEntries, + CancellationToken cancellationToken) { var processedCatalogEntries = new ConcurrentBag(); await catalogEntries.ForEachAsync(_maxDegreeOfParallelism, async catalogEntry => { var packageId = catalogEntry.PackageId; - var packageVersion = catalogEntry.PackageVersion; + var normalizedPackageVersion = catalogEntry.NormalizedPackageVersion; if (catalogEntry.EntryType == Schema.DataTypes.PackageDetails.ToString()) { - var properties = GetTelemetryProperties(packageId, packageVersion); + var properties = GetTelemetryProperties(catalogEntry); using (_telemetryService.TrackDuration(TelemetryConstants.ProcessPackageDetailsSeconds, properties)) { - var sourceUri = new Uri(ContentBaseAddress, $"{packageId}.{packageVersion}.nupkg"); + var packageFileName = PackageUtility.GetPackageFileName( + packageId, + normalizedPackageVersion); + var sourceUri = new Uri(_contentBaseAddress, packageFileName); var destinationStorage = _storageFactory.Create(packageId); - var destinationUri = destinationStorage.GetUri(DnxMaker.GetRelativeAddressNupkg(packageId, packageVersion)); + var destinationRelativeUri = DnxMaker.GetRelativeAddressNupkg( + packageId, + normalizedPackageVersion); + var destinationUri = destinationStorage.GetUri(destinationRelativeUri); var isNupkgSynchronized = await destinationStorage.AreSynchronized(sourceUri, destinationUri); - var isPackageInIndex = await _dnxMaker.HasPackageInIndexAsync(destinationStorage, packageId, packageVersion, cancellationToken); - - if (isNupkgSynchronized && isPackageInIndex) + var isPackageInIndex = await _dnxMaker.HasPackageInIndexAsync( + destinationStorage, + packageId, + normalizedPackageVersion, + cancellationToken); + var areRequiredPropertiesPresent = await AreRequiredPropertiesPresentAsync(destinationStorage, destinationUri); + + if (isNupkgSynchronized && isPackageInIndex && areRequiredPropertiesPresent) { - _logger.LogInformation("No changes detected: {Id}/{Version}", packageId, packageVersion); + _logger.LogInformation("No changes detected: {Id}/{Version}", packageId, normalizedPackageVersion); + return; } - if (isNupkgSynchronized || await ProcessPackageDetailsAsync(client, packageId, packageVersion, sourceUri, cancellationToken)) + if ((isNupkgSynchronized && areRequiredPropertiesPresent) + || await ProcessPackageDetailsAsync( + client, + packageId, + normalizedPackageVersion, + sourceUri, + cancellationToken)) { processedCatalogEntries.Add(catalogEntry); } @@ -115,11 +140,11 @@ private async Task> ProcessCatalogEntriesAsync(Collect } else if (catalogEntry.EntryType == Schema.DataTypes.PackageDelete.ToString()) { - var properties = GetTelemetryProperties(packageId, packageVersion); + var properties = GetTelemetryProperties(catalogEntry); using (_telemetryService.TrackDuration(TelemetryConstants.ProcessPackageDeleteSeconds, properties)) { - await ProcessPackageDeleteAsync(packageId, packageVersion, cancellationToken); + await ProcessPackageDeleteAsync(packageId, normalizedPackageVersion, cancellationToken); processedCatalogEntries.Add(catalogEntry); } @@ -129,7 +154,24 @@ private async Task> ProcessCatalogEntriesAsync(Collect return processedCatalogEntries; } - private async Task UpdatePackageVersionIndexAsync(IEnumerable catalogEntries, CancellationToken cancellationToken) + private async Task AreRequiredPropertiesPresentAsync(Storage destinationStorage, Uri destinationUri) + { + var azureStorage = destinationStorage as IAzureStorage; + + if (azureStorage == null) + { + return true; + } + + return await azureStorage.HasPropertiesAsync( + destinationUri, + DnxConstants.ApplicationOctetStreamContentType, + DnxConstants.DefaultCacheControl); + } + + private async Task UpdatePackageVersionIndexAsync( + IEnumerable catalogEntries, + CancellationToken cancellationToken) { var catalogEntryGroups = catalogEntries.GroupBy(catalogEntry => catalogEntry.PackageId); @@ -150,11 +192,11 @@ private async Task UpdatePackageVersionIndexAsync(IEnumerable cata { if (catalogEntry.EntryType == Schema.DataTypes.PackageDetails.ToString()) { - versions.Add(NuGetVersion.Parse(catalogEntry.PackageVersion)); + versions.Add(NuGetVersion.Parse(catalogEntry.NormalizedPackageVersion)); } else if (catalogEntry.EntryType == Schema.DataTypes.PackageDelete.ToString()) { - versions.Remove(NuGetVersion.Parse(catalogEntry.PackageVersion)); + versions.Remove(NuGetVersion.Parse(catalogEntry.NormalizedPackageVersion)); } } }, cancellationToken); @@ -162,12 +204,109 @@ private async Task UpdatePackageVersionIndexAsync(IEnumerable cata foreach (var catalogEntry in catalogEntryGroup) { - _logger.LogInformation("Commit: {Id}/{Version}", packageId, catalogEntry.PackageVersion); + _logger.LogInformation("Commit: {Id}/{Version}", packageId, catalogEntry.NormalizedPackageVersion); } }); } private async Task ProcessPackageDetailsAsync( + HttpClient client, + string packageId, + string normalizedPackageVersion, + Uri sourceUri, + CancellationToken cancellationToken) + { + if (await ProcessPackageDetailsViaStorageAsync( + packageId, + normalizedPackageVersion, + cancellationToken)) + { + return true; + } + + _telemetryService.TrackMetric( + TelemetryConstants.UsePackageSourceFallback, + metric: 1, + properties: GetTelemetryProperties(packageId, normalizedPackageVersion)); + + return await ProcessPackageDetailsViaHttpAsync( + client, + packageId, + normalizedPackageVersion, + sourceUri, + cancellationToken); + } + + private async Task ProcessPackageDetailsViaStorageAsync( + string packageId, + string normalizedPackageVersion, + CancellationToken cancellationToken) + { + if (_sourceStorage == null) + { + return false; + } + + var packageFileName = PackageUtility.GetPackageFileName(packageId, normalizedPackageVersion); + var sourceUri = _sourceStorage.ResolveUri(packageFileName); + + var sourceBlob = await _sourceStorage.GetCloudBlockBlobReferenceAsync(sourceUri); + + if (await sourceBlob.ExistsAsync(cancellationToken)) + { + // It's possible (though unlikely) that the blob may change between reads. Reading a blob with a + // single GET request returns the whole blob in a consistent state, but we're reading the blob many + // different times. To detect the blob changing between reads, we check the ETag again later. + // If the ETag's differ, we'll fall back to using a single HTTP GET request. + var token1 = await _sourceStorage.GetOptimisticConcurrencyControlTokenAsync(sourceUri, cancellationToken); + + var nuspec = await GetNuspecAsync(sourceBlob, packageId, cancellationToken); + + if (string.IsNullOrEmpty(nuspec)) + { + _logger.LogWarning( + "No .nuspec available for {Id}/{Version}. Falling back to HTTP processing.", + packageId, + normalizedPackageVersion); + } + else + { + await _dnxMaker.AddPackageAsync( + _sourceStorage, + nuspec, + packageId, + normalizedPackageVersion, + cancellationToken); + + var token2 = await _sourceStorage.GetOptimisticConcurrencyControlTokenAsync(sourceUri, cancellationToken); + + if (token1 == token2) + { + _logger.LogInformation("Added .nupkg and .nuspec for package {Id}/{Version}", packageId, normalizedPackageVersion); + + return true; + } + else + { + _telemetryService.TrackMetric( + TelemetryConstants.BlobModified, + metric: 1, + properties: GetTelemetryProperties(packageId, normalizedPackageVersion)); + } + } + } + else + { + _telemetryService.TrackMetric( + TelemetryConstants.NonExistentBlob, + metric: 1, + properties: GetTelemetryProperties(packageId, normalizedPackageVersion)); + } + + return false; + } + + private async Task ProcessPackageDetailsViaHttpAsync( HttpClient client, string id, string version, @@ -210,28 +349,33 @@ private async Task UpdatePackageVersionIndexAsync(IEnumerable cata return true; } - private async Task ProcessPackageDeleteAsync(string id, string version, CancellationToken cancellationToken) + private async Task ProcessPackageDeleteAsync( + string packageId, + string normalizedPackageVersion, + CancellationToken cancellationToken) { await _dnxMaker.UpdatePackageVersionIndexAsync( - id, - versions => versions.Remove(NuGetVersion.Parse(version)), + packageId, + versions => versions.Remove(NuGetVersion.Parse(normalizedPackageVersion)), cancellationToken); - await _dnxMaker.DeletePackageAsync(id, version, cancellationToken); + await _dnxMaker.DeletePackageAsync(packageId, normalizedPackageVersion, cancellationToken); - _logger.LogInformation("Commit delete: {Id}/{Version}", id, version); + _logger.LogInformation("Commit delete: {Id}/{Version}", packageId, normalizedPackageVersion); } - private static void AssertNoMultipleEntriesForSamePackageIdentity(DateTime commitTimeStamp, IEnumerable catalogEntries) + private static void AssertNoMultipleEntriesForSamePackageIdentity( + DateTime commitTimeStamp, + IEnumerable catalogEntries) { var catalogEntriesForSamePackageIdentity = catalogEntries.GroupBy( catalogEntry => new { catalogEntry.PackageId, - catalogEntry.PackageVersion + catalogEntry.NormalizedPackageVersion }) .Where(group => group.Count() > 1) - .Select(group => $"{group.Key.PackageId} {group.Key.PackageVersion}"); + .Select(group => $"{group.Key.PackageId} {group.Key.NormalizedPackageVersion}"); if (catalogEntriesForSamePackageIdentity.Any()) { @@ -241,6 +385,17 @@ private static void AssertNoMultipleEntriesForSamePackageIdentity(DateTime commi } } + private static async Task GetNuspecAsync( + ICloudBlockBlob sourceBlob, + string packageId, + CancellationToken cancellationToken) + { + using (var stream = await sourceBlob.GetStreamAsync(cancellationToken)) + { + return GetNuspec(stream, packageId); + } + } + private static string GetNuspec(Stream stream, string id) { string name = $"{id}.nuspec"; @@ -274,26 +429,31 @@ private static string GetNuspec(Stream stream, string id) return null; } - private static Dictionary GetTelemetryProperties(string packageId, string packageVersion) + private static Dictionary GetTelemetryProperties(CatalogEntry catalogEntry) + { + return GetTelemetryProperties(catalogEntry.PackageId, catalogEntry.NormalizedPackageVersion); + } + + private static Dictionary GetTelemetryProperties(string packageId, string normalizedPackageVersion) { return new Dictionary() { { TelemetryConstants.Id, packageId }, - { TelemetryConstants.Version, packageVersion } + { TelemetryConstants.Version, normalizedPackageVersion } }; } private sealed class CatalogEntry { internal string PackageId { get; } - internal string PackageVersion { get; } + internal string NormalizedPackageVersion { get; } internal string EntryType { get; } internal JToken Entry { get; } - internal CatalogEntry(string packageId, string packageVersion, string entryType, JToken entry) + internal CatalogEntry(string packageId, string normalizedPackageVersion, string entryType, JToken entry) { PackageId = packageId; - PackageVersion = packageVersion; + NormalizedPackageVersion = normalizedPackageVersion; EntryType = entryType; Entry = entry; } diff --git a/src/Catalog/Dnx/DnxConstants.cs b/src/Catalog/Dnx/DnxConstants.cs new file mode 100644 index 000000000..385f520a7 --- /dev/null +++ b/src/Catalog/Dnx/DnxConstants.cs @@ -0,0 +1,20 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using NuGet.Services.Metadata.Catalog.Persistence; + +namespace NuGet.Services.Metadata.Catalog.Dnx +{ + internal static class DnxConstants + { + internal const string ApplicationOctetStreamContentType = "application/octet-stream"; + internal const string DefaultCacheControl = "max-age=120"; + + internal static readonly IReadOnlyDictionary RequiredBlobProperties = new Dictionary() + { + { StorageConstants.CacheControl, DefaultCacheControl }, + { StorageConstants.ContentType, ApplicationOctetStreamContentType } + }; + } +} \ No newline at end of file diff --git a/src/Catalog/Dnx/DnxMaker.cs b/src/Catalog/Dnx/DnxMaker.cs index a5b3dc378..478d7a509 100644 --- a/src/Catalog/Dnx/DnxMaker.cs +++ b/src/Catalog/Dnx/DnxMaker.cs @@ -26,8 +26,8 @@ public DnxMaker(StorageFactory storageFactory) public async Task AddPackageAsync( Stream nupkgStream, string nuspec, - string id, - string version, + string packageId, + string normalizedPackageVersion, CancellationToken cancellationToken) { if (nupkgStream == null) @@ -40,23 +40,57 @@ public DnxMaker(StorageFactory storageFactory) throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(nuspec)); } - if (string.IsNullOrEmpty(id)) + if (string.IsNullOrEmpty(packageId)) { - throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(id)); + throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(packageId)); } - if (string.IsNullOrEmpty(version)) + if (string.IsNullOrEmpty(normalizedPackageVersion)) { - throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(version)); + throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(normalizedPackageVersion)); } cancellationToken.ThrowIfCancellationRequested(); - var storage = _storageFactory.Create(id); - var normalizedVersion = NuGetVersionUtility.NormalizeVersion(version); + var storage = _storageFactory.Create(packageId); + var nuspecUri = await SaveNuspecAsync(storage, packageId, normalizedPackageVersion, nuspec, cancellationToken); + var nupkgUri = await SaveNupkgAsync(nupkgStream, storage, packageId, normalizedPackageVersion, cancellationToken); + + return new DnxEntry(nupkgUri, nuspecUri); + } + + public async Task AddPackageAsync( + IStorage sourceStorage, + string nuspec, + string packageId, + string normalizedPackageVersion, + CancellationToken cancellationToken) + { + if (sourceStorage == null) + { + throw new ArgumentNullException(nameof(sourceStorage)); + } + + if (string.IsNullOrEmpty(nuspec)) + { + throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(nuspec)); + } - var nuspecUri = await SaveNuspecAsync(storage, id, normalizedVersion, nuspec, cancellationToken); - var nupkgUri = await SaveNupkgAsync(nupkgStream, storage, id, normalizedVersion, cancellationToken); + if (string.IsNullOrEmpty(packageId)) + { + throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(packageId)); + } + + if (string.IsNullOrEmpty(normalizedPackageVersion)) + { + throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(normalizedPackageVersion)); + } + + cancellationToken.ThrowIfCancellationRequested(); + + var destinationStorage = _storageFactory.Create(packageId); + var nuspecUri = await SaveNuspecAsync(destinationStorage, packageId, normalizedPackageVersion, nuspec, cancellationToken); + var nupkgUri = await CopyNupkgAsync(sourceStorage, destinationStorage, packageId, normalizedPackageVersion, cancellationToken); return new DnxEntry(nupkgUri, nuspecUri); } @@ -111,8 +145,9 @@ private async Task SaveNuspecAsync(Storage storage, string id, string versi { var relativeAddress = GetRelativeAddressNuspec(id, version); var nuspecUri = new Uri(storage.BaseAddress, relativeAddress); + var content = new StringStorageContent(nuspec, "text/xml", DnxConstants.DefaultCacheControl); - await storage.SaveAsync(nuspecUri, new StringStorageContent(nuspec, "text/xml", "max-age=120"), cancellationToken); + await storage.SaveAsync(nuspecUri, content, cancellationToken); return nuspecUri; } @@ -195,10 +230,36 @@ private StorageContent CreateContent(IEnumerable versions) private async Task SaveNupkgAsync(Stream nupkgStream, Storage storage, string id, string version, CancellationToken cancellationToken) { Uri nupkgUri = new Uri(storage.BaseAddress, GetRelativeAddressNupkg(id, version)); - await storage.SaveAsync(nupkgUri, new StreamStorageContent(nupkgStream, "application/octet-stream", "max-age=120"), cancellationToken); + var content = new StreamStorageContent( + nupkgStream, + DnxConstants.ApplicationOctetStreamContentType, + DnxConstants.DefaultCacheControl); + + await storage.SaveAsync(nupkgUri, content, cancellationToken); + return nupkgUri; } + private async Task CopyNupkgAsync( + IStorage sourceStorage, + Storage destinationStorage, + string id, string version, CancellationToken cancellationToken) + { + var packageFileName = PackageUtility.GetPackageFileName(id, version); + var sourceUri = sourceStorage.ResolveUri(packageFileName); + var destinationRelativeUri = GetRelativeAddressNupkg(id, version); + var destinationUri = destinationStorage.ResolveUri(destinationRelativeUri); + + await sourceStorage.CopyAsync( + sourceUri, + destinationStorage, + destinationUri, + DnxConstants.RequiredBlobProperties, + cancellationToken); + + return destinationUri; + } + private async Task DeleteNuspecAsync(Storage storage, string id, string version, CancellationToken cancellationToken) { string relativeAddress = GetRelativeAddressNuspec(id, version); diff --git a/src/Catalog/Helpers/PackageUtility.cs b/src/Catalog/Helpers/PackageUtility.cs index 31dc2b80a..22cf8dd8c 100644 --- a/src/Catalog/Helpers/PackageUtility.cs +++ b/src/Catalog/Helpers/PackageUtility.cs @@ -7,7 +7,7 @@ namespace NuGet.Services.Metadata.Catalog.Helpers { public static class PackageUtility { - public static string GetPackageFileNameLowercase(string packageId, string packageVersion) + public static string GetPackageFileName(string packageId, string packageVersion) { if (string.IsNullOrEmpty(packageId)) { @@ -19,7 +19,7 @@ public static string GetPackageFileNameLowercase(string packageId, string packag throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(packageVersion)); } - return $"{packageId.ToLowerInvariant()}.{packageVersion.ToLowerInvariant()}.nupkg"; + return $"{packageId}.{packageVersion}.nupkg"; } } } \ No newline at end of file diff --git a/src/Catalog/NuGet.Services.Metadata.Catalog.csproj b/src/Catalog/NuGet.Services.Metadata.Catalog.csproj index a0d574452..b8eb0d5b6 100644 --- a/src/Catalog/NuGet.Services.Metadata.Catalog.csproj +++ b/src/Catalog/NuGet.Services.Metadata.Catalog.csproj @@ -93,6 +93,9 @@ ..\..\packages\WindowsAzure.Storage.8.2.1\lib\net45\Microsoft.WindowsAzure.Storage.dll + + ..\..\packages\Microsoft.Azure.Storage.DataMovement.0.6.0\lib\net45\Microsoft.WindowsAzure.Storage.DataMovement.dll + ..\..\packages\Newtonsoft.Json.9.0.1\lib\net45\Newtonsoft.Json.dll True @@ -173,6 +176,7 @@ + @@ -192,6 +196,8 @@ + + diff --git a/src/Catalog/PackageCatalogItemCreator.cs b/src/Catalog/PackageCatalogItemCreator.cs index dada18f2d..7fc5841de 100644 --- a/src/Catalog/PackageCatalogItemCreator.cs +++ b/src/Catalog/PackageCatalogItemCreator.cs @@ -82,25 +82,30 @@ public sealed class PackageCatalogItemCreator : IPackageCatalogItemCreator return item; } - private async Task GetPackageViaStorageAsync(FeedPackageDetails packageItem, CancellationToken cancellationToken) + private async Task GetPackageViaStorageAsync( + FeedPackageDetails packageItem, + CancellationToken cancellationToken) { PackageCatalogItem item = null; - var packageFileName = PackageUtility.GetPackageFileNameLowercase(packageItem.PackageId, packageItem.PackageVersion); - var blob = await _storage.GetCloudBlockBlobReferenceAsync(packageFileName); + var packageId = packageItem.PackageId.ToLowerInvariant(); + var packageVersion = packageItem.PackageVersion.ToLowerInvariant(); + var packageFileName = PackageUtility.GetPackageFileName(packageId, packageVersion); + var blobUri = _storage.ResolveUri(packageFileName); + var blob = await _storage.GetCloudBlockBlobReferenceAsync(blobUri); - if (blob == null) + if (!await blob.ExistsAsync(cancellationToken)) { _telemetryService.TrackMetric( TelemetryConstants.NonExistentBlob, metric: 1, - properties: GetProperties(packageItem, blob)); + properties: GetProperties(packageId, packageVersion, blob)); return item; } using (_telemetryService.TrackDuration( TelemetryConstants.PackageBlobReadSeconds, - GetProperties(packageItem, blob: null))) + GetProperties(packageId, packageVersion, blob: null))) { await blob.FetchAttributesAsync(cancellationToken); @@ -146,7 +151,7 @@ private async Task GetPackageViaStorageAsync(FeedPackageDeta _telemetryService.TrackMetric( TelemetryConstants.BlobModified, metric: 1, - properties: GetProperties(packageItem, blob)); + properties: GetProperties(packageId, packageVersion, blob)); } } } @@ -155,7 +160,7 @@ private async Task GetPackageViaStorageAsync(FeedPackageDeta _telemetryService.TrackMetric( TelemetryConstants.NonExistentPackageHash, metric: 1, - properties: GetProperties(packageItem, blob)); + properties: GetProperties(packageId, packageVersion, blob)); } } @@ -223,11 +228,19 @@ private async Task GetPackageViaHttpAsync(FeedPackageDetails } private static Dictionary GetProperties(FeedPackageDetails packageItem, ICloudBlockBlob blob) + { + return GetProperties( + packageItem.PackageId.ToLowerInvariant(), + packageItem.PackageVersion.ToLowerInvariant(), + blob); + } + + private static Dictionary GetProperties(string packageId, string packageVersion, ICloudBlockBlob blob) { var properties = new Dictionary() { - { TelemetryConstants.Id, packageItem.PackageId.ToLowerInvariant() }, - { TelemetryConstants.Version, packageItem.PackageVersion.ToLowerInvariant() } + { TelemetryConstants.Id, packageId }, + { TelemetryConstants.Version, packageVersion } }; if (blob != null) diff --git a/src/Catalog/Persistence/AggregateStorage.cs b/src/Catalog/Persistence/AggregateStorage.cs index b786db82a..08fc30304 100644 --- a/src/Catalog/Persistence/AggregateStorage.cs +++ b/src/Catalog/Persistence/AggregateStorage.cs @@ -11,16 +11,16 @@ namespace NuGet.Services.Metadata.Catalog.Persistence public class AggregateStorage : Storage { public delegate StorageContent WriteSecondaryStorageContentInterceptor( - Uri primaryStorageBaseUri, - Uri primaryResourceUri, + Uri primaryStorageBaseUri, + Uri primaryResourceUri, Uri secondaryStorageBaseUri, - Uri secondaryResourceUri, + Uri secondaryResourceUri, StorageContent content); private readonly Storage _primaryStorage; private readonly Storage[] _secondaryStorage; private readonly WriteSecondaryStorageContentInterceptor _writeSecondaryStorageContentInterceptor; - + public AggregateStorage(Uri baseAddress, Storage primaryStorage, Storage[] secondaryStorage, WriteSecondaryStorageContentInterceptor writeSecondaryStorageContentInterceptor) : base(baseAddress) @@ -28,11 +28,21 @@ public class AggregateStorage : Storage _primaryStorage = primaryStorage; _secondaryStorage = secondaryStorage; _writeSecondaryStorageContentInterceptor = writeSecondaryStorageContentInterceptor; - + BaseAddress = _primaryStorage.BaseAddress; } - - protected override Task OnSave(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) + + protected override Task OnCopyAsync( + Uri sourceUri, + IStorage destinationStorage, + Uri destinationUri, + IReadOnlyDictionary destinationProperties, + CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + + protected override Task OnSaveAsync(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) { var tasks = new List(); tasks.Add(_primaryStorage.SaveAsync(resourceUri, content, cancellationToken)); @@ -46,9 +56,9 @@ protected override Task OnSave(Uri resourceUri, StorageContent content, Cancella if (_writeSecondaryStorageContentInterceptor != null) { secondaryContent = _writeSecondaryStorageContentInterceptor( - _primaryStorage.BaseAddress, - resourceUri, - storage.BaseAddress, + _primaryStorage.BaseAddress, + resourceUri, + storage.BaseAddress, secondaryResourceUri, content); } @@ -58,12 +68,12 @@ protected override Task OnSave(Uri resourceUri, StorageContent content, Cancella return Task.WhenAll(tasks); } - protected override Task OnLoad(Uri resourceUri, CancellationToken cancellationToken) + protected override Task OnLoadAsync(Uri resourceUri, CancellationToken cancellationToken) { return _primaryStorage.LoadAsync(resourceUri, cancellationToken); } - protected override Task OnDelete(Uri resourceUri, CancellationToken cancellationToken) + protected override Task OnDeleteAsync(Uri resourceUri, CancellationToken cancellationToken) { var tasks = new List(); tasks.Add(_primaryStorage.DeleteAsync(resourceUri, cancellationToken)); diff --git a/src/Catalog/Persistence/AggregateStorageFactory.cs b/src/Catalog/Persistence/AggregateStorageFactory.cs index 0469e29fb..cb82f0a4b 100644 --- a/src/Catalog/Persistence/AggregateStorageFactory.cs +++ b/src/Catalog/Persistence/AggregateStorageFactory.cs @@ -10,13 +10,23 @@ public class AggregateStorageFactory : StorageFactory { private readonly AggregateStorage.WriteSecondaryStorageContentInterceptor _writeSecondaryStorageContentInterceptor; - public AggregateStorageFactory(StorageFactory primaryStorageFactory, StorageFactory[] secondaryStorageFactories) - : this(primaryStorageFactory, secondaryStorageFactories, null) + public AggregateStorageFactory( + StorageFactory primaryStorageFactory, + StorageFactory[] secondaryStorageFactories, + bool verbose) + : this( + primaryStorageFactory, + secondaryStorageFactories, + writeSecondaryStorageContentInterceptor: null, + verbose: verbose) { } - public AggregateStorageFactory(StorageFactory primaryStorageFactory, StorageFactory[] secondaryStorageFactories, - AggregateStorage.WriteSecondaryStorageContentInterceptor writeSecondaryStorageContentInterceptor) + public AggregateStorageFactory( + StorageFactory primaryStorageFactory, + StorageFactory[] secondaryStorageFactories, + AggregateStorage.WriteSecondaryStorageContentInterceptor writeSecondaryStorageContentInterceptor, + bool verbose) { PrimaryStorageFactory = primaryStorageFactory; SecondaryStorageFactories = secondaryStorageFactories; @@ -24,6 +34,7 @@ public AggregateStorageFactory(StorageFactory primaryStorageFactory, StorageFact BaseAddress = PrimaryStorageFactory.BaseAddress; DestinationAddress = PrimaryStorageFactory.DestinationAddress; + Verbose = verbose; } public override Storage Create(string name = null) diff --git a/src/Catalog/Persistence/AzureCloudBlockBlob.cs b/src/Catalog/Persistence/AzureCloudBlockBlob.cs index ce074449a..efe6272b9 100644 --- a/src/Catalog/Persistence/AzureCloudBlockBlob.cs +++ b/src/Catalog/Persistence/AzureCloudBlockBlob.cs @@ -20,12 +20,12 @@ public sealed class AzureCloudBlockBlob : ICloudBlockBlob public AzureCloudBlockBlob(CloudBlockBlob blob) { - if (blob == null) - { - throw new ArgumentNullException(nameof(blob)); - } + _blob = blob ?? throw new ArgumentNullException(nameof(blob)); + } - _blob = blob; + public Task ExistsAsync(CancellationToken cancellationToken) + { + return _blob.ExistsAsync(); } public async Task FetchAttributesAsync(CancellationToken cancellationToken) diff --git a/src/Catalog/Persistence/AzureStorage.cs b/src/Catalog/Persistence/AzureStorage.cs index 228d44b70..ec7373a2e 100644 --- a/src/Catalog/Persistence/AzureStorage.cs +++ b/src/Catalog/Persistence/AzureStorage.cs @@ -11,37 +11,60 @@ using System.Threading.Tasks; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; +using Microsoft.WindowsAzure.Storage.DataMovement; using Microsoft.WindowsAzure.Storage.RetryPolicies; namespace NuGet.Services.Metadata.Catalog.Persistence { public class AzureStorage : Storage, IAzureStorage { + private readonly bool _compressContent; private readonly CloudBlobDirectory _directory; private readonly BlobRequestOptions _blobRequestOptions; + private readonly bool _useServerSideCopy; public static readonly TimeSpan DefaultServerTimeout = TimeSpan.FromSeconds(30); public static readonly TimeSpan DefaultMaxExecutionTime = TimeSpan.FromMinutes(10); - public AzureStorage(CloudStorageAccount account, - string containerName, - string path, - Uri baseAddress) - : this(account, containerName, path, baseAddress, DefaultMaxExecutionTime, DefaultServerTimeout) + public AzureStorage( + CloudStorageAccount account, + string containerName, + string path, + Uri baseAddress, + bool useServerSideCopy, + bool compressContent, + bool verbose) + : this( + account, + containerName, + path, + baseAddress, + DefaultMaxExecutionTime, + DefaultServerTimeout, + useServerSideCopy, + compressContent, + verbose) { } - public AzureStorage(CloudStorageAccount account, - string containerName, - string path, - Uri baseAddress, - TimeSpan maxExecutionTime, - TimeSpan serverTimeout) + public AzureStorage( + CloudStorageAccount account, + string containerName, + string path, + Uri baseAddress, + TimeSpan maxExecutionTime, + TimeSpan serverTimeout, + bool useServerSideCopy, + bool compressContent, + bool verbose) : this(account.CreateCloudBlobClient().GetContainerReference(containerName).GetDirectoryReference(path), baseAddress, maxExecutionTime, serverTimeout) { + _useServerSideCopy = useServerSideCopy; + _compressContent = compressContent; + Verbose = verbose; } private AzureStorage(CloudBlobDirectory directory, Uri baseAddress, TimeSpan maxExecutionTime, TimeSpan serverTimeout) @@ -59,8 +82,6 @@ private AzureStorage(CloudBlobDirectory directory, Uri baseAddress, TimeSpan max } } - ResetStatistics(); - _blobRequestOptions = new BlobRequestOptions() { ServerTimeout = serverTimeout, @@ -69,13 +90,26 @@ private AzureStorage(CloudBlobDirectory directory, Uri baseAddress, TimeSpan max }; } - public bool CompressContent + public override async Task GetOptimisticConcurrencyControlTokenAsync( + Uri resourceUri, + CancellationToken cancellationToken) { - get; - set; + if (resourceUri == null) + { + throw new ArgumentNullException(nameof(resourceUri)); + } + + cancellationToken.ThrowIfCancellationRequested(); + + string blobName = GetName(resourceUri); + CloudBlockBlob blob = _directory.GetBlockBlobReference(blobName); + + await blob.FetchAttributesAsync(cancellationToken); + + return new OptimisticConcurrencyControlToken(blob.Properties.ETag); } - static Uri GetDirectoryUri(CloudBlobDirectory directory) + private static Uri GetDirectoryUri(CloudBlobDirectory directory) { Uri uri = new UriBuilder(directory.Uri) { @@ -119,15 +153,70 @@ private StorageListItem GetStorageListItem(IListBlobItem listBlobItem) return new StorageListItem(listBlobItem.Uri, lastModified); } - // save - protected override async Task OnSave(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) + protected override async Task OnCopyAsync( + Uri sourceUri, + IStorage destinationStorage, + Uri destinationUri, + IReadOnlyDictionary destinationProperties, + CancellationToken cancellationToken) + { + var azureDestinationStorage = destinationStorage as AzureStorage; + + if (azureDestinationStorage == null) + { + throw new NotImplementedException("Copying is only supported from Azure storage to Azure storage."); + } + + string sourceName = GetName(sourceUri); + string destinationName = azureDestinationStorage.GetName(destinationUri); + + CloudBlockBlob sourceBlob = _directory.GetBlockBlobReference(sourceName); + CloudBlockBlob destinationBlob = azureDestinationStorage._directory.GetBlockBlobReference(destinationName); + + var context = new SingleTransferContext(); + + if (destinationProperties?.Count > 0) + { + context.SetAttributesCallback = new SetAttributesCallback((destination) => + { + var blob = (CloudBlockBlob)destination; + + // The copy statement copied all properties from the source blob to the destination blob; however, + // there may be required properties on destination blob, all of which may have not already existed + // on the source blob at the time of copy. + foreach (var property in destinationProperties) + { + switch (property.Key) + { + case StorageConstants.CacheControl: + blob.Properties.CacheControl = property.Value; + break; + + case StorageConstants.ContentType: + blob.Properties.ContentType = property.Value; + break; + + default: + throw new NotImplementedException($"Storage property '{property.Value}' is not supported."); + } + } + }); + } + + context.ShouldOverwriteCallback = new ShouldOverwriteCallback((source, destination) => true); + + await TransferManager.CopyAsync(sourceBlob, destinationBlob, _useServerSideCopy, options: null, context: context); + } + + protected override async Task OnSaveAsync(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) { string name = GetName(resourceUri); CloudBlockBlob blob = _directory.GetBlockBlobReference(name); blob.Properties.ContentType = content.ContentType; blob.Properties.CacheControl = content.CacheControl; - if (CompressContent) + + if (_compressContent) { blob.Properties.ContentEncoding = "gzip"; using (Stream stream = content.GetContentStream()) @@ -140,12 +229,14 @@ protected override async Task OnSave(Uri resourceUri, StorageContent content, Ca } destinationStream.Seek(0, SeekOrigin.Begin); + await blob.UploadFromStreamAsync(destinationStream, - accessCondition: null, - options: _blobRequestOptions, - operationContext: null, - cancellationToken: cancellationToken); - Trace.WriteLine(String.Format("Saved compressed blob {0} to container {1}", blob.Uri.ToString(), _directory.Container.Name)); + accessCondition: null, + options: _blobRequestOptions, + operationContext: null, + cancellationToken: cancellationToken); + + Trace.WriteLine(string.Format("Saved compressed blob {0} to container {1}", blob.Uri.ToString(), _directory.Container.Name)); } } else @@ -153,13 +244,15 @@ protected override async Task OnSave(Uri resourceUri, StorageContent content, Ca using (Stream stream = content.GetContentStream()) { await blob.UploadFromStreamAsync(stream, - accessCondition: null, - options: _blobRequestOptions, - operationContext: null, - cancellationToken: cancellationToken); - Trace.WriteLine(String.Format("Saved uncompressed blob {0} to container {1}", blob.Uri.ToString(), _directory.Container.Name)); + accessCondition: null, + options: _blobRequestOptions, + operationContext: null, + cancellationToken: cancellationToken); } + + Trace.WriteLine(string.Format("Saved uncompressed blob {0} to container {1}", blob.Uri.ToString(), _directory.Container.Name)); } + await TryTakeBlobSnapshotAsync(blob); } @@ -202,8 +295,7 @@ private async Task TryTakeBlobSnapshotAsync(CloudBlockBlob blob) } } - // load - protected override async Task OnLoad(Uri resourceUri, CancellationToken cancellationToken) + protected override async Task OnLoadAsync(Uri resourceUri, CancellationToken cancellationToken) { // the Azure SDK will treat a starting / as an absolute URL, // while we may be working in a subdirectory of a storage container @@ -254,8 +346,7 @@ protected override async Task OnLoad(Uri resourceUri, Cancellati return null; } - // delete - protected override async Task OnDelete(Uri resourceUri, CancellationToken cancellationToken) + protected override async Task OnDeleteAsync(Uri resourceUri, CancellationToken cancellationToken) { string name = GetName(resourceUri); @@ -292,25 +383,34 @@ public override async Task AreSynchronized(Uri firstResourceUri, Uri secon return !(await source.ExistsAsync()); } - public async Task GetCloudBlockBlobReferenceAsync(string name) + public async Task GetCloudBlockBlobReferenceAsync(Uri blobUri) { - Uri uri = ResolveUri(name); - string blobName = GetName(uri); + string blobName = GetName(blobUri); CloudBlockBlob blob = _directory.GetBlockBlobReference(blobName); + var blobExists = await blob.ExistsAsync(); - if (await blob.ExistsAsync()) + if (Verbose && !blobExists) { - return new AzureCloudBlockBlob(blob); + Trace.WriteLine($"The blob {blobUri.AbsoluteUri} does not exist."); } - if (Verbose) + return new AzureCloudBlockBlob(blob); + } + + public async Task HasPropertiesAsync(Uri blobUri, string contentType, string cacheControl) + { + var blobName = GetName(blobUri); + var blob = _directory.GetBlockBlobReference(blobName); + + if (await blob.ExistsAsync()) { - Trace.WriteLine($"The blob {uri} does not exist."); + await blob.FetchAttributesAsync(); + + return string.Equals(blob.Properties.ContentType, contentType) + && string.Equals(blob.Properties.CacheControl, cacheControl); } - // We could return a reference even when the blob does not exist; - // however, there's currently no scenario for this. - return null; + return false; } } } \ No newline at end of file diff --git a/src/Catalog/Persistence/AzureStorageFactory.cs b/src/Catalog/Persistence/AzureStorageFactory.cs index 2518c4888..e1ffa2d47 100644 --- a/src/Catalog/Persistence/AzureStorageFactory.cs +++ b/src/Catalog/Persistence/AzureStorageFactory.cs @@ -14,19 +14,25 @@ public class AzureStorageFactory : StorageFactory private readonly Uri _differentBaseAddress = null; private readonly TimeSpan _maxExecutionTime; private readonly TimeSpan _serverTimeout; - - public AzureStorageFactory(CloudStorageAccount account, - string containerName, - TimeSpan maxExecutionTime, - TimeSpan serverTimeout, - string path = null, - Uri baseAddress = null) + private readonly bool _useServerSideCopy; + + public AzureStorageFactory( + CloudStorageAccount account, + string containerName, + TimeSpan maxExecutionTime, + TimeSpan serverTimeout, + string path, + Uri baseAddress, + bool useServerSideCopy, + bool compressContent, + bool verbose) { _account = account; _containerName = containerName; _path = null; _maxExecutionTime = maxExecutionTime; _serverTimeout = serverTimeout; + _useServerSideCopy = useServerSideCopy; if (path != null) { @@ -62,9 +68,12 @@ public class AzureStorageFactory : StorageFactory blobEndpointBuilder.Port = -1; DestinationAddress = new Uri(blobEndpointBuilder.Uri, containerName + "/" + _path ?? string.Empty); + + CompressContent = compressContent; + Verbose = verbose; } - public bool CompressContent { get; set; } + public bool CompressContent { get; } public override Storage Create(string name = null) { @@ -79,8 +88,16 @@ public override Storage Create(string name = null) newBase = new Uri(_differentBaseAddress, name + "/"); } - return new AzureStorage(_account, _containerName, path, newBase, _maxExecutionTime, _serverTimeout) - { Verbose = Verbose, CompressContent = CompressContent }; + return new AzureStorage( + _account, + _containerName, + path, + newBase, + _maxExecutionTime, + _serverTimeout, + _useServerSideCopy, + CompressContent, + Verbose); } } -} +} \ No newline at end of file diff --git a/src/Catalog/Persistence/FileStorage.cs b/src/Catalog/Persistence/FileStorage.cs index 050fec49b..530c0c892 100644 --- a/src/Catalog/Persistence/FileStorage.cs +++ b/src/Catalog/Persistence/FileStorage.cs @@ -12,10 +12,12 @@ namespace NuGet.Services.Metadata.Catalog.Persistence { public class FileStorage : Storage { - public FileStorage(string baseAddress, string path) - : this(new Uri(baseAddress), path) { } + public FileStorage(string baseAddress, string path, bool verbose) + : this(new Uri(baseAddress), path, verbose) + { + } - public FileStorage(Uri baseAddress, string path) + public FileStorage(Uri baseAddress, string path, bool verbose) : base(baseAddress) { Path = path; @@ -26,7 +28,7 @@ public FileStorage(Uri baseAddress, string path) directoryInfo.Create(); } - ResetStatistics(); + Verbose = verbose; } //File exists @@ -39,24 +41,30 @@ public override Task> ListAsync(CancellationToken c { DirectoryInfo directoryInfo = new DirectoryInfo(Path); var files = directoryInfo.GetFiles("*", SearchOption.AllDirectories) - .Select(file => + .Select(file => new StorageListItem(GetUri(file.FullName.Replace(Path, string.Empty)), file.LastWriteTimeUtc)); return Task.FromResult(files.AsEnumerable()); } public string Path - { + { get; set; } - // save - - protected override async Task OnSave(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) + protected override Task OnCopyAsync( + Uri sourceUri, + IStorage destinationStorage, + Uri destinationUri, + IReadOnlyDictionary destinationProperties, + CancellationToken cancellationToken) { - SaveCount++; + throw new NotImplementedException(); + } + protected override async Task OnSaveAsync(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) + { TraceMethod("SAVE", resourceUri); string name = GetName(resourceUri); @@ -88,13 +96,11 @@ protected override async Task OnSave(Uri resourceUri, StorageContent content, Ca using (FileStream stream = File.Create(path + name)) { - await content.GetContentStream().CopyToAsync(stream,4096, cancellationToken); + await content.GetContentStream().CopyToAsync(stream, 4096, cancellationToken); } } - // load - - protected override async Task OnLoad(Uri resourceUri, CancellationToken cancellationToken) + protected override async Task OnLoadAsync(Uri resourceUri, CancellationToken cancellationToken) { string name = GetName(resourceUri); @@ -125,9 +131,7 @@ protected override async Task OnLoad(Uri resourceUri, Cancellati return null; } - // delete - - protected override async Task OnDelete(Uri resourceUri, CancellationToken cancellationToken) + protected override async Task OnDeleteAsync(Uri resourceUri, CancellationToken cancellationToken) { string name = GetName(resourceUri); @@ -152,7 +156,7 @@ protected override async Task OnDelete(Uri resourceUri, CancellationToken cancel FileInfo fileInfo = new FileInfo(filename); if (fileInfo.Exists) { - await Task.Run(() => { fileInfo.Delete(); },cancellationToken); + await Task.Run(() => { fileInfo.Delete(); }, cancellationToken); } } } diff --git a/src/Catalog/Persistence/FileStorageFactory.cs b/src/Catalog/Persistence/FileStorageFactory.cs index a9cbfbc3e..be8296988 100644 --- a/src/Catalog/Persistence/FileStorageFactory.cs +++ b/src/Catalog/Persistence/FileStorageFactory.cs @@ -7,12 +7,14 @@ namespace NuGet.Services.Metadata.Catalog.Persistence { public class FileStorageFactory : StorageFactory { - string _path; - - public FileStorageFactory(Uri baseAddress, string path) + private readonly string _path; + + public FileStorageFactory(Uri baseAddress, string path, bool verbose) { BaseAddress = new Uri(baseAddress.ToString().TrimEnd('/') + '/'); _path = path.TrimEnd('\\') + '\\'; + + Verbose = verbose; } public override Storage Create(string name = null) @@ -20,7 +22,7 @@ public override Storage Create(string name = null) string fileSystemPath = (name == null) ? _path.Trim('\\') : _path + name; string uriPath = name ?? string.Empty; - return new FileStorage(BaseAddress + uriPath, fileSystemPath) { Verbose = Verbose }; + return new FileStorage(BaseAddress + uriPath, fileSystemPath, Verbose); } } -} +} \ No newline at end of file diff --git a/src/Catalog/Persistence/IAzureStorage.cs b/src/Catalog/Persistence/IAzureStorage.cs index 1ea3e2728..96c4e1733 100644 --- a/src/Catalog/Persistence/IAzureStorage.cs +++ b/src/Catalog/Persistence/IAzureStorage.cs @@ -1,12 +1,14 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; using System.Threading.Tasks; namespace NuGet.Services.Metadata.Catalog.Persistence { public interface IAzureStorage : IStorage { - Task GetCloudBlockBlobReferenceAsync(string name); + Task GetCloudBlockBlobReferenceAsync(Uri uri); + Task HasPropertiesAsync(Uri blobUri, string contentType, string cacheControl); } } \ No newline at end of file diff --git a/src/Catalog/Persistence/ICloudBlockBlob.cs b/src/Catalog/Persistence/ICloudBlockBlob.cs index 2166419f6..aff323fa7 100644 --- a/src/Catalog/Persistence/ICloudBlockBlob.cs +++ b/src/Catalog/Persistence/ICloudBlockBlob.cs @@ -14,6 +14,7 @@ public interface ICloudBlockBlob string ETag { get; } Uri Uri { get; } + Task ExistsAsync(CancellationToken cancellationToken); Task FetchAttributesAsync(CancellationToken cancellationToken); Task> GetMetadataAsync(CancellationToken cancellationToken); Task GetStreamAsync(CancellationToken cancellationToken); diff --git a/src/Catalog/Persistence/IStorage.cs b/src/Catalog/Persistence/IStorage.cs index dcf7a0cff..2c24dbb65 100644 --- a/src/Catalog/Persistence/IStorage.cs +++ b/src/Catalog/Persistence/IStorage.cs @@ -12,7 +12,16 @@ public interface IStorage { Uri BaseAddress { get; } + Task CopyAsync( + Uri sourceUri, + IStorage destinationStorage, + Uri destinationUri, + IReadOnlyDictionary destinationProperties, + CancellationToken cancellation); Task DeleteAsync(Uri resourceUri, CancellationToken cancellationToken); + Task GetOptimisticConcurrencyControlTokenAsync( + Uri resourceUri, + CancellationToken cancellationToken); Task> ListAsync(CancellationToken cancellationToken); Task LoadAsync(Uri resourceUri, CancellationToken cancellationToken); Task LoadStringAsync(Uri resourceUri, CancellationToken cancellationToken); diff --git a/src/Catalog/Persistence/OptimisticConcurrencyControlToken.cs b/src/Catalog/Persistence/OptimisticConcurrencyControlToken.cs new file mode 100644 index 000000000..0855ce6cd --- /dev/null +++ b/src/Catalog/Persistence/OptimisticConcurrencyControlToken.cs @@ -0,0 +1,65 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.Metadata.Catalog.Persistence +{ + public sealed class OptimisticConcurrencyControlToken : IEquatable + { + private readonly string _token; + + public static readonly OptimisticConcurrencyControlToken Null = new OptimisticConcurrencyControlToken(token: null); + + public OptimisticConcurrencyControlToken(string token) + { + _token = token; + } + + public bool Equals(OptimisticConcurrencyControlToken other) + { + var azureToken = other as OptimisticConcurrencyControlToken; + + if (azureToken == null) + { + return false; + } + + return _token == azureToken._token; + } + + public override bool Equals(object obj) + { + return Equals(obj as OptimisticConcurrencyControlToken); + } + + public override int GetHashCode() + { + return _token.GetHashCode(); + } + + public static bool operator ==( + OptimisticConcurrencyControlToken token1, + OptimisticConcurrencyControlToken token2) + { + if (((object)token1) == null || ((object)token2) == null) + { + return object.Equals(token1, token2); + } + + return token1.Equals(token2); + } + + public static bool operator !=( + OptimisticConcurrencyControlToken token1, + OptimisticConcurrencyControlToken token2) + { + if (((object)token1) == null || ((object)token2) == null) + { + return !object.Equals(token1, token2); + } + + return !token1.Equals(token2); + } + } +} \ No newline at end of file diff --git a/src/Catalog/Persistence/Storage.cs b/src/Catalog/Persistence/Storage.cs index 342abe50a..23a85279b 100644 --- a/src/Catalog/Persistence/Storage.cs +++ b/src/Catalog/Persistence/Storage.cs @@ -16,10 +16,7 @@ namespace NuGet.Services.Metadata.Catalog.Persistence public abstract class Storage : IStorage { public Uri BaseAddress { get; protected set; } - public bool Verbose { get; set; } - public int SaveCount { get; protected set; } - public int LoadCount { get; protected set; } - public int DeleteCount { get; protected set; } + public bool Verbose { get; protected set; } public Storage(Uri baseAddress) { @@ -32,21 +29,56 @@ public override string ToString() return BaseAddress.ToString(); } - protected abstract Task OnSave(Uri resourceUri, StorageContent content, CancellationToken cancellationToken); - protected abstract Task OnLoad(Uri resourceUri, CancellationToken cancellationToken); - protected abstract Task OnDelete(Uri resourceUri, CancellationToken cancellationToken); + protected abstract Task OnCopyAsync( + Uri sourceUri, + IStorage destinationStorage, + Uri destinationUri, + IReadOnlyDictionary destinationProperties, + CancellationToken cancellationToken); + protected abstract Task OnSaveAsync(Uri resourceUri, StorageContent content, CancellationToken cancellationToken); + protected abstract Task OnLoadAsync(Uri resourceUri, CancellationToken cancellationToken); + protected abstract Task OnDeleteAsync(Uri resourceUri, CancellationToken cancellationToken); + + public virtual Task GetOptimisticConcurrencyControlTokenAsync( + Uri resourceUri, + CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } - public async Task SaveAsync(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) + public async Task CopyAsync( + Uri sourceUri, + IStorage destinationStorage, + Uri destinationUri, + IReadOnlyDictionary destinationProperties, + CancellationToken cancellationToken) { - SaveCount++; + TraceMethod(nameof(CopyAsync), sourceUri); + + var stopwatch = Stopwatch.StartNew(); + + try + { + await OnCopyAsync(sourceUri, destinationStorage, destinationUri, destinationProperties, cancellationToken); + } + catch (Exception e) + { + TraceException(nameof(CopyAsync), sourceUri, e); + throw; + } + TraceExecutionTime(nameof(CopyAsync), sourceUri, stopwatch.ElapsedMilliseconds); + } + + public async Task SaveAsync(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) + { TraceMethod(nameof(SaveAsync), resourceUri); - Stopwatch sw = new Stopwatch(); - sw.Start(); + + var stopwatch = Stopwatch.StartNew(); try { - await OnSave(resourceUri, content, cancellationToken); + await OnSaveAsync(resourceUri, content, cancellationToken); } catch (Exception e) { @@ -54,22 +86,20 @@ public async Task SaveAsync(Uri resourceUri, StorageContent content, Cancellatio throw; } - sw.Stop(); - TraceExecutionTime(nameof(SaveAsync), resourceUri, sw.ElapsedMilliseconds); + TraceExecutionTime(nameof(SaveAsync), resourceUri, stopwatch.ElapsedMilliseconds); } public async Task LoadAsync(Uri resourceUri, CancellationToken cancellationToken) { - LoadCount++; StorageContent storageContent = null; TraceMethod(nameof(LoadAsync), resourceUri); - Stopwatch sw = new Stopwatch(); - sw.Start(); + + var stopwatch = Stopwatch.StartNew(); try { - storageContent = await OnLoad(resourceUri, cancellationToken); + storageContent = await OnLoadAsync(resourceUri, cancellationToken); } catch (Exception e) { @@ -77,22 +107,20 @@ public async Task LoadAsync(Uri resourceUri, CancellationToken c throw; } - sw.Stop(); - TraceExecutionTime(nameof(LoadAsync), resourceUri, sw.ElapsedMilliseconds); + TraceExecutionTime(nameof(LoadAsync), resourceUri, stopwatch.ElapsedMilliseconds); + return storageContent; } public async Task DeleteAsync(Uri resourceUri, CancellationToken cancellationToken) { - DeleteCount++; - TraceMethod(nameof(DeleteAsync), resourceUri); - Stopwatch sw = new Stopwatch(); - sw.Start(); + + var stopwatch = Stopwatch.StartNew(); try { - await OnDelete(resourceUri, cancellationToken); + await OnDeleteAsync(resourceUri, cancellationToken); } catch (StorageException e) { @@ -116,8 +144,7 @@ public async Task DeleteAsync(Uri resourceUri, CancellationToken cancellationTok throw; } - sw.Stop(); - TraceExecutionTime(nameof(DeleteAsync), resourceUri, sw.ElapsedMilliseconds); + TraceExecutionTime(nameof(DeleteAsync), resourceUri, stopwatch.ElapsedMilliseconds); } public async Task LoadStringAsync(Uri resourceUri, CancellationToken cancellationToken) @@ -141,13 +168,6 @@ public async Task LoadStringAsync(Uri resourceUri, CancellationToken can public abstract bool Exists(string fileName); - public void ResetStatistics() - { - SaveCount = 0; - LoadCount = 0; - DeleteCount = 0; - } - public Uri ResolveUri(string relativeUri) { return new Uri(BaseAddress, relativeUri); diff --git a/src/Catalog/Persistence/StorageConstants.cs b/src/Catalog/Persistence/StorageConstants.cs new file mode 100644 index 000000000..4374b678a --- /dev/null +++ b/src/Catalog/Persistence/StorageConstants.cs @@ -0,0 +1,11 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.Metadata.Catalog.Persistence +{ + public static class StorageConstants + { + public const string CacheControl = "CacheControl"; + public const string ContentType = "ContentType"; + } +} \ No newline at end of file diff --git a/src/Catalog/Persistence/StorageFactory.cs b/src/Catalog/Persistence/StorageFactory.cs index 9aa6e6842..533e437b4 100644 --- a/src/Catalog/Persistence/StorageFactory.cs +++ b/src/Catalog/Persistence/StorageFactory.cs @@ -1,5 +1,6 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + using System; namespace NuGet.Services.Metadata.Catalog.Persistence @@ -13,7 +14,7 @@ public abstract class StorageFactory : IStorageFactory // For telemetry only public Uri DestinationAddress { get; protected set; } - public bool Verbose { get; set; } + public bool Verbose { get; protected set; } public override string ToString() { diff --git a/src/Catalog/Registration/FlatContainerPackagePathProvider.cs b/src/Catalog/Registration/FlatContainerPackagePathProvider.cs index c52e5ad4e..46a58d867 100644 --- a/src/Catalog/Registration/FlatContainerPackagePathProvider.cs +++ b/src/Catalog/Registration/FlatContainerPackagePathProvider.cs @@ -13,13 +13,14 @@ public FlatContainerPackagePathProvider(string container) { _container = container; } + public string GetPackagePath(string id, string version) { - version = NuGetVersionUtility.NormalizeVersion(version); - - var packageFileName = PackageUtility.GetPackageFileNameLowercase(id, version); + var idLowerCase = id.ToLowerInvariant(); + var versionLowerCase = NuGetVersionUtility.NormalizeVersion(version).ToLowerInvariant(); + var packageFileName = PackageUtility.GetPackageFileName(idLowerCase, versionLowerCase); - return $"{_container}/{id.ToLowerInvariant()}/{version.ToLowerInvariant()}/{packageFileName}"; + return $"{_container}/{idLowerCase}/{versionLowerCase}/{packageFileName}"; } } } diff --git a/src/Catalog/Registration/PackagesFolderPackagePathProvider.cs b/src/Catalog/Registration/PackagesFolderPackagePathProvider.cs index 9b794b6db..7425f6ff4 100644 --- a/src/Catalog/Registration/PackagesFolderPackagePathProvider.cs +++ b/src/Catalog/Registration/PackagesFolderPackagePathProvider.cs @@ -9,9 +9,10 @@ public class PackagesFolderPackagePathProvider : IPackagePathProvider { public string GetPackagePath(string id, string version) { - version = NuGetVersionUtility.NormalizeVersion(version); + var idLowerCase = id.ToLowerInvariant(); + var versionLowerCase = NuGetVersionUtility.NormalizeVersion(version).ToLowerInvariant(); - var packageFileName = PackageUtility.GetPackageFileNameLowercase(id, version); + var packageFileName = PackageUtility.GetPackageFileName(idLowerCase, versionLowerCase); return $"packages/{packageFileName}"; } diff --git a/src/Catalog/Registration/RecordingStorage.cs b/src/Catalog/Registration/RecordingStorage.cs index 29b7ec391..a4455b0b4 100644 --- a/src/Catalog/Registration/RecordingStorage.cs +++ b/src/Catalog/Registration/RecordingStorage.cs @@ -24,6 +24,23 @@ public RecordingStorage(IStorage storage) public HashSet Loaded { get; private set; } public HashSet Saved { get; private set; } + public Task GetOptimisticConcurrencyControlTokenAsync( + Uri resourceUri, + CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + + public Task CopyAsync( + Uri sourceUri, + IStorage destinationStorage, + Uri destinationUri, + IReadOnlyDictionary destinationProperties, + CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + public Task SaveAsync(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) { Task result = _innerStorage.SaveAsync(resourceUri, content, cancellationToken); diff --git a/src/Catalog/Telemetry/TelemetryConstants.cs b/src/Catalog/Telemetry/TelemetryConstants.cs index 5f1092e6a..3325b0fa3 100644 --- a/src/Catalog/Telemetry/TelemetryConstants.cs +++ b/src/Catalog/Telemetry/TelemetryConstants.cs @@ -35,6 +35,7 @@ public static class TelemetryConstants public const string StatusCode = "StatusCode"; public const string Success = "Success"; public const string Uri = "Uri"; + public const string UsePackageSourceFallback = "UsePackageSourceFallback"; public const string Version = "Version"; } } \ No newline at end of file diff --git a/src/Catalog/app.config b/src/Catalog/app.config index e310fa8d7..f785a25e1 100644 --- a/src/Catalog/app.config +++ b/src/Catalog/app.config @@ -34,6 +34,10 @@ + + + + diff --git a/src/Catalog/packages.config b/src/Catalog/packages.config index c0e55404e..de74bac52 100644 --- a/src/Catalog/packages.config +++ b/src/Catalog/packages.config @@ -5,6 +5,7 @@ + diff --git a/src/Ng/Arguments.cs b/src/Ng/Arguments.cs index d9b51a7c6..6129049c2 100644 --- a/src/Ng/Arguments.cs +++ b/src/Ng/Arguments.cs @@ -51,6 +51,13 @@ public static class Arguments public const string StorageServerTimeoutInSeconds = "storageServerTimeoutInSeconds"; public const string HttpClientTimeoutInSeconds = "httpClientTimeoutInSeconds"; + public const string StorageAccountNamePreferredPackageSourceStorage = "storageAccountNamePreferredPackageSourceStorage"; + public const string StorageKeyValuePreferredPackageSourceStorage = "storageKeyValuePreferredPackageSourceStorage"; + public const string StorageContainerPreferredPackageSourceStorage = "storageContainerPreferredPackageSourceStorage"; + + public const string PreferAlternatePackageSourceStorage = "preferAlternatePackageSourceStorage"; + + public const string StorageUseServerSideCopy = "storageUseServerSideCopy"; #endregion #region Catalog2Lucene @@ -102,11 +109,7 @@ public static class Arguments public const string StorageKeyValueAuditing = "storageKeyValueAuditing"; public const string StoragePathAuditing = "storagePathAuditing"; public const string StorageTypeAuditing = "storageTypeAuditing"; - public const string StorageAccountNamePreferredPackageSourceStorage = "storageAccountNamePreferredPackageSourceStorage"; - public const string StorageKeyValuePreferredPackageSourceStorage = "storageKeyValuePreferredPackageSourceStorage"; - public const string StorageContainerPreferredPackageSourceStorage = "storageContainerPreferredPackageSourceStorage"; - public const string PreferAlternatePackageSourceStorage = "preferAlternatePackageSourceStorage"; public const string SkipCreatedPackagesProcessing = "skipCreatedPackagesProcessing"; #endregion diff --git a/src/Ng/CommandHelpers.cs b/src/Ng/CommandHelpers.cs index 8f17779f5..07c072c0f 100644 --- a/src/Ng/CommandHelpers.cs +++ b/src/Ng/CommandHelpers.cs @@ -114,10 +114,8 @@ public static RegistrationStorageFactories CreateRegistrationStorageFactories(ID var aggregateStorageFactory = new CatalogAggregateStorageFactory( storageFactory, new[] { compressedStorageFactory }, - secondaryStorageBaseUrlRewriter.Rewrite) - { - Verbose = verbose - }; + secondaryStorageBaseUrlRewriter.Rewrite, + verbose); legacyStorageFactory = aggregateStorageFactory; } @@ -139,6 +137,7 @@ public static CatalogStorageFactory CreateStorageFactory(IDictionary(argumentNameMap[Arguments.StorageSuffix]); var storageOperationMaxExecutionTime = MaxExecutionTime(arguments.GetOrDefault(argumentNameMap[Arguments.StorageOperationMaxExecutionTimeInSeconds])); var storageServerTimeout = MaxExecutionTime(arguments.GetOrDefault(argumentNameMap[Arguments.StorageServerTimeoutInSeconds])); + var storageUseServerSideCopy = arguments.GetOrDefault(argumentNameMap[Arguments.StorageUseServerSideCopy]); var credentials = new StorageCredentials(storageAccountName, storageKeyValue); var account = string.IsNullOrEmpty(storageSuffix) ? - new CloudStorageAccount(credentials, useHttps: true) : - new CloudStorageAccount(credentials, storageSuffix, useHttps: true); - return new CatalogAzureStorageFactory(account, - storageContainer, - storageOperationMaxExecutionTime, - storageServerTimeout, - storagePath, - storageBaseAddress) - { Verbose = verbose, CompressContent = compressed }; + new CloudStorageAccount(credentials, useHttps: true) : + new CloudStorageAccount(credentials, storageSuffix, useHttps: true); + + return new CatalogAzureStorageFactory( + account, + storageContainer, + storageOperationMaxExecutionTime, + storageServerTimeout, + storagePath, + storageBaseAddress, + storageUseServerSideCopy, + compressed, + verbose); } throw new ArgumentException($"Unrecognized storageType \"{storageType}\""); } diff --git a/src/Ng/Jobs/Catalog2DnxJob.cs b/src/Ng/Jobs/Catalog2DnxJob.cs index c27bc445f..316741eb1 100644 --- a/src/Ng/Jobs/Catalog2DnxJob.cs +++ b/src/Ng/Jobs/Catalog2DnxJob.cs @@ -10,6 +10,7 @@ using NuGet.Services.Configuration; using NuGet.Services.Metadata.Catalog; using NuGet.Services.Metadata.Catalog.Dnx; +using NuGet.Services.Metadata.Catalog.Persistence; namespace Ng.Jobs { @@ -45,7 +46,12 @@ public override string GetUsage() + $"[-{Arguments.Verbose} true|false] " + $"[-{Arguments.Interval} ]" + $"[-{Arguments.HttpClientTimeoutInSeconds} ]" - + $"[-{Arguments.StorageSuffix} ]"; + + $"[-{Arguments.StorageSuffix} ]" + + $"[-{Arguments.PreferAlternatePackageSourceStorage} true|false " + + $"-{Arguments.StorageAccountNamePreferredPackageSourceStorage} " + + $"-{Arguments.StorageKeyValuePreferredPackageSourceStorage} " + + $"-{Arguments.StorageContainerPreferredPackageSourceStorage} " + + $"-{Arguments.StorageUseServerSideCopy} true|false]"; } protected override void Init(IDictionary arguments, CancellationToken cancellationToken) @@ -57,20 +63,33 @@ protected override void Init(IDictionary arguments, Cancellation var httpClientTimeoutInSeconds = arguments.GetOrDefault(Arguments.HttpClientTimeoutInSeconds); var httpClientTimeout = httpClientTimeoutInSeconds.HasValue ? (TimeSpan?)TimeSpan.FromSeconds(httpClientTimeoutInSeconds.Value) : null; - Logger.LogInformation("CONFIG source: \"{ConfigSource}\" storage: \"{Storage}\"", source, storageFactory); + StorageFactory preferredPackageSourceStorageFactory = null; + IAzureStorage preferredPackageSourceStorage = null; + + var preferAlternatePackageSourceStorage = arguments.GetOrDefault(Arguments.PreferAlternatePackageSourceStorage, defaultValue: false); + + if (preferAlternatePackageSourceStorage) + { + preferredPackageSourceStorageFactory = CommandHelpers.CreateSuffixedStorageFactory("PreferredPackageSourceStorage", arguments, verbose); + preferredPackageSourceStorage = preferredPackageSourceStorageFactory.Create() as IAzureStorage; + } + + Logger.LogInformation("CONFIG source: \"{ConfigSource}\" storage: \"{Storage}\" preferred package source storage: \"{PreferredPackageSourceStorage}\"", + source, + storageFactory, + preferredPackageSourceStorageFactory); Logger.LogInformation("HTTP client timeout: {Timeout}", httpClientTimeout); _collector = new DnxCatalogCollector( new Uri(source), storageFactory, + preferredPackageSourceStorage, + contentBaseAddress == null ? null : new Uri(contentBaseAddress), TelemetryService, Logger, MaxDegreeOfParallelism, CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose), - httpClientTimeout) - { - ContentBaseAddress = contentBaseAddress == null ? null : new Uri(contentBaseAddress) - }; + httpClientTimeout); var storage = storageFactory.Create(); _front = new DurableCursor(storage.ResolveUri("cursor.json"), storage, MemoryCursor.MinValue); diff --git a/src/Ng/Scripts/Catalog2DnxV3.cmd b/src/Ng/Scripts/Catalog2DnxV3.cmd index 2e44be2d8..e9af1f345 100644 --- a/src/Ng/Scripts/Catalog2DnxV3.cmd +++ b/src/Ng/Scripts/Catalog2DnxV3.cmd @@ -15,6 +15,11 @@ start /w Ng.exe catalog2dnx ^ -storageAccountName #{Jobs.ngcatalog2dnx.StorageAccount.Name} ^ -storageKeyValue #{Jobs.ngcatalog2dnx.StorageAccount.Key} ^ -storageContainer #{Jobs.ngcatalog2dnx.Container} ^ + -preferAlternatePackageSourceStorage #{Jobs.ngcatalog2dnx.PreferAlternatePackageSourceStorage} ^ + -storageAccountNamePreferredPackageSourceStorage #{Jobs.ngcatalog2dnx.PreferredPackageSourceStorageAccountName} ^ + -storageKeyValuePreferredPackageSourceStorage #{Jobs.ngcatalog2dnx.PreferredPackageSourceStorageAccountKey} ^ + -storageContainerPreferredPackageSourceStorage #{Jobs.ngcatalog2dnx.PreferredPackageSourceStorageContainerName} ^ + -storageUseServerSideCopy #{Jobs.ngcatalog2dnx.StorageUseServerSideCopy} ^ -instrumentationkey #{Jobs.common.v3.Logging.InstrumentationKey} ^ -vaultName #{Deployment.Azure.KeyVault.VaultName} ^ -clientId #{Deployment.Azure.KeyVault.ClientId} ^ diff --git a/src/Ng/Scripts/Catalog2DnxV3China.cmd b/src/Ng/Scripts/Catalog2DnxV3China.cmd index cfb2e12a7..4d2984135 100644 --- a/src/Ng/Scripts/Catalog2DnxV3China.cmd +++ b/src/Ng/Scripts/Catalog2DnxV3China.cmd @@ -15,6 +15,11 @@ start /w Ng.exe catalog2dnx ^ -storageAccountName #{Jobs.common.China.v3.StorageAccountName} ^ -storageKeyValue #{Jobs.common.China.v3.StorageKey} ^ -storageContainer #{Jobs.ngcatalog2dnx.Container} ^ + -preferAlternatePackageSourceStorage #{Jobs.ngcatalog2dnx.PreferAlternatePackageSourceStorage} ^ + -storageAccountNamePreferredPackageSourceStorage #{Jobs.ngcatalog2dnx.PreferredPackageSourceStorageAccountName} ^ + -storageKeyValuePreferredPackageSourceStorage #{Jobs.ngcatalog2dnx.PreferredPackageSourceStorageAccountKey} ^ + -storageContainerPreferredPackageSourceStorage #{Jobs.ngcatalog2dnx.PreferredPackageSourceStorageContainerName} ^ + -storageUseServerSideCopy #{Jobs.China.ngcatalog2dnx.StorageUseServerSideCopy} ^ -instrumentationkey #{Jobs.common.v3.Logging.InstrumentationKey} ^ -vaultName #{Deployment.Azure.KeyVault.VaultName} ^ -clientId #{Deployment.Azure.KeyVault.ClientId} ^ diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj b/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj index b9643ef69..5d65dea40 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj @@ -36,14 +36,14 @@ ..\..\packages\Microsoft.Azure.KeyVault.Core.1.0.0\lib\net40\Microsoft.Azure.KeyVault.Core.dll - - ..\..\packages\Microsoft.Data.Edm.5.6.4\lib\net40\Microsoft.Data.Edm.dll + + ..\..\packages\Microsoft.Data.Edm.5.8.2\lib\net40\Microsoft.Data.Edm.dll - - ..\..\packages\Microsoft.Data.OData.5.6.4\lib\net40\Microsoft.Data.OData.dll + + ..\..\packages\Microsoft.Data.OData.5.8.2\lib\net40\Microsoft.Data.OData.dll - - ..\..\packages\Microsoft.Data.Services.Client.5.6.4\lib\net40\Microsoft.Data.Services.Client.dll + + ..\..\packages\Microsoft.Data.Services.Client.5.8.2\lib\net40\Microsoft.Data.Services.Client.dll ..\..\packages\Microsoft.Extensions.DependencyInjection.Abstractions.1.0.0\lib\netstandard1.0\Microsoft.Extensions.DependencyInjection.Abstractions.dll @@ -54,8 +54,8 @@ ..\..\packages\Microsoft.Extensions.Logging.Abstractions.1.0.0\lib\netstandard1.1\Microsoft.Extensions.Logging.Abstractions.dll - - ..\..\packages\WindowsAzure.Storage.7.1.2\lib\net40\Microsoft.WindowsAzure.Storage.dll + + ..\..\packages\WindowsAzure.Storage.8.2.1\lib\net45\Microsoft.WindowsAzure.Storage.dll ..\..\packages\Newtonsoft.Json.9.0.1\lib\net45\Newtonsoft.Json.dll @@ -115,8 +115,8 @@ - - ..\..\packages\System.Spatial.5.6.4\lib\net40\System.Spatial.dll + + ..\..\packages\System.Spatial.5.8.2\lib\net40\System.Spatial.dll diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/app.config b/src/NuGet.Services.Metadata.Catalog.Monitoring/app.config index 8da4767d3..506d0ddbe 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/app.config +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/app.config @@ -46,6 +46,10 @@ + + + + \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/packages.config b/src/NuGet.Services.Metadata.Catalog.Monitoring/packages.config index 7b4c4dea9..a2282424b 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/packages.config +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/packages.config @@ -2,9 +2,9 @@ - - - + + + @@ -28,16 +28,20 @@ + + + + - + - + \ No newline at end of file diff --git a/src/V3PerPackage/EnqueueCommand.cs b/src/V3PerPackage/EnqueueCommand.cs index 89f425d6a..8fce726a1 100644 --- a/src/V3PerPackage/EnqueueCommand.cs +++ b/src/V3PerPackage/EnqueueCommand.cs @@ -23,7 +23,8 @@ public async Task ExecuteAsync(bool restart) { var fileSystemStorage = new FileStorageFactory( new Uri("http://localhost/"), - Directory.GetCurrentDirectory()); + Directory.GetCurrentDirectory(), + verbose: false); var front = new DurableCursor( new Uri("http://localhost/cursor.json"), @@ -42,4 +43,4 @@ public async Task ExecuteAsync(bool restart) await _collector.Run(front, back, CancellationToken.None); } } -} +} \ No newline at end of file diff --git a/src/V3PerPackage/PerBatchProcessor.cs b/src/V3PerPackage/PerBatchProcessor.cs index 9d4927008..726dce597 100644 --- a/src/V3PerPackage/PerBatchProcessor.cs +++ b/src/V3PerPackage/PerBatchProcessor.cs @@ -218,20 +218,20 @@ private async Task ExecuteCatalog2DnxAsync(PerBatchContext context, IReadOnlyLis context.Process.FlatContainerStoragePath); var storageFactory = serviceProvider.GetRequiredService(); + IAzureStorage preferredPackageSourceStorage = null; var httpClientTimeout = TimeSpan.FromMinutes(10); var maxDegreeOfParallelism = ServicePointManager.DefaultConnectionLimit; var collector = new DnxCatalogCollector( catalogIndexUri, storageFactory, + preferredPackageSourceStorage, + context.Global.ContentBaseAddress, serviceProvider.GetRequiredService(), serviceProvider.GetRequiredService(), maxDegreeOfParallelism, () => serviceProvider.GetRequiredService(), - httpClientTimeout) - { - ContentBaseAddress = context.Global.ContentBaseAddress, - }; + httpClientTimeout); var lowercasePackageIds = packageContexts.Select(x => x.PackageId.ToLowerInvariant()); using (await _stringLocker.AcquireAsync(lowercasePackageIds, TimeSpan.FromMinutes(5))) diff --git a/tests/CatalogTests/CatalogTests.csproj b/tests/CatalogTests/CatalogTests.csproj index 799bf08b6..470807c27 100644 --- a/tests/CatalogTests/CatalogTests.csproj +++ b/tests/CatalogTests/CatalogTests.csproj @@ -202,7 +202,11 @@ + + + + diff --git a/tests/CatalogTests/Helpers/PackageUtilityTests.cs b/tests/CatalogTests/Helpers/PackageUtilityTests.cs index c035fd4a0..edf342f44 100644 --- a/tests/CatalogTests/Helpers/PackageUtilityTests.cs +++ b/tests/CatalogTests/Helpers/PackageUtilityTests.cs @@ -12,10 +12,10 @@ public class PackageUtilityTests [Theory] [InlineData(null)] [InlineData("")] - public void GetPackageFileNameLowercase_WhenPackageIdIsNullOrEmpty_Throws(string packageId) + public void GetPackageFileName_WhenPackageIdIsNullOrEmpty_Throws(string packageId) { var exception = Assert.Throws( - () => PackageUtility.GetPackageFileNameLowercase(packageId, packageVersion: "a")); + () => PackageUtility.GetPackageFileName(packageId, packageVersion: "a")); Assert.Equal("packageId", exception.ParamName); } @@ -23,10 +23,10 @@ public void GetPackageFileNameLowercase_WhenPackageIdIsNullOrEmpty_Throws(string [Theory] [InlineData(null)] [InlineData("")] - public void GetPackageFileNameLowercase_WhenPackageVersionIsNullOrEmpty_Throws(string packageVersion) + public void GetPackageFileName_WhenPackageVersionIsNullOrEmpty_Throws(string packageVersion) { var exception = Assert.Throws( - () => PackageUtility.GetPackageFileNameLowercase(packageId: "a", packageVersion: packageVersion)); + () => PackageUtility.GetPackageFileName(packageId: "a", packageVersion: packageVersion)); Assert.Equal("packageVersion", exception.ParamName); } @@ -34,11 +34,11 @@ public void GetPackageFileNameLowercase_WhenPackageVersionIsNullOrEmpty_Throws(s [Theory] [InlineData("a", "b")] [InlineData("A", "B")] - public void GetPackageFileNameLowercase_WithValidArguments_ReturnsFileName(string packageId, string packageVersion) + public void GetPackageFileName_WithValidArguments_ReturnsFileName(string packageId, string packageVersion) { - var packageFileName = PackageUtility.GetPackageFileNameLowercase(packageId, packageVersion); + var packageFileName = PackageUtility.GetPackageFileName(packageId, packageVersion); - Assert.Equal("a.b.nupkg", packageFileName); + Assert.Equal($"{packageId}.{packageVersion}.nupkg", packageFileName); } } } \ No newline at end of file diff --git a/tests/CatalogTests/PackageCatalogItemCreatorTests.cs b/tests/CatalogTests/PackageCatalogItemCreatorTests.cs index c3f8d2300..55471bf0d 100644 --- a/tests/CatalogTests/PackageCatalogItemCreatorTests.cs +++ b/tests/CatalogTests/PackageCatalogItemCreatorTests.cs @@ -145,13 +145,22 @@ public async Task CreateAsync_WhenHttpPackageSourceReturnsPackage_ReturnsInstanc public class WhenStorageIsIAzureStorage { [Fact] - public async Task CreateAsync_WhenAzureStorageReturnsNullBlob_ReturnsInstanceFromHttpPackageSource() + public async Task CreateAsync_WhenAzureStorageReturnsNonexistantBlob_ReturnsInstanceFromHttpPackageSource() { using (var test = new Test()) { + test.Storage.Setup(x => x.ResolveUri(test.PackageFileName)) + .Returns(test.ContentUri); + test.Storage.Setup(x => x.GetCloudBlockBlobReferenceAsync( - It.Is(name => name == test.PackageFileName))) - .ReturnsAsync((ICloudBlockBlob)null); + It.Is(uri => uri == test.ContentUri))) + .ReturnsAsync(test.Blob.Object); + + test.Blob.Setup(x => x.ExistsAsync(It.IsAny())) + .ReturnsAsync(false); + + test.Blob.SetupGet(x => x.Uri) + .Returns(test.ContentUri); var item = await test.Creator.CreateAsync( test.FeedPackageDetails, @@ -167,9 +176,10 @@ public async Task CreateAsync_WhenAzureStorageReturnsNullBlob_ReturnsInstanceFro Assert.Equal("NonExistentBlob", call.Name); Assert.Equal(1UL, call.Metric); - Assert.Equal(2, call.Properties.Count); + Assert.Equal(3, call.Properties.Count); Assert.Equal(test.PackageId.ToLowerInvariant(), call.Properties["Id"]); Assert.Equal(test.PackageVersion.ToLowerInvariant(), call.Properties["Version"]); + Assert.Equal(test.ContentUri.AbsoluteUri, call.Properties["Uri"]); } } @@ -178,10 +188,16 @@ public async Task CreateAsync_WhenBlobDoesNotHavePackageHash_ReturnsInstanceFrom { using (var test = new Test()) { + test.Storage.Setup(x => x.ResolveUri(test.PackageFileName)) + .Returns(test.ContentUri); + test.Storage.Setup(x => x.GetCloudBlockBlobReferenceAsync( - It.Is(name => name == test.PackageFileName))) + It.Is(uri => uri == test.ContentUri))) .ReturnsAsync(test.Blob.Object); + test.Blob.Setup(x => x.ExistsAsync(It.IsAny())) + .ReturnsAsync(true); + test.Blob.Setup(x => x.FetchAttributesAsync(It.IsAny())) .Returns(Task.FromResult(0)); @@ -192,7 +208,7 @@ public async Task CreateAsync_WhenBlobDoesNotHavePackageHash_ReturnsInstanceFrom .ReturnsAsync(new Dictionary()); test.Blob.SetupGet(x => x.Uri) - .Returns(test.PackageUri); + .Returns(test.ContentUri); var item = await test.Creator.CreateAsync( test.FeedPackageDetails, @@ -211,7 +227,7 @@ public async Task CreateAsync_WhenBlobDoesNotHavePackageHash_ReturnsInstanceFrom Assert.Equal(3, call.Properties.Count); Assert.Equal(test.PackageId.ToLowerInvariant(), call.Properties["Id"]); Assert.Equal(test.PackageVersion.ToLowerInvariant(), call.Properties["Version"]); - Assert.Equal(test.PackageUri.AbsoluteUri, call.Properties["Uri"]); + Assert.Equal(test.ContentUri.AbsoluteUri, call.Properties["Uri"]); } } @@ -220,10 +236,16 @@ public async Task CreateAsync_WhenBlobHasPackageHash_ReturnsInstanceFromBlobStor { using (var test = new Test()) { + test.Storage.Setup(x => x.ResolveUri(test.PackageFileName)) + .Returns(test.ContentUri); + test.Storage.Setup(x => x.GetCloudBlockBlobReferenceAsync( - It.Is(name => name == test.PackageFileName))) + It.Is(uri => uri == test.ContentUri))) .ReturnsAsync(test.Blob.Object); + test.Blob.Setup(x => x.ExistsAsync(It.IsAny())) + .ReturnsAsync(true); + test.Blob.Setup(x => x.FetchAttributesAsync(It.IsAny())) .Returns(Task.FromResult(0)); @@ -261,10 +283,16 @@ public async Task CreateAsync_WhenBlobChangesBetweenReads_ReturnsInstanceFromHtt { using (var test = new Test()) { + test.Storage.Setup(x => x.ResolveUri(test.PackageFileName)) + .Returns(test.ContentUri); + test.Storage.Setup(x => x.GetCloudBlockBlobReferenceAsync( - It.Is(name => name == test.PackageFileName))) + It.Is(uri => uri == test.ContentUri))) .ReturnsAsync(test.Blob.Object); + test.Blob.Setup(x => x.ExistsAsync(It.IsAny())) + .ReturnsAsync(true); + test.Blob.Setup(x => x.FetchAttributesAsync(It.IsAny())) .Returns(Task.FromResult(0)); @@ -285,7 +313,7 @@ public async Task CreateAsync_WhenBlobChangesBetweenReads_ReturnsInstanceFromHtt .Returns(Task.FromResult(0)); test.Blob.SetupGet(x => x.Uri) - .Returns(test.PackageUri); + .Returns(test.ContentUri); var item = await test.Creator.CreateAsync( test.FeedPackageDetails, @@ -304,7 +332,7 @@ public async Task CreateAsync_WhenBlobChangesBetweenReads_ReturnsInstanceFromHtt Assert.Equal(3, call.Properties.Count); Assert.Equal(test.PackageId.ToLowerInvariant(), call.Properties["Id"]); Assert.Equal(test.PackageVersion.ToLowerInvariant(), call.Properties["Version"]); - Assert.Equal(test.PackageUri.AbsoluteUri, call.Properties["Uri"]); + Assert.Equal(test.ContentUri.AbsoluteUri, call.Properties["Uri"]); } } } diff --git a/tests/CatalogTests/Persistence/AggregateStorageTests.cs b/tests/CatalogTests/Persistence/AggregateStorageTests.cs new file mode 100644 index 000000000..65beeda5f --- /dev/null +++ b/tests/CatalogTests/Persistence/AggregateStorageTests.cs @@ -0,0 +1,47 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using NgTests.Infrastructure; +using NuGet.Services.Metadata.Catalog.Persistence; +using Xunit; + +namespace CatalogTests.Persistence +{ + public class AggregateStorageTests + { + private readonly AggregateStorage _storage; + + public AggregateStorageTests() + { + _storage = CreateNewAggregateStorage(); + } + + [Fact] + public async Task CopyAsync_Always_Throws() + { + var destinationStorage = CreateNewAggregateStorage(); + var sourceFileUri = _storage.ResolveUri("a"); + var destinationFileUri = destinationStorage.ResolveUri("a"); + + await Assert.ThrowsAsync( + () => _storage.CopyAsync( + sourceFileUri, + destinationStorage, + destinationFileUri, + destinationProperties: null, + cancellationToken: CancellationToken.None)); + } + + private static AggregateStorage CreateNewAggregateStorage() + { + return new AggregateStorage( + new Uri("https://nuget.test"), + new MemoryStorage(), + secondaryStorage: null, + writeSecondaryStorageContentInterceptor: null); + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Persistence/AzureCloudBlockBlobTests.cs b/tests/CatalogTests/Persistence/AzureCloudBlockBlobTests.cs index 10ca0abd2..45680d5a7 100644 --- a/tests/CatalogTests/Persistence/AzureCloudBlockBlobTests.cs +++ b/tests/CatalogTests/Persistence/AzureCloudBlockBlobTests.cs @@ -11,7 +11,7 @@ using NuGet.Services.Metadata.Catalog.Persistence; using Xunit; -namespace CatalogTests +namespace CatalogTests.Persistence { public class AzureCloudBlockBlobTests { @@ -32,6 +32,19 @@ public void Constructor_WhenBlobIsNull_Throws() Assert.Equal("blob", exception.ParamName); } + [Fact] + public async Task ExistsAsync_CallsUnderlyingMethod() + { + _underlyingBlob.Setup(x => x.ExistsAsync()) + .ReturnsAsync(true); + + var blob = new AzureCloudBlockBlob(_underlyingBlob.Object); + + Assert.True(await blob.ExistsAsync(CancellationToken.None)); + + _underlyingBlob.VerifyAll(); + } + [Fact] public async Task FetchAttributesAsync_CallsUnderlyingMethod() { diff --git a/tests/CatalogTests/Persistence/FileStorageTests.cs b/tests/CatalogTests/Persistence/FileStorageTests.cs new file mode 100644 index 000000000..7c92be8f6 --- /dev/null +++ b/tests/CatalogTests/Persistence/FileStorageTests.cs @@ -0,0 +1,43 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using NuGet.Services.Metadata.Catalog.Persistence; +using Xunit; + +namespace CatalogTests.Persistence +{ + public class FileStorageTests + { + private readonly FileStorage _storage; + + public FileStorageTests() + { + _storage = CreateNewFileStorage(); + } + + [Fact] + public async Task CopyAsync_Always_Throws() + { + var destinationStorage = CreateNewFileStorage(); + var sourceFileUri = _storage.ResolveUri("a"); + var destinationFileUri = destinationStorage.ResolveUri("a"); + + await Assert.ThrowsAsync( + () => _storage.CopyAsync( + sourceFileUri, + destinationStorage, + destinationFileUri, + destinationProperties: null, + cancellationToken: CancellationToken.None)); + } + + private static FileStorage CreateNewFileStorage() + { + return new FileStorage(Path.GetTempPath(), Guid.NewGuid().ToString("N"), verbose: false); + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Persistence/OptimisticConcurrencyControlTokenTests.cs b/tests/CatalogTests/Persistence/OptimisticConcurrencyControlTokenTests.cs new file mode 100644 index 000000000..8361e0b30 --- /dev/null +++ b/tests/CatalogTests/Persistence/OptimisticConcurrencyControlTokenTests.cs @@ -0,0 +1,59 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using NuGet.Services.Metadata.Catalog.Persistence; +using Xunit; + +namespace CatalogTests.Persistence +{ + public class OptimisticConcurrencyControlTokenTests + { + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData("a")] + public void WhenTokensAreEqual_EqualityTestsSucceed(string innerToken) + { + var token1 = new OptimisticConcurrencyControlToken(innerToken); + var token2 = new OptimisticConcurrencyControlToken(innerToken); + + Assert.True(token1.Equals(token2)); + Assert.True(token1 == token2); + Assert.False(token1 != token2); + } + + [Theory] + [InlineData(null, "")] + [InlineData("", "a")] + [InlineData("a", "b")] + public void WhenTokensAreNotEqual_EqualityTestsFail(string innerToken1, string innerToken2) + { + var token1 = new OptimisticConcurrencyControlToken(innerToken1); + var token2 = new OptimisticConcurrencyControlToken(innerToken2); + + Assert.False(token1.Equals(token2)); + Assert.False(token1 == token2); + Assert.True(token1 != token2); + } + + [Fact] + public void Null_EqualsNullToken() + { + var token1 = OptimisticConcurrencyControlToken.Null; + var token2 = OptimisticConcurrencyControlToken.Null; + + Assert.True(token1.Equals(token2)); + Assert.True(token1 == token2); + Assert.False(token1 != token2); + } + + [Fact] + public void GetHashCode_EqualsInnerTokenGetHashCode() + { + var innerToken = "abc"; + var token = new OptimisticConcurrencyControlToken(innerToken); + + Assert.Equal(innerToken.GetHashCode(), token.GetHashCode()); + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Registration/RecordingStorageTests.cs b/tests/CatalogTests/Registration/RecordingStorageTests.cs new file mode 100644 index 000000000..fe92eff81 --- /dev/null +++ b/tests/CatalogTests/Registration/RecordingStorageTests.cs @@ -0,0 +1,43 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using NgTests.Infrastructure; +using NuGet.Services.Metadata.Catalog.Registration; +using Xunit; + +namespace CatalogTests.Registration +{ + public class RecordingStorageTests + { + private readonly RecordingStorage _storage; + + public RecordingStorageTests() + { + _storage = CreateNewRecordingStorage(); + } + + [Fact] + public async Task CopyAsync_Always_Throws() + { + var destinationStorage = CreateNewRecordingStorage(); + var sourceFileUri = _storage.ResolveUri("a"); + var destinationFileUri = destinationStorage.ResolveUri("a"); + + await Assert.ThrowsAsync( + () => _storage.CopyAsync( + sourceFileUri, + destinationStorage, + destinationFileUri, + destinationProperties: null, + cancellationToken: CancellationToken.None)); + } + + private static RecordingStorage CreateNewRecordingStorage() + { + return new RecordingStorage(new MemoryStorage()); + } + } +} \ No newline at end of file diff --git a/tests/NgTests/AggregateStorageTests.cs b/tests/NgTests/AggregateStorageTests.cs index c01200650..c9afe4ede 100644 --- a/tests/NgTests/AggregateStorageTests.cs +++ b/tests/NgTests/AggregateStorageTests.cs @@ -159,16 +159,9 @@ public async Task CorrectlyReplacesRegistrationBaseUrlsInSecondaryStorage() protected AggregateStorageFactory Create(MemoryStorage primaryStorage, MemoryStorage secondaryStorage, params MemoryStorage[] storages) { - var storageFactories = new List(); - storageFactories.Add(new TestStorageFactory(name => primaryStorage.WithName(name))); - storageFactories.Add(new TestStorageFactory(name => secondaryStorage.WithName(name))); - - foreach (var storage in storages) - { - storageFactories.Add(new TestStorageFactory(name => storage.WithName(name))); - } + const AggregateStorage.WriteSecondaryStorageContentInterceptor interceptor = null; - return new AggregateStorageFactory(storageFactories.First(), storageFactories.Skip(1).ToArray()); + return CreateWithInterceptor(interceptor, primaryStorage, secondaryStorage, storages); } protected AggregateStorageFactory CreateWithInterceptor(AggregateStorage.WriteSecondaryStorageContentInterceptor interceptor, MemoryStorage primaryStorage, MemoryStorage secondaryStorage, params MemoryStorage[] storages) @@ -182,7 +175,11 @@ protected AggregateStorageFactory CreateWithInterceptor(AggregateStorage.WriteSe storageFactories.Add(new TestStorageFactory(name => storage.WithName(name))); } - return new AggregateStorageFactory(storageFactories.First(), storageFactories.Skip(1).ToArray(), interceptor); + return new AggregateStorageFactory( + storageFactories.First(), + storageFactories.Skip(1).ToArray(), + interceptor, + verbose: false); } protected void AssertUriAndContentExists(MemoryStorage storage, Uri uri, string expectedContent) diff --git a/tests/NgTests/Data/Catalogs.cs b/tests/NgTests/Data/Catalogs.cs index 69bc4fc4e..56041e544 100644 --- a/tests/NgTests/Data/Catalogs.cs +++ b/tests/NgTests/Data/Catalogs.cs @@ -204,5 +204,24 @@ public static MemoryStorage CreateTestCatalogWithMultipleEntriesWithSamePackageI return catalogStorage; } + + public static MemoryStorage CreateTestCatalogWithOnePackage() + { + var catalogStorage = new MemoryStorage(); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "index.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageWithOnePackageIndex)); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "page0.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageWithOnePackagePage)); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "data/2015.10.12.10.08.54/listedpackage.1.0.0.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageWithThreePackagesListedPackage100)); + + return catalogStorage; + } } } \ No newline at end of file diff --git a/tests/NgTests/Data/TestCatalogEntries.Designer.cs b/tests/NgTests/Data/TestCatalogEntries.Designer.cs index 36da47895..2971298ba 100644 --- a/tests/NgTests/Data/TestCatalogEntries.Designer.cs +++ b/tests/NgTests/Data/TestCatalogEntries.Designer.cs @@ -192,6 +192,56 @@ public class TestCatalogEntries { } } + /// + /// Looks up a localized string similar to { + /// "@id": "http://tempuri.org/index.json", + /// "@type": [ + /// "CatalogRoot", + /// "AppendOnlyCatalog", + /// "Permalink" + /// ], + /// "commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + /// "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + /// "count": 1, + /// "items": [ + /// { + /// "@id": "http://tempuri.org/page0.json", + /// "@type": "CatalogPage", + /// "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + /// "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + /// "count": 4 + /// } + /// ], + /// "nuget:las [rest of string was truncated]";. + /// + public static string TestCatalogStorageWithOnePackageIndex { + get { + return ResourceManager.GetString("TestCatalogStorageWithOnePackageIndex", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "http://tempuri.org/page0.json", + /// "@type": "CatalogPage", + /// "commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + /// "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + /// "count": 1, + /// "items": [ + /// { + /// "@id": "http://tempuri.org/data/2015.10.12.10.08.54/listedpackage.1.0.0.json", + /// "@type": "nuget:PackageDetails", + /// "commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + /// "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + /// "nuget:id": "ListedPackage", + /// "nuget:ve [rest of string was truncated]";. + /// + public static string TestCatalogStorageWithOnePackagePage { + get { + return ResourceManager.GetString("TestCatalogStorageWithOnePackagePage", resourceCulture); + } + } + /// /// Looks up a localized string similar to { /// "@id": "http://tempuri.org/index.json", diff --git a/tests/NgTests/Data/TestCatalogEntries.resx b/tests/NgTests/Data/TestCatalogEntries.resx index b68021905..9351db962 100644 --- a/tests/NgTests/Data/TestCatalogEntries.resx +++ b/tests/NgTests/Data/TestCatalogEntries.resx @@ -468,6 +468,97 @@ "@type": "xsd:dateTime" } } +} + + + { + "@id": "http://tempuri.org/index.json", + "@type": [ + "CatalogRoot", + "AppendOnlyCatalog", + "Permalink" + ], + "commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + "count": 1, + "items": [ + { + "@id": "http://tempuri.org/page0.json", + "@type": "CatalogPage", + "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + "count": 4 + } + ], + "nuget:lastCreated": "2015-10-12T10:08:54.1506742Z", + "nuget:lastDeleted": "2015-10-12T10:08:54.1506742Z", + "nuget:lastEdited": "2015-10-12T10:08:54.1506742Z", + "@context": { + "@vocab": "http://schema.nuget.org/catalog#", + "nuget": "http://schema.nuget.org/schema#", + "items": { + "@id": "item", + "@container": "@set" + }, + "parent": { + "@type": "@id" + }, + "commitTimeStamp": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastCreated": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastEdited": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastDeleted": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + } + } +} + + + { + "@id": "http://tempuri.org/page0.json", + "@type": "CatalogPage", + "commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + "count": 1, + "items": [ + { + "@id": "http://tempuri.org/data/2015.10.12.10.08.54/listedpackage.1.0.0.json", + "@type": "nuget:PackageDetails", + "commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + "nuget:id": "ListedPackage", + "nuget:version": "1.0.0" + } + ], + "parent": "http://tempuri.org/index.json", + "@context": { + "@vocab": "http://schema.nuget.org/catalog#", + "nuget": "http://schema.nuget.org/schema#", + "items": { + "@id": "item", + "@container": "@set" + }, + "parent": { + "@type": "@id" + }, + "commitTimeStamp": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastCreated": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastEdited": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastDeleted": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + } + } } diff --git a/tests/NgTests/DnxCatalogCollectorTests.cs b/tests/NgTests/DnxCatalogCollectorTests.cs index 44e73a06d..f676964cb 100644 --- a/tests/NgTests/DnxCatalogCollectorTests.cs +++ b/tests/NgTests/DnxCatalogCollectorTests.cs @@ -28,6 +28,8 @@ public class DnxCatalogCollectorTests private const string _nuspecData = "nuspec data"; private const int _maxDegreeOfParallelism = 20; private static readonly HttpContent _noContent = new ByteArrayContent(new byte[0]); + private const IAzureStorage _nullPreferredPackageSourceStorage = null; + private static readonly Uri _contentBaseAddress = new Uri("http://tempuri.org/packages/"); private MemoryStorage _catalogToDnxStorage; private TestStorageFactory _catalogToDnxStorageFactory; @@ -50,13 +52,12 @@ public DnxCatalogCollectorTests() _target = new DnxCatalogCollector( new Uri("http://tempuri.org/index.json"), _catalogToDnxStorageFactory, - new Mock().Object, + _nullPreferredPackageSourceStorage, + _contentBaseAddress, + Mock.Of(), _logger, _maxDegreeOfParallelism, - () => _mockServer) - { - ContentBaseAddress = new Uri("http://tempuri.org/packages/") - }; + () => _mockServer); _cursorJsonUri = _catalogToDnxStorage.ResolveUri("cursor.json"); } @@ -72,6 +73,8 @@ public void Constructor_WhenIndexIsNull_Throws() () => new DnxCatalogCollector( index, new TestStorageFactory(), + _nullPreferredPackageSourceStorage, + _contentBaseAddress, Mock.Of(), Mock.Of(), maxDegreeOfParallelism: 1, @@ -93,8 +96,10 @@ public void Constructor_WhenStorageFactoryIsNull_Throws() () => new DnxCatalogCollector( new Uri("https://nuget.test"), storageFactory, - Mock.Of(), - Mock.Of(), + _nullPreferredPackageSourceStorage, + contentBaseAddress: null, + telemetryService: Mock.Of(), + logger: Mock.Of(), maxDegreeOfParallelism: 1, handlerFunc: () => clientHandler, httpClientTimeout: TimeSpan.Zero)); @@ -106,16 +111,16 @@ public void Constructor_WhenStorageFactoryIsNull_Throws() [Fact] public void Constructor_WhenTelemetryServiceIsNull_Throws() { - ITelemetryService telemetryService = null; - using (var clientHandler = new HttpClientHandler()) { var exception = Assert.Throws( () => new DnxCatalogCollector( new Uri("https://nuget.test"), new TestStorageFactory(), - telemetryService, - Mock.Of(), + _nullPreferredPackageSourceStorage, + contentBaseAddress: null, + telemetryService: null, + logger: Mock.Of(), maxDegreeOfParallelism: 1, handlerFunc: () => clientHandler, httpClientTimeout: TimeSpan.Zero)); @@ -135,6 +140,8 @@ public void Constructor_WhenLoggerIsNull_Throws() () => new DnxCatalogCollector( new Uri("https://nuget.test"), new TestStorageFactory(), + _nullPreferredPackageSourceStorage, + null, Mock.Of(), logger, maxDegreeOfParallelism: 1, @@ -156,9 +163,11 @@ public void Constructor_WhenMaxDegreeOfParallelismIsLessThanOne_Throws(int maxDe () => new DnxCatalogCollector( new Uri("https://nuget.test"), new TestStorageFactory(), - Mock.Of(), - Mock.Of(), - maxDegreeOfParallelism, + _nullPreferredPackageSourceStorage, + contentBaseAddress: null, + telemetryService: Mock.Of(), + logger: Mock.Of(), + maxDegreeOfParallelism: maxDegreeOfParallelism, handlerFunc: () => clientHandler, httpClientTimeout: TimeSpan.Zero)); @@ -173,8 +182,10 @@ public void Constructor_WhenHandlerFuncIsNull_InstantiatesClass() new DnxCatalogCollector( new Uri("https://nuget.test"), new TestStorageFactory(), - Mock.Of(), - Mock.Of(), + _nullPreferredPackageSourceStorage, + contentBaseAddress: null, + telemetryService: Mock.Of(), + logger: Mock.Of(), maxDegreeOfParallelism: 1, handlerFunc: null, httpClientTimeout: TimeSpan.Zero); @@ -188,8 +199,10 @@ public void Constructor_WhenHttpClientTimeoutIsNull_InstantiatesClass() new DnxCatalogCollector( new Uri("https://nuget.test"), new TestStorageFactory(), - Mock.Of(), - Mock.Of(), + _nullPreferredPackageSourceStorage, + contentBaseAddress: null, + telemetryService: Mock.Of(), + logger: Mock.Of(), maxDegreeOfParallelism: 1, handlerFunc: () => clientHandler, httpClientTimeout: null); @@ -385,20 +398,20 @@ public async Task Run_WithPackageCreatedThenDeleted_LeavesNoArtifacts() } [Fact] - public async Task Run_WhenPackageIsAlreadySynchronized_SkipsPackage() + public async Task Run_WithNonIAzureStorage_WhenPackageIsAlreadySynchronizedAndHasRequiredProperties_SkipsPackage() { - var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json"); - var nupkgUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.1/listedpackage.1.0.1.nupkg"); - var nuspecUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.1/listedpackage.nuspec"); - var nupkgStream = File.OpenRead("Packages\\ListedPackage.1.0.1.zip"); - var expectedNupkg = GetStreamBytes(nupkgStream); - _catalogToDnxStorage = new SynchronizedMemoryStorage(new[] { new Uri("http://tempuri.org/packages/listedpackage.1.0.1.nupkg"), }); _catalogToDnxStorageFactory = new TestStorageFactory(name => _catalogToDnxStorage.WithName(name)); + var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json"); + var nupkgUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.1/listedpackage.1.0.1.nupkg"); + var nuspecUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.1/listedpackage.nuspec"); + var nupkgStream = File.OpenRead("Packages\\ListedPackage.1.0.1.zip"); + var expectedNupkg = GetStreamBytes(nupkgStream); + await _catalogToDnxStorage.SaveAsync( new Uri("http://tempuri.org/listedpackage/index.json"), new StringStorageContent(GetExpectedIndexJsonContent("1.0.1")), @@ -407,13 +420,12 @@ public async Task Run_WhenPackageIsAlreadySynchronized_SkipsPackage() _target = new DnxCatalogCollector( new Uri("http://tempuri.org/index.json"), _catalogToDnxStorageFactory, - new Mock().Object, + _nullPreferredPackageSourceStorage, + _contentBaseAddress, + Mock.Of(), _logger, _maxDegreeOfParallelism, - () => _mockServer) - { - ContentBaseAddress = new Uri("http://tempuri.org/packages/") - }; + () => _mockServer); var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete(); await _mockServer.AddStorageAsync(catalogStorage); @@ -441,12 +453,118 @@ public async Task Run_WhenPackageIsAlreadySynchronized_SkipsPackage() } [Fact] - public async Task Run_WhenPackageIsAlreadySynchronizedButNotInIndex_ProcessesPackage() + public async Task Run_WithIAzureStorage_WhenPackageIsAlreadySynchronizedAndHasRequiredProperties_SkipsPackage() { + _catalogToDnxStorage = new AzureSynchronizedMemoryStorageStub(new[] + { + new Uri("http://tempuri.org/packages/listedpackage.1.0.0.nupkg") + }, areRequiredPropertiesPresentAsync: true); + _catalogToDnxStorageFactory = new TestStorageFactory(name => _catalogToDnxStorage.WithName(name)); + var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json"); - var nupkgUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.1/listedpackage.1.0.1.nupkg"); - var nuspecUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.1/listedpackage.nuspec"); + var nupkgUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.0/listedpackage.1.0.0.nupkg"); + var nuspecUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.0/listedpackage.nuspec"); + var nupkgStream = File.OpenRead("Packages\\ListedPackage.1.0.0.zip"); + var expectedNupkg = GetStreamBytes(nupkgStream); + + await _catalogToDnxStorage.SaveAsync( + new Uri("http://tempuri.org/listedpackage/index.json"), + new StringStorageContent(GetExpectedIndexJsonContent("1.0.0")), + CancellationToken.None); + + _target = new DnxCatalogCollector( + new Uri("http://tempuri.org/index.json"), + _catalogToDnxStorageFactory, + _nullPreferredPackageSourceStorage, + _contentBaseAddress, + Mock.Of(), + _logger, + _maxDegreeOfParallelism, + () => _mockServer); + + var catalogStorage = Catalogs.CreateTestCatalogWithOnePackage(); + await _mockServer.AddStorageAsync(catalogStorage); + + _mockServer.SetAction( + "/packages/listedpackage.1.0.0.nupkg", + request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(nupkgStream) })); + + var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); + ReadCursor back = MemoryCursor.CreateMax(); + + await _target.Run(front, back, CancellationToken.None); + + Assert.Equal(2, _catalogToDnxStorage.Content.Count); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri)); + Assert.False(_catalogToDnxStorage.Content.ContainsKey(nupkgUri)); + Assert.False(_catalogToDnxStorage.Content.ContainsKey(nuspecUri)); + Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(_cursorJsonUri)); + Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(indexJsonUri, out var indexJson)); + Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nupkgUri)); + Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nuspecUri)); + + Assert.Equal(GetExpectedIndexJsonContent("1.0.0"), Encoding.UTF8.GetString(indexJson)); + } + + [Fact] + public async Task Run_WithFakeIAzureStorage_WhenPackageIsAlreadySynchronizedButDoesNotHaveRequiredProperties_ProcessesPackage() + { + _catalogToDnxStorage = new AzureSynchronizedMemoryStorageStub(new[] + { + new Uri("http://tempuri.org/packages/listedpackage.1.0.0.nupkg") + }, areRequiredPropertiesPresentAsync: false); + _catalogToDnxStorageFactory = new TestStorageFactory(name => _catalogToDnxStorage.WithName(name)); + + var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json"); + var nupkgUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.0/listedpackage.1.0.0.nupkg"); + var nuspecUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.0/listedpackage.nuspec"); + var nupkgStream = File.OpenRead("Packages\\ListedPackage.1.0.0.zip"); + var expectedNupkg = GetStreamBytes(nupkgStream); + + await _catalogToDnxStorage.SaveAsync( + new Uri("http://tempuri.org/listedpackage/index.json"), + new StringStorageContent(GetExpectedIndexJsonContent("1.0.0")), + CancellationToken.None); + + _target = new DnxCatalogCollector( + new Uri("http://tempuri.org/index.json"), + _catalogToDnxStorageFactory, + _nullPreferredPackageSourceStorage, + _contentBaseAddress, + Mock.Of(), + _logger, + _maxDegreeOfParallelism, + () => _mockServer); + + var catalogStorage = Catalogs.CreateTestCatalogWithOnePackage(); + await _mockServer.AddStorageAsync(catalogStorage); + + _mockServer.SetAction( + "/packages/listedpackage.1.0.0.nupkg", + request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(nupkgStream) })); + + var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); + ReadCursor back = MemoryCursor.CreateMax(); + + await _target.Run(front, back, CancellationToken.None); + + Assert.Equal(4, _catalogToDnxStorage.Content.Count); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri)); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(nupkgUri)); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(nuspecUri)); + Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(_cursorJsonUri)); + Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(indexJsonUri, out var indexJson)); + Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(nupkgUri)); + Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(nuspecUri)); + + Assert.Equal(GetExpectedIndexJsonContent("1.0.0"), Encoding.UTF8.GetString(indexJson)); + } + [Fact] + public async Task Run_WhenPackageIsAlreadySynchronizedButNotInIndex_ProcessesPackage() + { _catalogToDnxStorage = new SynchronizedMemoryStorage(new[] { new Uri("http://tempuri.org/packages/listedpackage.1.0.1.nupkg"), @@ -455,16 +573,19 @@ public async Task Run_WhenPackageIsAlreadySynchronizedButNotInIndex_ProcessesPac _mockServer = new MockServerHttpClientHandler(); _mockServer.SetAction("/", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK))); + var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json"); + var nupkgUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.1/listedpackage.1.0.1.nupkg"); + var nuspecUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.1/listedpackage.nuspec"); + _target = new DnxCatalogCollector( new Uri("http://tempuri.org/index.json"), _catalogToDnxStorageFactory, - new Mock().Object, + _nullPreferredPackageSourceStorage, + _contentBaseAddress, + Mock.Of(), new Mock().Object, _maxDegreeOfParallelism, - () => _mockServer) - { - ContentBaseAddress = new Uri("http://tempuri.org/packages/") - }; + () => _mockServer); var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete(); @@ -711,11 +832,11 @@ private static MemoryStream CreateZipStreamWithEntry(string name, string content private class SynchronizedMemoryStorage : MemoryStorage { - private readonly HashSet _synchronizedUris; + protected HashSet SynchronizedUris { get; private set; } public SynchronizedMemoryStorage(IEnumerable synchronizedUris) { - _synchronizedUris = new HashSet(synchronizedUris); + SynchronizedUris = new HashSet(synchronizedUris); } protected SynchronizedMemoryStorage( @@ -725,12 +846,12 @@ public SynchronizedMemoryStorage(IEnumerable synchronizedUris) HashSet synchronizedUris) : base(baseAddress, content, contentBytes) { - _synchronizedUris = synchronizedUris; + SynchronizedUris = synchronizedUris; } public override Task AreSynchronized(Uri firstResourceUri, Uri secondResourceUri) { - return Task.FromResult(_synchronizedUris.Contains(firstResourceUri)); + return Task.FromResult(SynchronizedUris.Contains(firstResourceUri)); } public override Storage WithName(string name) @@ -739,7 +860,56 @@ public override Storage WithName(string name) new Uri(BaseAddress + name), Content, ContentBytes, - _synchronizedUris); + SynchronizedUris); + } + } + + private class AzureSynchronizedMemoryStorageStub : SynchronizedMemoryStorage, IAzureStorage + { + private readonly bool _areRequiredPropertiesPresentAsync; + + internal AzureSynchronizedMemoryStorageStub( + IEnumerable synchronizedUris, + bool areRequiredPropertiesPresentAsync) + : base(synchronizedUris) + { + _areRequiredPropertiesPresentAsync = areRequiredPropertiesPresentAsync; + } + + protected AzureSynchronizedMemoryStorageStub( + Uri baseAddress, + ConcurrentDictionary content, + ConcurrentDictionary contentBytes, + HashSet synchronizedUris, + bool areRequiredPropertiesPresentAsync) + : base(baseAddress, content, contentBytes, synchronizedUris) + { + _areRequiredPropertiesPresentAsync = areRequiredPropertiesPresentAsync; + } + + public override Storage WithName(string name) + { + return new AzureSynchronizedMemoryStorageStub( + new Uri(BaseAddress + name), + Content, + ContentBytes, + SynchronizedUris, + _areRequiredPropertiesPresentAsync); + } + + public Task GetCloudBlockBlobReferenceAsync(Uri blobUri) + { + throw new NotImplementedException(); + } + + public Task GetCloudBlockBlobReferenceAsync(string name) + { + throw new NotImplementedException(); + } + + public Task HasPropertiesAsync(Uri blobUri, string contentType, string cacheControl) + { + return Task.FromResult(_areRequiredPropertiesPresentAsync); } } } diff --git a/tests/NgTests/DnxMakerTests.cs b/tests/NgTests/DnxMakerTests.cs index 6fff9092b..f0dc73792 100644 --- a/tests/NgTests/DnxMakerTests.cs +++ b/tests/NgTests/DnxMakerTests.cs @@ -2,15 +2,18 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Text; using System.Threading; using System.Threading.Tasks; +using Moq; using Newtonsoft.Json.Linq; using NgTests.Infrastructure; using NuGet.Services.Metadata.Catalog.Dnx; using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Persistence; using NuGet.Versioning; using Xunit; @@ -18,6 +21,11 @@ namespace NgTests { public class DnxMakerTests { + private const string _expectedCacheControl = "max-age=120"; + private const string _expectedNuspecContentType = "text/xml"; + private const string _expectedPackageContentType = "application/octet-stream"; + private const string _expectedPackageVersionIndexJsonCacheControl = "no-store"; + private const string _expectedPackageVersionIndexJsonContentType = "application/json"; private const string _packageId = "testid"; private const string _nupkgData = "nupkg data"; private const string _nuspecData = "nuspec data"; @@ -174,8 +182,8 @@ public async Task AddPackageAsync_WhenNupkgStreamIsNull_Throws() () => maker.AddPackageAsync( nupkgStream: null, nuspec: "a", - id: "b", - version: "c", + packageId: "b", + normalizedPackageVersion: "c", cancellationToken: CancellationToken.None)); Assert.Equal("nupkgStream", exception.ParamName); @@ -192,8 +200,8 @@ public async Task AddPackageAsync_WhenNuspecIsNullOrEmpty_Throws(string nuspec) () => maker.AddPackageAsync( Stream.Null, nuspec, - id: "a", - version: "b", + packageId: "a", + normalizedPackageVersion: "b", cancellationToken: CancellationToken.None)); Assert.Equal("nuspec", exception.ParamName); @@ -203,7 +211,7 @@ public async Task AddPackageAsync_WhenNuspecIsNullOrEmpty_Throws(string nuspec) [Theory] [InlineData(null)] [InlineData("")] - public async Task AddPackageAsync_WhenIdIsNullOrEmpty_Throws(string id) + public async Task AddPackageAsync_WhenPackageIdIsNullOrEmpty_Throws(string packageId) { var maker = CreateDnxMaker(); @@ -211,18 +219,18 @@ public async Task AddPackageAsync_WhenIdIsNullOrEmpty_Throws(string id) () => maker.AddPackageAsync( Stream.Null, nuspec: "a", - id: id, - version: "b", + packageId: packageId, + normalizedPackageVersion: "b", cancellationToken: CancellationToken.None)); - Assert.Equal("id", exception.ParamName); + Assert.Equal("packageId", exception.ParamName); Assert.StartsWith("The argument must not be null or empty.", exception.Message); } [Theory] [InlineData(null)] [InlineData("")] - public async Task AddPackageAsync_WhenVersionIsNullOrEmpty_Throws(string version) + public async Task AddPackageAsync_WhenNormalizedPackageVersionIsNullOrEmpty_Throws(string normalizedPackageVersion) { var maker = CreateDnxMaker(); @@ -230,11 +238,11 @@ public async Task AddPackageAsync_WhenVersionIsNullOrEmpty_Throws(string version () => maker.AddPackageAsync( Stream.Null, nuspec: "a", - id: "b", - version: version, + packageId: "b", + normalizedPackageVersion: normalizedPackageVersion, cancellationToken: CancellationToken.None)); - Assert.Equal("version", exception.ParamName); + Assert.Equal("normalizedPackageVersion", exception.ParamName); Assert.StartsWith("The argument must not be null or empty.", exception.Message); } @@ -247,8 +255,8 @@ public async Task AddPackageAsync_WhenCancelled_Throws() () => maker.AddPackageAsync( Stream.Null, nuspec: "a", - id: "b", - version: "c", + packageId: "b", + normalizedPackageVersion: "c", cancellationToken: new CancellationToken(canceled: true))); } @@ -274,13 +282,133 @@ public async Task AddPackageAsync_WithValidVersion_PopulatesStorageWithNupkgAndN Assert.Equal(2, catalogToDnxStorage.Content.Count); Assert.Equal(2, storageForPackage.Content.Count); - Verify(catalogToDnxStorage, expectedNupkg, _nupkgData); - Verify(catalogToDnxStorage, expectedNuspec, _nuspecData); - Verify(storageForPackage, expectedNupkg, _nupkgData); - Verify(storageForPackage, expectedNuspec, _nuspecData); + Verify(catalogToDnxStorage, expectedNupkg, _nupkgData, _expectedCacheControl, _expectedPackageContentType); + Verify(catalogToDnxStorage, expectedNuspec, _nuspecData, _expectedCacheControl, _expectedNuspecContentType); + Verify(storageForPackage, expectedNupkg, _nupkgData, _expectedCacheControl, _expectedPackageContentType); + Verify(storageForPackage, expectedNuspec, _nuspecData, _expectedCacheControl, _expectedNuspecContentType); } } + [Fact] + public async Task AddPackageAsync_WithStorage_WhenSourceStorageIsNull_Throws() + { + var maker = CreateDnxMaker(); + + var exception = await Assert.ThrowsAsync( + () => maker.AddPackageAsync( + sourceStorage: null, + nuspec: "a", + packageId: "b", + normalizedPackageVersion: "c", + cancellationToken: CancellationToken.None)); + + Assert.Equal("sourceStorage", exception.ParamName); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public async Task AddPackageAsync_WithStorage_WhenNuspecIsNullOrEmpty_Throws(string nuspec) + { + var maker = CreateDnxMaker(); + + var exception = await Assert.ThrowsAsync( + () => maker.AddPackageAsync( + Mock.Of(), + nuspec, + packageId: "a", + normalizedPackageVersion: "b", + cancellationToken: CancellationToken.None)); + + Assert.Equal("nuspec", exception.ParamName); + Assert.StartsWith("The argument must not be null or empty.", exception.Message); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public async Task AddPackageAsync_WithStorage_WhenPackageIdIsNullOrEmpty_Throws(string id) + { + var maker = CreateDnxMaker(); + + var exception = await Assert.ThrowsAsync( + () => maker.AddPackageAsync( + Mock.Of(), + nuspec: "a", + packageId: id, + normalizedPackageVersion: "b", + cancellationToken: CancellationToken.None)); + + Assert.Equal("packageId", exception.ParamName); + Assert.StartsWith("The argument must not be null or empty.", exception.Message); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public async Task AddPackageAsync_WithStorage_WhenNormalizedPackageVersionIsNullOrEmpty_Throws(string version) + { + var maker = CreateDnxMaker(); + + var exception = await Assert.ThrowsAsync( + () => maker.AddPackageAsync( + Mock.Of(), + nuspec: "a", + packageId: "b", + normalizedPackageVersion: version, + cancellationToken: CancellationToken.None)); + + Assert.Equal("normalizedPackageVersion", exception.ParamName); + Assert.StartsWith("The argument must not be null or empty.", exception.Message); + } + + [Fact] + public async Task AddPackageAsync_WithStorage_WhenCancelled_Throws() + { + var maker = CreateDnxMaker(); + + await Assert.ThrowsAsync( + () => maker.AddPackageAsync( + Mock.Of(), + nuspec: "a", + packageId: "b", + normalizedPackageVersion: "c", + cancellationToken: new CancellationToken(canceled: true))); + } + + [Theory] + [MemberData(nameof(PackageVersions))] + public async Task AddPackageAsync_WithStorage_WithIStorage_PopulatesStorageWithNupkgAndNuspec(string version) + { + var catalogToDnxStorage = new AzureStorageStub(); + var catalogToDnxStorageFactory = new TestStorageFactory(name => catalogToDnxStorage.WithName(name)); + var maker = new DnxMaker(catalogToDnxStorageFactory); + var normalizedVersion = NuGetVersionUtility.NormalizeVersion(version); + var sourceStorage = new AzureStorageStub(); + + var dnxEntry = await maker.AddPackageAsync( + sourceStorage, + _nuspecData, + _packageId, + normalizedVersion, + CancellationToken.None); + + var expectedNuspecUri = new Uri($"{catalogToDnxStorage.BaseAddress}{_packageId}/{normalizedVersion}/{_packageId}.nuspec"); + var expectedNupkgUri = new Uri($"{catalogToDnxStorage.BaseAddress}{_packageId}/{normalizedVersion}/{_packageId}.{normalizedVersion}.nupkg"); + var expectedSourceUri = new Uri(sourceStorage.BaseAddress, $"{_packageId}.{normalizedVersion}.nupkg"); + var storageForPackage = (MemoryStorage)catalogToDnxStorageFactory.Create(_packageId); + + Assert.Equal(expectedNuspecUri, dnxEntry.Nuspec); + Assert.Equal(expectedNupkgUri, dnxEntry.Nupkg); + Assert.Equal(2, catalogToDnxStorage.Content.Count); + Assert.Equal(2, storageForPackage.Content.Count); + + Verify(catalogToDnxStorage, expectedNupkgUri, expectedSourceUri.AbsoluteUri, _expectedCacheControl, _expectedPackageContentType); + Verify(catalogToDnxStorage, expectedNuspecUri, _nuspecData, _expectedCacheControl, _expectedNuspecContentType); + Verify(storageForPackage, expectedNupkgUri, expectedSourceUri.AbsoluteUri, _expectedCacheControl, _expectedPackageContentType); + Verify(storageForPackage, expectedNuspecUri, _nuspecData, _expectedCacheControl, _expectedNuspecContentType); + } + [Theory] [InlineData(null)] [InlineData("")] @@ -415,8 +543,8 @@ public async Task UpdatePackageVersionIndexAsync_WithValidVersion_CreatesIndex(s Assert.Equal(1, catalogToDnxStorage.Content.Count); Assert.Equal(1, storageForPackage.Content.Count); - Verify(catalogToDnxStorage, indexJsonUri, expectedContent); - Verify(storageForPackage, indexJsonUri, expectedContent); + Verify(catalogToDnxStorage, indexJsonUri, expectedContent, _expectedPackageVersionIndexJsonCacheControl, _expectedPackageVersionIndexJsonContentType); + Verify(storageForPackage, indexJsonUri, expectedContent, _expectedPackageVersionIndexJsonCacheControl, _expectedPackageVersionIndexJsonContentType); Assert.Equal(new[] { normalizedVersion }, versions); } @@ -548,9 +676,17 @@ private static string GetExpectedIndexJsonContent(string version) return $"{{\r\n \"versions\": [\r\n \"{version}\"\r\n ]\r\n}}"; } - private static void Verify(MemoryStorage storage, Uri uri, string expectedContent) + private static void Verify( + MemoryStorage storage, + Uri uri, + string expectedContent, + string expectedCacheControl, + string expectedContentType) { - Assert.True(storage.Content.ContainsKey(uri)); + Assert.True(storage.Content.TryGetValue(uri, out var content)); + Assert.Equal(expectedCacheControl, content.CacheControl); + Assert.Equal(expectedContentType, content.ContentType); + Assert.True(storage.ContentBytes.TryGetValue(uri, out var bytes)); Assert.Equal(Encoding.UTF8.GetBytes(expectedContent), bytes); @@ -567,5 +703,65 @@ private static void Verify(MemoryStorage storage, Uri uri, string expectedConten Assert.InRange(list.LastModifiedUtc.Value, utc.AddMinutes(-1), utc); } } + + private sealed class AzureStorageStub : MemoryStorage, IAzureStorage + { + internal AzureStorageStub() + { + } + + private AzureStorageStub( + Uri baseAddress, + ConcurrentDictionary content, + ConcurrentDictionary contentBytes) + : base(baseAddress, content, contentBytes) + { + } + + public override Storage WithName(string name) + { + return new AzureStorageStub( + new Uri(BaseAddress + name), + Content, + ContentBytes); + } + + public Task GetCloudBlockBlobReferenceAsync(Uri blobUri) + { + throw new NotImplementedException(); + } + + public Task GetCloudBlockBlobReferenceAsync(string name) + { + throw new NotImplementedException(); + } + + public Task HasPropertiesAsync(Uri blobUri, string contentType, string cacheControl) + { + throw new NotImplementedException(); + } + + protected override Task OnCopyAsync( + Uri sourceUri, + IStorage destinationStorage, + Uri destinationUri, + IReadOnlyDictionary destinationProperties, + CancellationToken cancellationToken) + { + var destinationMemoryStorage = (AzureStorageStub)destinationStorage; + + string cacheControl = null; + string contentType = null; + + destinationProperties?.TryGetValue(StorageConstants.CacheControl, out cacheControl); + destinationProperties?.TryGetValue(StorageConstants.ContentType, out contentType); + + destinationMemoryStorage.Content[destinationUri] = new StringStorageContent(sourceUri.AbsoluteUri, contentType, cacheControl); + destinationMemoryStorage.ContentBytes[destinationUri] = Encoding.UTF8.GetBytes(sourceUri.AbsoluteUri); + destinationMemoryStorage.ListMock[destinationUri] = new StorageListItem(destinationUri, DateTime.UtcNow); + + return Task.FromResult(0); + } + } } } \ No newline at end of file diff --git a/tests/NgTests/Infrastructure/MemoryStorage.cs b/tests/NgTests/Infrastructure/MemoryStorage.cs index e29bd8ecc..d986b67b9 100644 --- a/tests/NgTests/Infrastructure/MemoryStorage.cs +++ b/tests/NgTests/Infrastructure/MemoryStorage.cs @@ -12,8 +12,7 @@ namespace NgTests.Infrastructure { - public class MemoryStorage - : Storage + public class MemoryStorage : Storage { public ConcurrentDictionary Content { get; } @@ -50,6 +49,11 @@ public MemoryStorage(Uri baseAddress) } } + public override Task GetOptimisticConcurrencyControlTokenAsync(Uri resourceUri, CancellationToken cancellationToken) + { + return Task.FromResult(OptimisticConcurrencyControlToken.Null); + } + private static StorageListItem CreateStorageListItem(Uri uri) { return new StorageListItem(uri, DateTime.UtcNow); @@ -60,7 +64,17 @@ public virtual Storage WithName(string name) return new MemoryStorage(new Uri(BaseAddress + name), Content, ContentBytes); } - protected override async Task OnSave(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) + protected override Task OnCopyAsync( + Uri sourceUri, + IStorage destinationStorage, + Uri destinationUri, + IReadOnlyDictionary destinationProperties, + CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + + protected override async Task OnSaveAsync(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) { Content[resourceUri] = content; @@ -77,7 +91,7 @@ protected override async Task OnSave(Uri resourceUri, StorageContent content, Ca ListMock[resourceUri] = CreateStorageListItem(resourceUri); } - protected override Task OnLoad(Uri resourceUri, CancellationToken cancellationToken) + protected override Task OnLoadAsync(Uri resourceUri, CancellationToken cancellationToken) { StorageContent content; @@ -86,7 +100,7 @@ protected override Task OnLoad(Uri resourceUri, CancellationToke return Task.FromResult(content); } - protected override Task OnDelete(Uri resourceUri, CancellationToken cancellationToken) + protected override Task OnDeleteAsync(Uri resourceUri, CancellationToken cancellationToken) { Content.TryRemove(resourceUri, out var content); ContentBytes.TryRemove(resourceUri, out var contentBytes);