From 615f455884a6143296f26a9cf2f746807d0f76ef Mon Sep 17 00:00:00 2001 From: Damon Tivel Date: Mon, 6 Aug 2018 10:12:26 -0700 Subject: [PATCH 01/15] Feed2Catalog: replace inner loops with one outer loop (#329) Resolve https://github.com/NuGet/Engineering/issues/1526. --- src/Ng/Jobs/Feed2CatalogJob.cs | 120 +++---- tests/NgTests/Feed2CatalogTests.cs | 492 ++++++++++++++++++----------- 2 files changed, 358 insertions(+), 254 deletions(-) diff --git a/src/Ng/Jobs/Feed2CatalogJob.cs b/src/Ng/Jobs/Feed2CatalogJob.cs index 3c4fe9943..a8e248f41 100644 --- a/src/Ng/Jobs/Feed2CatalogJob.cs +++ b/src/Ng/Jobs/Feed2CatalogJob.cs @@ -115,9 +115,9 @@ protected override async Task RunInternal(CancellationToken cancellationToken) using (TelemetryService.TrackDuration(TelemetryConstants.JobLoopSeconds)) using (var client = CreateHttpClient()) { - uint packagesDeleted = 0; - uint packagesCreated = 0; - uint packagesEdited = 0; + uint packagesDeleted; + uint packagesCreated; + uint packagesEdited; client.Timeout = Timeout; @@ -127,36 +127,35 @@ protected override async Task RunInternal(CancellationToken cancellationToken) Logger, PreferredPackageSourceStorage); - // baseline timestamps - var catalogProperties = await FeedHelpers.GetCatalogPropertiesAsync(CatalogStorage, TelemetryService, cancellationToken); - var lastCreated = catalogProperties.LastCreated ?? (StartDate ?? Constants.DateTimeMinValueUtc); - var lastEdited = catalogProperties.LastEdited ?? lastCreated; - var lastDeleted = catalogProperties.LastDeleted ?? lastCreated; - - if (lastDeleted == Constants.DateTimeMinValueUtc) + do { - lastDeleted = lastCreated; - } + packagesDeleted = 0; + packagesCreated = 0; + packagesEdited = 0; - try - { - // fetch and add all DELETED packages - if (lastDeleted > Constants.DateTimeMinValueUtc) + // baseline timestamps + var catalogProperties = await FeedHelpers.GetCatalogPropertiesAsync(CatalogStorage, TelemetryService, cancellationToken); + var lastCreated = catalogProperties.LastCreated ?? (StartDate ?? Constants.DateTimeMinValueUtc); + var lastEdited = catalogProperties.LastEdited ?? lastCreated; + var lastDeleted = catalogProperties.LastDeleted ?? lastCreated; + + if (lastDeleted == Constants.DateTimeMinValueUtc) + { + lastDeleted = SkipCreatedPackagesProcessing ? lastEdited : lastCreated; + } + + try { - using (TelemetryService.TrackDuration(TelemetryConstants.DeletedPackagesSeconds)) + if (lastDeleted > Constants.DateTimeMinValueUtc) { - SortedList> deletedPackages; - var previousLastDeleted = Constants.DateTimeMinValueUtc; - do + using (TelemetryService.TrackDuration(TelemetryConstants.DeletedPackagesSeconds)) { - // Get deleted packages Logger.LogInformation("CATALOG LastDeleted: {CatalogDeletedTime}", lastDeleted.ToString("O")); - deletedPackages = await GetDeletedPackages(AuditingStorage, lastDeleted); + var deletedPackages = await GetDeletedPackages(AuditingStorage, lastDeleted); - var deletedPackagesCount = deletedPackages.SelectMany(x => x.Value).Count(); - Logger.LogInformation("FEED DeletedPackages: {DeletedPackagesCount}", deletedPackagesCount); - packagesDeleted += (uint)deletedPackagesCount; + packagesDeleted = (uint)deletedPackages.SelectMany(x => x.Value).Count(); + Logger.LogInformation("FEED DeletedPackages: {DeletedPackagesCount}", packagesDeleted); // We want to ensure a commit only contains each package once at most. // Therefore we segment by package id + version. @@ -169,33 +168,19 @@ protected override async Task RunInternal(CancellationToken cancellationToken) // Wait for one second to ensure the next catalog commit gets a new timestamp Thread.Sleep(TimeSpan.FromSeconds(1)); } - - if (previousLastDeleted == lastDeleted) - { - break; - } - previousLastDeleted = lastDeleted; } - while (deletedPackages.Count > 0); } - } - if (!SkipCreatedPackagesProcessing) - { - using (TelemetryService.TrackDuration(TelemetryConstants.CreatedPackagesSeconds)) + if (!SkipCreatedPackagesProcessing) { - // THEN fetch and add all newly CREATED packages - in order - SortedList> createdPackages; - var previousLastCreated = Constants.DateTimeMinValueUtc; - do + using (TelemetryService.TrackDuration(TelemetryConstants.CreatedPackagesSeconds)) { Logger.LogInformation("CATALOG LastCreated: {CatalogLastCreatedTime}", lastCreated.ToString("O")); - createdPackages = await GetCreatedPackages(client, Gallery, lastCreated, Top); + var createdPackages = await GetCreatedPackages(client, Gallery, lastCreated, Top); - var createdPackagesCount = createdPackages.SelectMany(x => x.Value).Count(); - Logger.LogInformation("FEED CreatedPackages: {CreatedPackagesCount}", createdPackagesCount); - packagesCreated += (uint)createdPackagesCount; + packagesCreated = (uint)createdPackages.SelectMany(x => x.Value).Count(); + Logger.LogInformation("FEED CreatedPackages: {CreatedPackagesCount}", packagesCreated); lastCreated = await FeedHelpers.DownloadMetadata2CatalogAsync( packageCatalogItemCreator, @@ -210,30 +195,17 @@ protected override async Task RunInternal(CancellationToken cancellationToken) cancellationToken: cancellationToken, telemetryService: TelemetryService, logger: Logger); - if (previousLastCreated == lastCreated) - { - break; - } - previousLastCreated = lastCreated; } - while (createdPackages.Count > 0); } - } - using (TelemetryService.TrackDuration(TelemetryConstants.EditedPackagesSeconds)) - { - // THEN fetch and add all EDITED packages - in order - SortedList> editedPackages; - var previousLastEdited = Constants.DateTimeMinValueUtc; - do + using (TelemetryService.TrackDuration(TelemetryConstants.EditedPackagesSeconds)) { Logger.LogInformation("CATALOG LastEdited: {CatalogLastEditedTime}", lastEdited.ToString("O")); - editedPackages = await GetEditedPackages(client, Gallery, lastEdited, Top); + var editedPackages = await GetEditedPackages(client, Gallery, lastEdited, Top); - var editedPackagesCount = editedPackages.SelectMany(x => x.Value).Count(); - Logger.LogInformation("FEED EditedPackages: {EditedPackagesCount}", editedPackagesCount); - packagesEdited += (uint)editedPackagesCount; + packagesEdited = (uint)editedPackages.SelectMany(x => x.Value).Count(); + Logger.LogInformation("FEED EditedPackages: {EditedPackagesCount}", packagesEdited); lastEdited = await FeedHelpers.DownloadMetadata2CatalogAsync( packageCatalogItemCreator, @@ -248,26 +220,20 @@ protected override async Task RunInternal(CancellationToken cancellationToken) cancellationToken: cancellationToken, telemetryService: TelemetryService, logger: Logger); - if (previousLastEdited == lastEdited) - { - break; - } - previousLastEdited = lastEdited; } - while (editedPackages.Count > 0); } - } - finally - { - TelemetryService.TrackMetric(TelemetryConstants.DeletedPackagesCount, packagesDeleted); - - if (!SkipCreatedPackagesProcessing) + finally { - TelemetryService.TrackMetric(TelemetryConstants.CreatedPackagesCount, packagesCreated); - } + TelemetryService.TrackMetric(TelemetryConstants.DeletedPackagesCount, packagesDeleted); - TelemetryService.TrackMetric(TelemetryConstants.EditedPackagesCount, packagesEdited); - } + if (!SkipCreatedPackagesProcessing) + { + TelemetryService.TrackMetric(TelemetryConstants.CreatedPackagesCount, packagesCreated); + } + + TelemetryService.TrackMetric(TelemetryConstants.EditedPackagesCount, packagesEdited); + } + } while (packagesDeleted > 0 || packagesCreated > 0 || packagesEdited > 0); } } diff --git a/tests/NgTests/Feed2CatalogTests.cs b/tests/NgTests/Feed2CatalogTests.cs index 5ddf37ae5..3fbce826b 100644 --- a/tests/NgTests/Feed2CatalogTests.cs +++ b/tests/NgTests/Feed2CatalogTests.cs @@ -38,15 +38,11 @@ public class Feed2CatalogTests private const int _top = 1; private bool _isDisposed; - private DateTime _catalogLastDeleted; - private DateTime _catalogLastCreated; - private DateTime _catalogLastEdited; private DateTime _feedLastCreated; private DateTime _feedLastEdited; private DateTimeOffset _timestamp; private bool _hasFirstRunOnceAsyncBeenCalledBefore; private int _lastFeedEntriesCount; - private int _catalogBatchesProcessed; private readonly List _packageOperations; private readonly Random _random; private readonly MemoryStorage _auditingStorage; @@ -58,10 +54,6 @@ public class Feed2CatalogTests public Feed2CatalogTests() { - _catalogLastDeleted = Constants.DateTimeMinValueUtc; - _catalogLastCreated = Constants.DateTimeMinValueUtc; - _catalogLastEdited = Constants.DateTimeMinValueUtc; - _server = new MockServerHttpClientHandler(); _random = new Random(); _packageOperations = new List(); @@ -93,43 +85,71 @@ public async Task RunInternal_WithNoCatalogAndNoActivity_DoesNotCreateCatalog(bo { InitializeTest(skipCreatedPackagesProcessing); - await RunInternalAndVerifyAsync(CancellationToken.None); + await RunInternalAndVerifyAsync(); } [Fact] public async Task RunInternal_WithNoCatalogAndCreatedPackageInFeed_CreatesCatalog() { - InitializeTest(); + InitializeTest(skipCreatedPackagesProcessing: false); - AddCreatedPackageToFeed(); + var package = AddCreatedPackageToFeed(); - await RunInternalAndVerifyAsync(CancellationToken.None); + await RunInternalAndVerifyAsync( + expectedLastCreated: package.ODataPackage.Created, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: Constants.DateTimeMinValueUtc); } [Fact] - public async Task RunInternal_WithNoCatalogAndCreatedPackageInFeedAndWithCreatedPackagesSkipped_DoesNotUpdateCatalog() + public async Task RunInternal_WithNoCatalogAndCreatedPackageInFeedAndWithCreatedPackagesSkipped_DoesNotCreateCatalog() { InitializeTest(skipCreatedPackagesProcessing: true); AddCreatedPackageToFeed(); - await RunInternalAndVerifyAsync(CancellationToken.None); + await RunInternalAndVerifyAsync(); } - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task RunInternal_WithCatalogAndNoActivity_DoesNotUpdateCatalog(bool skipCreatedPackagesProcessing) + [Fact] + public async Task RunInternal_WithCatalogAndNoActivity_DoesNotUpdateCatalog() { - InitializeTest(skipCreatedPackagesProcessing); + InitializeTest(skipCreatedPackagesProcessing: false); - AddCreatedPackageToFeed(); + var package = AddCreatedPackageToFeed(); // Create the catalog. - await RunInternalAndVerifyAsync(CancellationToken.None); + await RunInternalAndVerifyAsync( + expectedLastCreated: package.ODataPackage.Created, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: Constants.DateTimeMinValueUtc); // Nothing new in the feed this time. - await RunInternalAndVerifyAsync(CancellationToken.None); + await RunInternalAndVerifyAsync( + expectedLastCreated: package.ODataPackage.Created, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: Constants.DateTimeMinValueUtc); + } + + [Fact] + public async Task RunInternal_WithCatalogAndNoActivityAndWithCreatedPackagesSkipped_DoesNotUpdateCatalog() + { + InitializeTest(skipCreatedPackagesProcessing: true); + + var package = CreatePackageCreationOrEdit(); + var editedPackage = AddEditedPackageToFeed(package); + + // Create the catalog. + await RunInternalAndVerifyAsync( + expectedLastCreated: editedPackage.ODataPackage.LastEdited, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: editedPackage.ODataPackage.LastEdited); + + // Nothing new in the feed this time. + await RunInternalAndVerifyAsync( + expectedLastCreated: editedPackage.ODataPackage.LastEdited, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: editedPackage.ODataPackage.LastEdited); } [Fact] @@ -137,7 +157,7 @@ public async Task RunInternal_WithCatalogAndNoActivity_DoesNotUpdateCatalog(bool // https://github.com/NuGet/NuGetGallery/issues/2841 public async Task RunInternal_WithPackagesWithSameCreatedTimeInFeedAndWhenProcessedInDifferentCatalogBatches_SkipsSecondEntry() { - InitializeTest(); + InitializeTest(skipCreatedPackagesProcessing: false); var package1 = AddCreatedPackageToFeed(); var package2 = AddCreatedPackageToFeed(); @@ -145,7 +165,11 @@ public async Task RunInternal_WithPackagesWithSameCreatedTimeInFeedAndWhenProces package2.ODataPackage.Created = package1.ODataPackage.Created; // Remove the "package2" argument if/when the bug is fixed. - await RunInternalAndVerifyAsync(CancellationToken.None, package2); + await RunInternalAndVerifyAsync( + expectedLastCreated: package1.ODataPackage.Created, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: Constants.DateTimeMinValueUtc, + skippedPackage: package2); } [Fact] @@ -153,12 +177,15 @@ public async Task RunInternal_WithPackagesWithSameCreatedTimeInFeedAndWhenProces // https://github.com/NuGet/NuGetGallery/issues/2841 public async Task RunInternal_WithPackagesWithSameLastEditedTimeInFeedAndWhenProcessedInDifferentCatalogBatches_SkipsSecondEntry() { - InitializeTest(); + InitializeTest(skipCreatedPackagesProcessing: false); var package1 = AddCreatedPackageToFeed(); var package2 = AddCreatedPackageToFeed(); - await RunInternalAndVerifyAsync(CancellationToken.None); + await RunInternalAndVerifyAsync( + expectedLastCreated: package2.ODataPackage.Created, + expectedLastDeleted: package1.ODataPackage.Created, + expectedLastEdited: Constants.DateTimeMinValueUtc); package1 = AddEditedPackageToFeed(package1); package2 = AddEditedPackageToFeed(package2); @@ -166,21 +193,31 @@ public async Task RunInternal_WithPackagesWithSameLastEditedTimeInFeedAndWhenPro package2.ODataPackage.LastEdited = package1.ODataPackage.LastEdited; // Remove the "package2" argument if/when the bug is fixed. - await RunInternalAndVerifyAsync(CancellationToken.None, package2); + await RunInternalAndVerifyAsync( + expectedLastCreated: package2.ODataPackage.Created, + expectedLastDeleted: package1.ODataPackage.Created, + expectedLastEdited: package1.ODataPackage.LastEdited, + skippedPackage: package2); } [Fact] public async Task RunInternal_WithCreatedPackagesInFeedAtDifferentTimes_UpdatesCatalog() { - InitializeTest(); + InitializeTest(skipCreatedPackagesProcessing: false); - AddCreatedPackageToFeed(); + var package1 = AddCreatedPackageToFeed(); - await RunInternalAndVerifyAsync(CancellationToken.None); + await RunInternalAndVerifyAsync( + expectedLastCreated: package1.ODataPackage.Created, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: Constants.DateTimeMinValueUtc); - AddCreatedPackageToFeed(); + var package2 = AddCreatedPackageToFeed(); - await RunInternalAndVerifyAsync(CancellationToken.None); + await RunInternalAndVerifyAsync( + expectedLastCreated: package2.ODataPackage.Created, + expectedLastDeleted: package1.ODataPackage.Created, + expectedLastEdited: Constants.DateTimeMinValueUtc); } [Fact] @@ -190,141 +227,243 @@ public async Task RunInternal_WithCreatedPackagesInFeedAtDifferentTimesAndWithCr AddCreatedPackageToFeed(); - await RunInternalAndVerifyAsync(CancellationToken.None); + await RunInternalAndVerifyAsync(); AddCreatedPackageToFeed(); - await RunInternalAndVerifyAsync(CancellationToken.None); + await RunInternalAndVerifyAsync(); } - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task RunInternal_WithCreatedPackageAndEditedPackageInFeedAtDifferentTimes_UpdatesCatalog(bool skipCreatedPackagesProcessing) + [Fact] + public async Task RunInternal_WithCreatedPackageAndEditedPackageInFeedAtDifferentTimes_UpdatesCatalog() { - InitializeTest(skipCreatedPackagesProcessing); + InitializeTest(skipCreatedPackagesProcessing: false); var package = AddCreatedPackageToFeed(); - await RunInternalAndVerifyAsync(CancellationToken.None); + await RunInternalAndVerifyAsync( + expectedLastCreated: package.ODataPackage.Created, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: Constants.DateTimeMinValueUtc); - AddEditedPackageToFeed(package); + var editedPackage = AddEditedPackageToFeed(package); - await RunInternalAndVerifyAsync(CancellationToken.None); + await RunInternalAndVerifyAsync( + expectedLastCreated: package.ODataPackage.Created, + expectedLastDeleted: package.ODataPackage.Created, + expectedLastEdited: editedPackage.ODataPackage.LastEdited.Value); } - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task RunInternal_WithCreatedPackageAndEditedPackageInFeedAtSameTime_UpdatesCatalog(bool skipCreatedPackagesProcessing) + [Fact] + public async Task RunInternal_WithCreatedPackageAndEditedPackageInFeedAtDifferentTimesAndWithCreatedPackagesSkipped_UpdatesCatalog() { - InitializeTest(skipCreatedPackagesProcessing); + InitializeTest(skipCreatedPackagesProcessing: true); var package = AddCreatedPackageToFeed(); - await RunInternalAndVerifyAsync(CancellationToken.None); + await RunInternalAndVerifyAsync( + expectedLastCreated: package.ODataPackage.Created, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: Constants.DateTimeMinValueUtc); - AddEditedPackageToFeed(package); + var editedPackage = AddEditedPackageToFeed(package); - await RunInternalAndVerifyAsync(CancellationToken.None); + await RunInternalAndVerifyAsync( + expectedLastCreated: editedPackage.ODataPackage.LastEdited.Value, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: editedPackage.ODataPackage.LastEdited.Value); } [Fact] - public async Task RunInternal_WithEditedPackagesAndWithCreatedPackagesSkipped_UpdatesCatalog() + public async Task RunInternal_WithCreatedPackageAndEditedPackageInFeedAtSameTime_UpdatesCatalog() + { + InitializeTest(skipCreatedPackagesProcessing: false); + + var package = AddCreatedPackageToFeed(); + + await RunInternalAndVerifyAsync( + expectedLastCreated: package.ODataPackage.Created, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: Constants.DateTimeMinValueUtc); + + var editedPackage = AddEditedPackageToFeed(package); + + await RunInternalAndVerifyAsync( + expectedLastCreated: package.ODataPackage.Created, + expectedLastDeleted: package.ODataPackage.Created, + expectedLastEdited: editedPackage.ODataPackage.LastEdited.Value); + } + + [Fact] + public async Task RunInternal_WithCreatedPackageAndEditedPackageInFeedAtSameTimeAndWithCreatedPackagesSkipped_UpdatesCatalog() { InitializeTest(skipCreatedPackagesProcessing: true); - var package = CreatePackageCreationOrEdit(); + var package = AddCreatedPackageToFeed(); - package = AddEditedPackageToFeed(package); + await RunInternalAndVerifyAsync( + expectedLastCreated: package.ODataPackage.Created, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: Constants.DateTimeMinValueUtc); - await RunInternalAndVerifyAsync(CancellationToken.None); + var editedPackage = AddEditedPackageToFeed(package); - package = AddEditedPackageToFeed(package); + await RunInternalAndVerifyAsync( + expectedLastCreated: editedPackage.ODataPackage.LastEdited.Value, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: editedPackage.ODataPackage.LastEdited.Value); + } - await RunInternalAndVerifyAsync(CancellationToken.None); + [Fact] + public async Task RunInternal_WithEditedPackagesAndWithCreatedPackagesSkipped_UpdatesCatalog() + { + InitializeTest(skipCreatedPackagesProcessing: true); - AddEditedPackageToFeed(package); + var package = CreatePackageCreationOrEdit(); + var lastDeleted = Constants.DateTimeMinValueUtc; - await RunInternalAndVerifyAsync(CancellationToken.None); + for (var i = 0; i < 3; ++i) + { + package = AddEditedPackageToFeed(package); + + await RunInternalAndVerifyAsync( + expectedLastCreated: package.ODataPackage.LastEdited.Value, + expectedLastDeleted: lastDeleted, + expectedLastEdited: package.ODataPackage.LastEdited.Value); + + if (lastDeleted == Constants.DateTimeMinValueUtc) + { + lastDeleted = package.ODataPackage.LastEdited.Value; + } + } } [Fact] public async Task RunInternal_WithCreatedPackageThenDeletedPackage_UpdatesCatalog() { - InitializeTest(); + InitializeTest(skipCreatedPackagesProcessing: false); var package = AddCreatedPackageToFeed(); - await RunInternalAndVerifyAsync(CancellationToken.None); + await RunInternalAndVerifyAsync( + expectedLastCreated: package.ODataPackage.Created, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: Constants.DateTimeMinValueUtc); - AddDeletedPackage(package); + var deletedPackage = AddDeletedPackage(package); - await RunInternalAndVerifyAsync(CancellationToken.None); + await RunInternalAndVerifyAsync( + expectedLastCreated: package.ODataPackage.Created, + expectedLastDeleted: deletedPackage.DeletionTime.UtcDateTime, + expectedLastEdited: Constants.DateTimeMinValueUtc); } [Fact] public async Task RunInternal_WithMultipleDeletedPackagesWithDifferentPackageIdentities_ProcessesAllDeletions() { - InitializeTest(); + InitializeTest(skipCreatedPackagesProcessing: false); var package1 = AddCreatedPackageToFeed(); var package2 = AddCreatedPackageToFeed(); - await RunInternalAndVerifyAsync(CancellationToken.None); + await RunInternalAndVerifyAsync( + expectedLastCreated: package2.ODataPackage.Created, + expectedLastDeleted: package1.ODataPackage.Created, + expectedLastEdited: Constants.DateTimeMinValueUtc); - AddDeletedPackage(package1); - AddDeletedPackage(package2); + var deletedPackage1 = AddDeletedPackage(package1); + var deletedPackage2 = AddDeletedPackage(package2); - await RunInternalAndVerifyAsync(CancellationToken.None); + await RunInternalAndVerifyAsync( + expectedLastCreated: package2.ODataPackage.Created, + expectedLastDeleted: deletedPackage2.DeletionTime.UtcDateTime, + expectedLastEdited: Constants.DateTimeMinValueUtc); } [Fact] public async Task RunInternal_WithMultipleDeletedPackagesWithSamePackageIdentity_PutsEachPackageInSeparateCommit() { - InitializeTest(); + InitializeTest(skipCreatedPackagesProcessing: false); var package = AddCreatedPackageToFeed(); - await RunInternalAndVerifyAsync(CancellationToken.None); + await RunInternalAndVerifyAsync( + expectedLastCreated: package.ODataPackage.Created, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: Constants.DateTimeMinValueUtc); var deletionTime = DateTimeOffset.UtcNow; - AddDeletedPackage(package, deletionTime.UtcDateTime, isSoftDelete: true); - AddDeletedPackage(package, deletionTime.UtcDateTime, isSoftDelete: false); + var deletedPackage1 = AddDeletedPackage(package, deletionTime.UtcDateTime, isSoftDelete: true); + var deletedPackage2 = AddDeletedPackage(package, deletionTime.UtcDateTime, isSoftDelete: false); - await RunInternalAndVerifyAsync(CancellationToken.None); + await RunInternalAndVerifyAsync( + expectedLastCreated: package.ODataPackage.Created, + expectedLastDeleted: deletionTime.UtcDateTime, + expectedLastEdited: Constants.DateTimeMinValueUtc); } [Fact] - public async Task RunInternal_WithMultipleCreatedPackages_ProcessesAllCreations() + public async Task RunInternal_WithDeletedPackageOlderThan15MinutesAgo_SkipsDeletedPackage() { - InitializeTest(); + InitializeTest(skipCreatedPackagesProcessing: false); - AddCreatedPackageToFeed(); - AddCreatedPackageToFeed(); + var package = AddCreatedPackageToFeed(); + + await RunInternalAndVerifyAsync( + expectedLastCreated: package.ODataPackage.Created, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: Constants.DateTimeMinValueUtc); - await RunInternalAndVerifyAsync(CancellationToken.None); + var deletedPackage = AddDeletedPackage(package, deletionTime: DateTime.UtcNow.AddMinutes(-16)); + + await RunInternalAndVerifyAsync( + expectedLastCreated: package.ODataPackage.Created, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: Constants.DateTimeMinValueUtc, + skippedPackage: deletedPackage); } [Fact] - public async Task RunInternal_WithMultipleEditedPackages_ProcessesAllEdits() + public async Task RunInternal_WithMultipleCreatedPackages_ProcessesAllCreations() { - InitializeTest(); + InitializeTest(skipCreatedPackagesProcessing: false); var package1 = AddCreatedPackageToFeed(); var package2 = AddCreatedPackageToFeed(); - await RunInternalAndVerifyAsync(CancellationToken.None); + await RunInternalAndVerifyAsync( + expectedLastCreated: package2.ODataPackage.Created, + expectedLastDeleted: package1.ODataPackage.Created, + expectedLastEdited: Constants.DateTimeMinValueUtc); + } - AddEditedPackageToFeed(package1); - AddEditedPackageToFeed(package2); + [Fact] + public async Task RunInternal_WithMultipleEditedPackages_ProcessesAllEdits() + { + InitializeTest(skipCreatedPackagesProcessing: false); - await RunInternalAndVerifyAsync(CancellationToken.None); + var package1 = AddCreatedPackageToFeed(); + var package2 = AddCreatedPackageToFeed(); + + // Create the catalog. + await RunInternalAndVerifyAsync( + expectedLastCreated: package2.ODataPackage.Created, + expectedLastDeleted: package1.ODataPackage.Created, + expectedLastEdited: Constants.DateTimeMinValueUtc); + + var editedPackage1 = AddEditedPackageToFeed(package1); + var editedPackage2 = AddEditedPackageToFeed(package2); + + // Now test multiple edits. + await RunInternalAndVerifyAsync( + expectedLastCreated: package2.ODataPackage.Created, + expectedLastDeleted: package1.ODataPackage.Created, + expectedLastEdited: editedPackage2.ODataPackage.LastEdited.Value); } [Fact] - public async Task CreatesNewCatalogFromCreatedAndEditedPackages() + public async Task CreatesNewCatalogFromCreatedAndEditedAndDeletedPackages() { // Arrange var catalogStorage = new MemoryStorage(); @@ -361,13 +500,13 @@ public async Task CreatesNewCatalogFromCreatedAndEditedPackages() await feed2catalogTestJob.RunOnce(CancellationToken.None); // Assert - Assert.Equal(6, catalogStorage.Content.Count); + Assert.Equal(7, catalogStorage.Content.Count); // Ensure catalog has index.json var catalogIndex = catalogStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("index.json")); Assert.NotNull(catalogIndex.Key); Assert.Contains("\"nuget:lastCreated\":\"2015-01-01T00:00:00Z\"", catalogIndex.Value.GetContentString()); - Assert.Contains("\"nuget:lastDeleted\":\"0001-01-01T00:00:00Z", catalogIndex.Value.GetContentString()); + Assert.Contains("\"nuget:lastDeleted\":\"2015-01-01T01:01:01.0748028Z\"", catalogIndex.Value.GetContentString()); Assert.Contains("\"nuget:lastEdited\":\"2015-01-01T00:00:00Z\"", catalogIndex.Value.GetContentString()); // Ensure catalog has page0.json @@ -416,9 +555,11 @@ public async Task CreatesNewCatalogFromCreatedAndEditedPackages() Assert.Contains("\"id\": \"TestPackage.SemVer2\",", package4.Value.GetContentString()); Assert.Contains("\"version\": \"1.0.0-alpha.1+githash\",", package4.Value.GetContentString()); - // Ensure catalog does not have the deleted "OtherPackage" as a fresh catalog should not care about deletes var package5 = catalogStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/otherpackage.1.0.0.json")); - Assert.Null(package5.Key); + Assert.NotNull(package5.Key); + Assert.Contains("\"PackageDelete\",", package5.Value.GetContentString()); + Assert.Contains("\"id\": \"OtherPackage\",", package5.Value.GetContentString()); + Assert.Contains("\"version\": \"1.0.0\",", package5.Value.GetContentString()); } [Fact] @@ -767,7 +908,7 @@ private static Task GetCreatedPackagesSecondRequest(HttpReq }); } - private void InitializeTest(bool skipCreatedPackagesProcessing = false) + private void InitializeTest(bool skipCreatedPackagesProcessing) { _skipCreatedPackagesProcessing = skipCreatedPackagesProcessing; @@ -848,7 +989,7 @@ private PackageCreationOrEdit AddEditedPackageToFeed(PackageCreationOrEdit entry return operation; } - private void AddDeletedPackage( + private PackageDeletion AddDeletedPackage( PackageCreationOrEdit operation, DateTime? deletionTime = null, bool isSoftDelete = false) @@ -872,10 +1013,18 @@ private PackageCreationOrEdit AddEditedPackageToFeed(PackageCreationOrEdit entry _auditingStorage.Content.TryAdd(uri, new JTokenStorageContent(auditRecord)); - _packageOperations.Add(new PackageDeletion(uri, auditRecord, deletionTime.Value)); + var deletion = new PackageDeletion(uri, auditRecord, deletionTime.Value); + + _packageOperations.Add(deletion); + + return deletion; } - private async Task RunInternalAndVerifyAsync(CancellationToken cancellationToken, PackageOperation skippedOperation = null) + private async Task RunInternalAndVerifyAsync( + DateTime? expectedLastCreated = null, + DateTime? expectedLastDeleted = null, + DateTime? expectedLastEdited = null, + PackageOperation skippedPackage = null) { if (_hasFirstRunOnceAsyncBeenCalledBefore) { @@ -890,12 +1039,16 @@ private async Task RunInternalAndVerifyAsync(CancellationToken cancellationToken PrepareFeed(); - await _job.RunOnce(cancellationToken); + await _job.RunOnce(CancellationToken.None); - VerifyCatalog(skippedOperation); + VerifyCatalog(expectedLastCreated, expectedLastDeleted, expectedLastEdited, skippedPackage); } - private void VerifyCatalog(PackageOperation skippedOperation) + private void VerifyCatalog( + DateTime? expectedLastCreated, + DateTime? expectedLastDeleted, + DateTime? expectedLastEdited, + PackageOperation skippedOperation) { List verifiablePackageOperations; @@ -925,8 +1078,6 @@ private void VerifyCatalog(PackageOperation skippedOperation) if (!isEmptyCatalogBatch) { - ++_catalogBatchesProcessed; - _lastFeedEntriesCount = verifiablePackageOperations.Count; } @@ -935,6 +1086,10 @@ private void VerifyCatalog(PackageOperation skippedOperation) + 1; // page0.json Assert.Equal(expectedCatalogEntryCount, _catalogStorage.Content.Count); + Assert.True(expectedLastCreated.HasValue); + Assert.True(expectedLastDeleted.HasValue); + Assert.True(expectedLastEdited.HasValue); + var indexUri = new Uri(_baseUri, "index.json"); var pageUri = new Uri(_baseUri, "page0.json"); @@ -943,9 +1098,11 @@ private void VerifyCatalog(PackageOperation skippedOperation) VerifyCatalogIndex( verifiablePackageOperations, - isEmptyCatalogBatch, indexUri, pageUri, + expectedLastCreated.Value, + expectedLastDeleted.Value, + expectedLastEdited.Value, out commitId, out commitTimeStamp); @@ -958,16 +1115,18 @@ private void VerifyCatalog(PackageOperation skippedOperation) out var commitTimeStampDateTime)); VerifyCatalogPage(verifiablePackageOperations, indexUri, pageUri, commitId, commitTimeStamp); - VerifyCatalogPackageItems(verifiablePackageOperations, commitId, commitTimeStamp, commitTimeStampDateTime); + VerifyCatalogPackageItems(verifiablePackageOperations); Assert.True(verifiablePackageOperations.All(packageOperation => !string.IsNullOrEmpty(packageOperation.CommitId))); } private void VerifyCatalogIndex( List packageOperations, - bool isEmptyCatalogBatch, Uri indexUri, Uri pageUri, + DateTime expectedLastCreated, + DateTime expectedLastDeleted, + DateTime expectedLastEdited, out string commitId, out string commitTimeStamp) { @@ -994,42 +1153,15 @@ private void VerifyCatalog(PackageOperation skippedOperation) Assert.Equal(1, index[CatalogConstants.Count].Value()); - if (_catalogBatchesProcessed > 1 && _catalogLastDeleted == Constants.DateTimeMinValueUtc && !isEmptyCatalogBatch) - { - _catalogLastDeleted = _catalogLastCreated; - } - - _catalogLastEdited = packageOperations.OfType() - .Where(entry => entry.ODataPackage.LastEdited.HasValue) - .Select(entry => entry.ODataPackage.LastEdited.Value) - .DefaultIfEmpty(Constants.DateTimeMinValueUtc) - .Max(); - - if (_skipCreatedPackagesProcessing) - { - _catalogLastCreated = _catalogLastEdited; - } - else - { - _catalogLastCreated = packageOperations.OfType() - .Max(entry => entry.ODataPackage.Created); - } - - _catalogLastDeleted = packageOperations.OfType() - .Select(package => package.DeletionTime) - .DefaultIfEmpty(_catalogLastDeleted) - .Select(deletionTime => deletionTime.UtcDateTime) - .Max(); - Assert.Equal( - _catalogLastCreated.ToString(_catalogDateTimeFormat), + expectedLastCreated.ToString(_catalogDateTimeFormat), index[CatalogConstants.NuGetLastCreated].Value()); Assert.Equal( - _catalogLastEdited.ToString(_catalogDateTimeFormat), - index[CatalogConstants.NuGetLastEdited].Value()); - Assert.Equal( - _catalogLastDeleted.ToString(_catalogDateTimeFormat), + expectedLastDeleted.ToString(_catalogDateTimeFormat), index[CatalogConstants.NuGetLastDeleted].Value()); + Assert.Equal( + expectedLastEdited.ToString(_catalogDateTimeFormat), + index[CatalogConstants.NuGetLastEdited].Value()); var expectedItems = new JArray( new JObject( @@ -1119,6 +1251,8 @@ private void VerifyCatalog(PackageOperation skippedOperation) DateTimeFormatInfo.CurrentInfo, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal); + expectedItem.CommitTimeStampDateTime = expectedTimestamp; + var expectedUri = GetPackageDetailsUri(expectedTimestamp, expectedItem); Assert.Equal(expectedUri.AbsoluteUri, actualItem[CatalogConstants.IdKeyword].Value()); @@ -1137,37 +1271,42 @@ private void VerifyCatalog(PackageOperation skippedOperation) VerifyContext(page); } - private void VerifyCatalogPackageItems( - List packageOperations, - string commitId, - string commitTimeStamp, - DateTime commitTimeStampDateTime) + private void VerifyCatalogPackageItems(List packageOperations) { - var lastEntry = packageOperations.Last(); - var packageDetailsUri = GetPackageDetailsUri(commitTimeStampDateTime, lastEntry); + PackageOperation previousPackageOperation = null; + + foreach (var packageOperation in packageOperations) + { + if (previousPackageOperation != null) + { + Assert.True(packageOperation.CommitTimeStampDateTime >= previousPackageOperation.CommitTimeStampDateTime); + } - Assert.True(_catalogStorage.Content.TryGetValue(packageDetailsUri, out var storage)); - Assert.IsType(storage); + Uri packageDetailsUri = GetPackageDetailsUri(packageOperation.CommitTimeStampDateTime, packageOperation); - var packageDetails = JObject.Parse(((StringStorageContent)storage).Content); - Assert.NotNull(packageDetails); + Assert.True(_catalogStorage.Content.TryGetValue(packageDetailsUri, out var storage)); + Assert.IsType(storage); - packageDetails = ReadJsonWithoutDateTimeHandling(packageDetails); + var packageDetails = JObject.Parse(((StringStorageContent)storage).Content); + Assert.NotNull(packageDetails); - if (lastEntry is PackageCreationOrEdit) - { - VerifyCatalogPackageDetails(commitId, commitTimeStamp, (PackageCreationOrEdit)lastEntry, packageDetailsUri, packageDetails); - } - else - { - VerifyCatalogPackageDelete(commitId, commitTimeStamp, (PackageDeletion)lastEntry, packageDetailsUri, packageDetails); + packageDetails = ReadJsonWithoutDateTimeHandling(packageDetails); + + if (packageOperation is PackageCreationOrEdit) + { + VerifyCatalogPackageDetails((PackageCreationOrEdit)packageOperation, packageDetailsUri, packageDetails); + } + else + { + VerifyCatalogPackageDelete((PackageDeletion)packageOperation, packageDetailsUri, packageDetails); + } + + previousPackageOperation = packageOperation; } } private void VerifyCatalogPackageDetails( - string commitId, - string commitTimeStamp, - PackageCreationOrEdit lastEntry, + PackageCreationOrEdit packageOperation, Uri packageDetailsUri, JObject packageDetails) { @@ -1178,29 +1317,29 @@ private void VerifyCatalog(PackageOperation skippedOperation) Assert.Equal( new JArray(CatalogConstants.PackageDetails, CatalogConstants.CatalogPermalink), packageDetails[CatalogConstants.TypeKeyword]); - Assert.Equal(lastEntry.Package.Author, packageDetails[CatalogConstants.Authors].Value()); - Assert.Equal(commitId, packageDetails[CatalogConstants.CatalogCommitId].Value()); - Assert.Equal(commitTimeStamp, packageDetails[CatalogConstants.CatalogCommitTimeStamp].Value()); + Assert.Equal(packageOperation.Package.Author, packageDetails[CatalogConstants.Authors].Value()); + Assert.Equal(packageOperation.CommitId, packageDetails[CatalogConstants.CatalogCommitId].Value()); + Assert.Equal(packageOperation.CommitTimeStamp, packageDetails[CatalogConstants.CatalogCommitTimeStamp].Value()); Assert.Equal( - lastEntry.ODataPackage.Created.ToString(_catalogDateTimeFormat), + packageOperation.ODataPackage.Created.ToString(_catalogDateTimeFormat), packageDetails[CatalogConstants.Created].Value()); - Assert.Equal(lastEntry.ODataPackage.Description, packageDetails[CatalogConstants.Description].Value()); - Assert.Equal(lastEntry.ODataPackage.Id, packageDetails[CatalogConstants.Id].Value()); + Assert.Equal(packageOperation.ODataPackage.Description, packageDetails[CatalogConstants.Description].Value()); + Assert.Equal(packageOperation.ODataPackage.Id, packageDetails[CatalogConstants.Id].Value()); Assert.False(packageDetails[CatalogConstants.IsPrerelease].Value()); Assert.Equal( - (lastEntry.ODataPackage.LastEdited ?? Constants.DateTimeMinValueUtc).ToString(_catalogDateTimeFormat), + (packageOperation.ODataPackage.LastEdited ?? Constants.DateTimeMinValueUtc).ToString(_catalogDateTimeFormat), packageDetails[CatalogConstants.LastEdited].Value()); - Assert.Equal(lastEntry.ODataPackage.Listed, packageDetails[CatalogConstants.Listed].Value()); - Assert.Equal(lastEntry.ODataPackage.Hash, packageDetails[CatalogConstants.PackageHash].Value()); + Assert.Equal(packageOperation.ODataPackage.Listed, packageDetails[CatalogConstants.Listed].Value()); + Assert.Equal(packageOperation.ODataPackage.Hash, packageDetails[CatalogConstants.PackageHash].Value()); Assert.Equal(Constants.Sha512, packageDetails[CatalogConstants.PackageHashAlgorithm].Value()); - Assert.Equal(lastEntry.Package.Stream.Length, packageDetails[CatalogConstants.PackageSize].Value()); + Assert.Equal(packageOperation.Package.Stream.Length, packageDetails[CatalogConstants.PackageSize].Value()); Assert.Equal( - lastEntry.ODataPackage.Published.ToString(_catalogDateTimeFormat), + packageOperation.ODataPackage.Published.ToString(_catalogDateTimeFormat), packageDetails[CatalogConstants.Published].Value()); - Assert.Equal(lastEntry.Package.Version.ToFullString(), packageDetails[CatalogConstants.VerbatimVersion].Value()); - Assert.Equal(lastEntry.Package.Version.ToNormalizedString(), packageDetails[CatalogConstants.Version].Value()); + Assert.Equal(packageOperation.Package.Version.ToFullString(), packageDetails[CatalogConstants.VerbatimVersion].Value()); + Assert.Equal(packageOperation.Package.Version.ToNormalizedString(), packageDetails[CatalogConstants.Version].Value()); - var expectedPackageEntries = GetPackageEntries(lastEntry.Package) + var expectedPackageEntries = GetPackageEntries(packageOperation.Package) .OrderBy(entry => entry.FullName) .Select(entry => new JObject( @@ -1261,9 +1400,7 @@ private void VerifyCatalog(PackageOperation skippedOperation) } private void VerifyCatalogPackageDelete( - string commitId, - string commitTimeStamp, - PackageDeletion lastEntry, + PackageDeletion packageOperation, Uri packageDeleteUri, JObject packageDelete) { @@ -1274,12 +1411,12 @@ private void VerifyCatalog(PackageOperation skippedOperation) Assert.Equal( new JArray(CatalogConstants.PackageDelete, CatalogConstants.CatalogPermalink), packageDelete[CatalogConstants.TypeKeyword]); - Assert.Equal(commitId, packageDelete[CatalogConstants.CatalogCommitId].Value()); - Assert.Equal(commitTimeStamp, packageDelete[CatalogConstants.CatalogCommitTimeStamp].Value()); - Assert.Equal(lastEntry.PackageIdentity.Id, packageDelete[CatalogConstants.Id].Value()); - Assert.Equal(lastEntry.PackageIdentity.Id, packageDelete[CatalogConstants.OriginalId].Value()); - Assert.Equal(lastEntry.Published.ToString(_catalogDateTimeFormat), packageDelete[CatalogConstants.Published].Value()); - Assert.Equal(lastEntry.PackageIdentity.Version.ToNormalizedString(), packageDelete[CatalogConstants.Version].Value()); + Assert.Equal(packageOperation.CommitId, packageDelete[CatalogConstants.CatalogCommitId].Value()); + Assert.Equal(packageOperation.CommitTimeStamp, packageDelete[CatalogConstants.CatalogCommitTimeStamp].Value()); + Assert.Equal(packageOperation.PackageIdentity.Id, packageDelete[CatalogConstants.Id].Value()); + Assert.Equal(packageOperation.PackageIdentity.Id, packageDelete[CatalogConstants.OriginalId].Value()); + Assert.Equal(packageOperation.Published.ToString(_catalogDateTimeFormat), packageDelete[CatalogConstants.Published].Value()); + Assert.Equal(packageOperation.PackageIdentity.Version.ToNormalizedString(), packageDelete[CatalogConstants.Version].Value()); var expectedContext = new JObject( new JProperty(CatalogConstants.VocabKeyword, CatalogConstants.NuGetSchemaUri), @@ -1508,6 +1645,7 @@ private abstract class PackageOperation { internal string CommitId { get; set; } internal string CommitTimeStamp { get; set; } + internal DateTime CommitTimeStampDateTime { get; set; } internal abstract PackageIdentity PackageIdentity { get; } } From 665f31512d85b9a3138fb6eb1f8b04ab292f08c5 Mon Sep 17 00:00:00 2001 From: Damon Tivel Date: Fri, 10 Aug 2018 14:43:59 -0700 Subject: [PATCH 02/15] V3: rename asynchronous IStorage methods (#336) Progress on https://github.com/NuGet/NuGetGallery/issues/6267. --- src/Catalog/CatalogWriterBase.cs | 16 ++--- src/Catalog/Dnx/DnxMaker.cs | 14 ++-- src/Catalog/DurableCursor.cs | 11 ++-- src/Catalog/Helpers/DeletionAuditEntry.cs | 4 +- src/Catalog/Helpers/FeedHelpers.cs | 2 +- src/Catalog/PackageCatalog.cs | 16 ++--- src/Catalog/Persistence/AggregateStorage.cs | 14 ++-- src/Catalog/Persistence/AzureStorage.cs | 2 +- src/Catalog/Persistence/FileStorage.cs | 5 +- src/Catalog/Persistence/IStorage.cs | 15 +++-- src/Catalog/Persistence/Storage.cs | 65 +++++++------------ src/Catalog/Registration/RecordingStorage.cs | 25 +++---- .../RegistrationMakerCatalogWriter.cs | 21 +++--- .../Registration/RegistrationPersistence.cs | 32 ++++----- src/Catalog/SingleGraphPersistence.cs | 9 +-- .../Status/PackageMonitoringStatusService.cs | 12 ++-- .../CatalogTests/Helpers/FeedHelpersTests.cs | 7 +- .../SortingGraphCollectorTests.cs | 6 +- tests/NgTests/AggregateStorageTests.cs | 20 +++--- tests/NgTests/DnxCatalogCollectorTests.cs | 26 ++++---- tests/NgTests/DnxMakerTests.cs | 10 +-- tests/NgTests/Feed2CatalogTests.cs | 4 +- tests/NgTests/Infrastructure/MemoryStorage.cs | 2 +- .../MockServerHttpClientHandlerExtensions.cs | 6 +- tests/NgTests/Package2CatalogJobTests.cs | 4 +- .../PackageMonitoringStatusServiceTests.cs | 22 +++---- .../NgTests/PackageTimestampMetadataTests.cs | 4 +- tests/NgTests/RegistrationCollectorTests.cs | 16 ++--- .../SearchIndexFromCatalogCollectorTests.cs | 4 +- 29 files changed, 191 insertions(+), 203 deletions(-) diff --git a/src/Catalog/CatalogWriterBase.cs b/src/Catalog/CatalogWriterBase.cs index 1c80c4203..7473bbe68 100644 --- a/src/Catalog/CatalogWriterBase.cs +++ b/src/Catalog/CatalogWriterBase.cs @@ -96,7 +96,7 @@ public virtual async Task> Commit(DateTime commitTimeStamp, IGr return newItemEntries.Keys.Select(s => new Uri(s)); } - async Task> SaveItems(Guid commitId, DateTime commitTimeStamp, CancellationToken cancellationToken) + private async Task> SaveItems(Guid commitId, DateTime commitTimeStamp, CancellationToken cancellationToken) { ConcurrentDictionary pageItems = new ConcurrentDictionary(); @@ -156,7 +156,7 @@ protected virtual ResourceSaveOperation CreateSaveOperationForItem(IStorage stor if (content != null) { - operation.SaveTask = storage.Save(resourceUri, content, cancellationToken); + operation.SaveTask = storage.SaveAsync(resourceUri, content, cancellationToken); } return operation; @@ -320,7 +320,7 @@ protected async Task SaveIndexResource(Uri resourceUri, Uri typeUri, Guid commit return entries; } - async Task SaveGraph(Uri resourceUri, IGraph graph, Uri typeUri, CancellationToken cancellationToken) + private async Task SaveGraph(Uri resourceUri, IGraph graph, Uri typeUri, CancellationToken cancellationToken) { if (GraphPersistence != null) { @@ -328,11 +328,11 @@ async Task SaveGraph(Uri resourceUri, IGraph graph, Uri typeUri, CancellationTok } else { - await Storage.Save(resourceUri, CreateIndexContent(graph, typeUri), cancellationToken); + await Storage.SaveAsync(resourceUri, CreateIndexContent(graph, typeUri), cancellationToken); } } - async Task LoadGraph(Uri resourceUri, CancellationToken cancellationToken) + private async Task LoadGraph(Uri resourceUri, CancellationToken cancellationToken) { if (GraphPersistence != null) { @@ -340,7 +340,7 @@ async Task LoadGraph(Uri resourceUri, CancellationToken cancellationToke } else { - return Utils.CreateGraph(await Storage.LoadString(resourceUri, cancellationToken)); + return Utils.CreateGraph(await Storage.LoadStringAsync(resourceUri, cancellationToken)); } } @@ -356,7 +356,7 @@ protected Uri CreatePageUri(Uri baseAddress, string relativeAddress) } } - void CheckScheme(Uri resourceUri, IGraph graph) + private void CheckScheme(Uri resourceUri, IGraph graph) { INode typePredicate = graph.CreateUriNode(Schema.Predicates.Type); @@ -378,4 +378,4 @@ void CheckScheme(Uri resourceUri, IGraph graph) } } } -} +} \ No newline at end of file diff --git a/src/Catalog/Dnx/DnxMaker.cs b/src/Catalog/Dnx/DnxMaker.cs index 2842dcff5..a5b3dc378 100644 --- a/src/Catalog/Dnx/DnxMaker.cs +++ b/src/Catalog/Dnx/DnxMaker.cs @@ -112,7 +112,7 @@ private async Task SaveNuspecAsync(Storage storage, string id, string versi var relativeAddress = GetRelativeAddressNuspec(id, version); var nuspecUri = new Uri(storage.BaseAddress, relativeAddress); - await storage.Save(nuspecUri, new StringStorageContent(nuspec, "text/xml", "max-age=120"), cancellationToken); + await storage.SaveAsync(nuspecUri, new StringStorageContent(nuspec, "text/xml", "max-age=120"), cancellationToken); return nuspecUri; } @@ -145,14 +145,14 @@ public async Task UpdatePackageVersionIndexAsync(string id, Action version.ToNormalizedString())), cancellationToken); + await storage.SaveAsync(resourceUri, CreateContent(result.Select(version => version.ToNormalizedString())), cancellationToken); } else { // Remove versions file if no versions are present if (storage.Exists(relativeAddress)) { - await storage.Delete(resourceUri, cancellationToken); + await storage.DeleteAsync(resourceUri, cancellationToken); } } } @@ -161,7 +161,7 @@ private async Task GetVersionsAsync(Storage storage, Cancellatio { var relativeAddress = "index.json"; var resourceUri = new Uri(storage.BaseAddress, relativeAddress); - var versions = GetVersions(await storage.LoadString(resourceUri, cancellationToken)); + var versions = GetVersions(await storage.LoadStringAsync(resourceUri, cancellationToken)); return new VersionsResult(relativeAddress, resourceUri, versions); } @@ -195,7 +195,7 @@ private StorageContent CreateContent(IEnumerable versions) private async Task SaveNupkgAsync(Stream nupkgStream, Storage storage, string id, string version, CancellationToken cancellationToken) { Uri nupkgUri = new Uri(storage.BaseAddress, GetRelativeAddressNupkg(id, version)); - await storage.Save(nupkgUri, new StreamStorageContent(nupkgStream, "application/octet-stream", "max-age=120"), cancellationToken); + await storage.SaveAsync(nupkgUri, new StreamStorageContent(nupkgStream, "application/octet-stream", "max-age=120"), cancellationToken); return nupkgUri; } @@ -205,7 +205,7 @@ private async Task DeleteNuspecAsync(Storage storage, string id, string version, Uri nuspecUri = new Uri(storage.BaseAddress, relativeAddress); if (storage.Exists(relativeAddress)) { - await storage.Delete(nuspecUri, cancellationToken); + await storage.DeleteAsync(nuspecUri, cancellationToken); } } @@ -215,7 +215,7 @@ private async Task DeleteNupkgAsync(Storage storage, string id, string version, Uri nupkgUri = new Uri(storage.BaseAddress, relativeAddress); if (storage.Exists(relativeAddress)) { - await storage.Delete(nupkgUri, cancellationToken); + await storage.DeleteAsync(nupkgUri, cancellationToken); } } diff --git a/src/Catalog/DurableCursor.cs b/src/Catalog/DurableCursor.cs index 55d8d99e5..b3b9e832e 100644 --- a/src/Catalog/DurableCursor.cs +++ b/src/Catalog/DurableCursor.cs @@ -1,10 +1,11 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using Newtonsoft.Json.Linq; -using NuGet.Services.Metadata.Catalog.Persistence; + using System; using System.Threading; using System.Threading.Tasks; +using Newtonsoft.Json.Linq; +using NuGet.Services.Metadata.Catalog.Persistence; namespace NuGet.Services.Metadata.Catalog { @@ -25,12 +26,12 @@ public override async Task Save(CancellationToken cancellationToken) { JObject obj = new JObject { { "value", Value.ToString("O") } }; StorageContent content = new StringStorageContent(obj.ToString(), "application/json", "no-store"); - await _storage.Save(_address, content, cancellationToken); + await _storage.SaveAsync(_address, content, cancellationToken); } public override async Task Load(CancellationToken cancellationToken) { - string json = await _storage.LoadString(_address, cancellationToken); + string json = await _storage.LoadStringAsync(_address, cancellationToken); if (json == null) { @@ -42,4 +43,4 @@ public override async Task Load(CancellationToken cancellationToken) Value = obj["value"].ToObject(); } } -} +} \ No newline at end of file diff --git a/src/Catalog/Helpers/DeletionAuditEntry.cs b/src/Catalog/Helpers/DeletionAuditEntry.cs index b2c3a712e..0d70e264e 100644 --- a/src/Catalog/Helpers/DeletionAuditEntry.cs +++ b/src/Catalog/Helpers/DeletionAuditEntry.cs @@ -52,7 +52,7 @@ public class DeletionAuditEntry { try { - return new DeletionAuditEntry(uri, await auditingStorage.LoadString(uri, cancellationToken)); + return new DeletionAuditEntry(uri, await auditingStorage.LoadStringAsync(uri, cancellationToken)); } catch (JsonReaderException) { @@ -220,7 +220,7 @@ private void InitValues() // Get all audit blobs (based on their filename which starts with a date that can be parsed). /// Filter on the and fields provided. var auditRecords = - (await auditingStorage.List(cancellationToken)).Where(filterAuditRecord); + (await auditingStorage.ListAsync(cancellationToken)).Where(filterAuditRecord); return (await Task.WhenAll( diff --git a/src/Catalog/Helpers/FeedHelpers.cs b/src/Catalog/Helpers/FeedHelpers.cs index 41ac59ea8..f8d67fb97 100644 --- a/src/Catalog/Helpers/FeedHelpers.cs +++ b/src/Catalog/Helpers/FeedHelpers.cs @@ -66,7 +66,7 @@ public static HttpClient CreateHttpClient(Func handlerFunc) var stopwatch = Stopwatch.StartNew(); var indexUri = storage.ResolveUri("index.json"); - var json = await storage.LoadString(indexUri, cancellationToken); + var json = await storage.LoadStringAsync(indexUri, cancellationToken); if (json != null) { diff --git a/src/Catalog/PackageCatalog.cs b/src/Catalog/PackageCatalog.cs index f9a8cd7c7..bdd86107f 100644 --- a/src/Catalog/PackageCatalog.cs +++ b/src/Catalog/PackageCatalog.cs @@ -1,13 +1,13 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; using System; using System.Globalization; using System.IO; using System.Threading; using System.Threading.Tasks; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; using VDS.RDF; namespace NuGet.Services.Metadata.Catalog @@ -21,8 +21,8 @@ public static IGraph CreateCommitMetadata(Uri indexUri, CommitMetadata commitMet if (commitMetadata.LastCreated != null) { graph.Assert( - graph.CreateUriNode(indexUri), - graph.CreateUriNode(Schema.Predicates.LastCreated), + graph.CreateUriNode(indexUri), + graph.CreateUriNode(Schema.Predicates.LastCreated), graph.CreateLiteralNode(commitMetadata.LastCreated.Value.ToString("O"), Schema.DataTypes.DateTime)); } if (commitMetadata.LastEdited != null) @@ -35,8 +35,8 @@ public static IGraph CreateCommitMetadata(Uri indexUri, CommitMetadata commitMet if (commitMetadata.LastDeleted != null) { graph.Assert( - graph.CreateUriNode(indexUri), - graph.CreateUriNode(Schema.Predicates.LastDeleted), + graph.CreateUriNode(indexUri), + graph.CreateUriNode(Schema.Predicates.LastDeleted), graph.CreateLiteralNode(commitMetadata.LastDeleted.Value.ToString("O"), Schema.DataTypes.DateTime)); } @@ -47,7 +47,7 @@ public static async Task ReadCommitMetadata(CatalogWriterBase wr { CommitMetadata commitMetadata = new CommitMetadata(); - string json = await writer.Storage.LoadString(writer.RootUri, cancellationToken); + string json = await writer.Storage.LoadStringAsync(writer.RootUri, cancellationToken); if (json != null) { @@ -77,4 +77,4 @@ public static async Task ReadCommitMetadata(CatalogWriterBase wr return null; } } -} +} \ No newline at end of file diff --git a/src/Catalog/Persistence/AggregateStorage.cs b/src/Catalog/Persistence/AggregateStorage.cs index 46e13d2e7..b786db82a 100644 --- a/src/Catalog/Persistence/AggregateStorage.cs +++ b/src/Catalog/Persistence/AggregateStorage.cs @@ -35,7 +35,7 @@ public class AggregateStorage : Storage protected override Task OnSave(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) { var tasks = new List(); - tasks.Add(_primaryStorage.Save(resourceUri, content, cancellationToken)); + tasks.Add(_primaryStorage.SaveAsync(resourceUri, content, cancellationToken)); foreach (var storage in _secondaryStorage) { @@ -52,7 +52,7 @@ protected override Task OnSave(Uri resourceUri, StorageContent content, Cancella secondaryResourceUri, content); } - tasks.Add(storage.Save(secondaryResourceUri, secondaryContent, cancellationToken)); + tasks.Add(storage.SaveAsync(secondaryResourceUri, secondaryContent, cancellationToken)); } return Task.WhenAll(tasks); @@ -60,20 +60,20 @@ protected override Task OnSave(Uri resourceUri, StorageContent content, Cancella protected override Task OnLoad(Uri resourceUri, CancellationToken cancellationToken) { - return _primaryStorage.Load(resourceUri, cancellationToken); + return _primaryStorage.LoadAsync(resourceUri, cancellationToken); } protected override Task OnDelete(Uri resourceUri, CancellationToken cancellationToken) { var tasks = new List(); - tasks.Add(_primaryStorage.Delete(resourceUri, cancellationToken)); + tasks.Add(_primaryStorage.DeleteAsync(resourceUri, cancellationToken)); foreach (var storage in _secondaryStorage) { var secondaryResourceUri = new Uri(resourceUri.ToString() .Replace(_primaryStorage.BaseAddress.ToString(), storage.BaseAddress.ToString())); - tasks.Add(storage.Delete(secondaryResourceUri, cancellationToken)); + tasks.Add(storage.DeleteAsync(secondaryResourceUri, cancellationToken)); } return Task.WhenAll(tasks); @@ -84,9 +84,9 @@ public override bool Exists(string fileName) return _primaryStorage.Exists(fileName); } - public override Task> List(CancellationToken cancellationToken) + public override Task> ListAsync(CancellationToken cancellationToken) { - return _primaryStorage.List(cancellationToken); + return _primaryStorage.ListAsync(cancellationToken); } public override Uri GetUri(string name) diff --git a/src/Catalog/Persistence/AzureStorage.cs b/src/Catalog/Persistence/AzureStorage.cs index 8511d370f..228d44b70 100644 --- a/src/Catalog/Persistence/AzureStorage.cs +++ b/src/Catalog/Persistence/AzureStorage.cs @@ -105,7 +105,7 @@ public override bool Exists(string fileName) return false; } - public override async Task> List(CancellationToken cancellationToken) + public override async Task> ListAsync(CancellationToken cancellationToken) { var files = await _directory.ListBlobsAsync(cancellationToken); diff --git a/src/Catalog/Persistence/FileStorage.cs b/src/Catalog/Persistence/FileStorage.cs index 357b1ea88..050fec49b 100644 --- a/src/Catalog/Persistence/FileStorage.cs +++ b/src/Catalog/Persistence/FileStorage.cs @@ -1,5 +1,6 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + using System; using System.Collections.Generic; using System.IO; @@ -34,7 +35,7 @@ public override bool Exists(string fileName) return File.Exists(fileName); } - public override Task> List(CancellationToken cancellationToken) + public override Task> ListAsync(CancellationToken cancellationToken) { DirectoryInfo directoryInfo = new DirectoryInfo(Path); var files = directoryInfo.GetFiles("*", SearchOption.AllDirectories) @@ -155,4 +156,4 @@ protected override async Task OnDelete(Uri resourceUri, CancellationToken cancel } } } -} +} \ No newline at end of file diff --git a/src/Catalog/Persistence/IStorage.cs b/src/Catalog/Persistence/IStorage.cs index f46988b54..dcf7a0cff 100644 --- a/src/Catalog/Persistence/IStorage.cs +++ b/src/Catalog/Persistence/IStorage.cs @@ -1,7 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + using System; -using System.Collections; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; @@ -10,12 +10,13 @@ namespace NuGet.Services.Metadata.Catalog.Persistence { public interface IStorage { - Task Save(Uri resourceUri, StorageContent content, CancellationToken cancellationToken); - Task Load(Uri resourceUri, CancellationToken cancellationToken); - Task Delete(Uri resourceUri, CancellationToken cancellationToken); - Task LoadString(Uri resourceUri, CancellationToken cancellationToken); Uri BaseAddress { get; } + + Task DeleteAsync(Uri resourceUri, CancellationToken cancellationToken); + Task> ListAsync(CancellationToken cancellationToken); + Task LoadAsync(Uri resourceUri, CancellationToken cancellationToken); + Task LoadStringAsync(Uri resourceUri, CancellationToken cancellationToken); Uri ResolveUri(string relativeUri); - Task> List(CancellationToken cancellationToken); + Task SaveAsync(Uri resourceUri, StorageContent content, CancellationToken cancellationToken); } -} +} \ No newline at end of file diff --git a/src/Catalog/Persistence/Storage.cs b/src/Catalog/Persistence/Storage.cs index 9d0a62897..342abe50a 100644 --- a/src/Catalog/Persistence/Storage.cs +++ b/src/Catalog/Persistence/Storage.cs @@ -1,5 +1,6 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + using System; using System.Collections.Generic; using System.Diagnostics; @@ -14,6 +15,12 @@ namespace NuGet.Services.Metadata.Catalog.Persistence { public abstract class Storage : IStorage { + public Uri BaseAddress { get; protected set; } + public bool Verbose { get; set; } + public int SaveCount { get; protected set; } + public int LoadCount { get; protected set; } + public int DeleteCount { get; protected set; } + public Storage(Uri baseAddress) { string s = baseAddress.OriginalString.TrimEnd('/') + '/'; @@ -29,11 +36,11 @@ public override string ToString() protected abstract Task OnLoad(Uri resourceUri, CancellationToken cancellationToken); protected abstract Task OnDelete(Uri resourceUri, CancellationToken cancellationToken); - public async Task Save(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) + public async Task SaveAsync(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) { SaveCount++; - TraceMethod(nameof(Save), resourceUri); + TraceMethod(nameof(SaveAsync), resourceUri); Stopwatch sw = new Stopwatch(); sw.Start(); @@ -43,20 +50,20 @@ public async Task Save(Uri resourceUri, StorageContent content, CancellationToke } catch (Exception e) { - TraceException(nameof(Save), resourceUri, e); + TraceException(nameof(SaveAsync), resourceUri, e); throw; } sw.Stop(); - TraceExecutionTime(nameof(Save), resourceUri, sw.ElapsedMilliseconds); + TraceExecutionTime(nameof(SaveAsync), resourceUri, sw.ElapsedMilliseconds); } - public async Task Load(Uri resourceUri, CancellationToken cancellationToken) + public async Task LoadAsync(Uri resourceUri, CancellationToken cancellationToken) { LoadCount++; StorageContent storageContent = null; - TraceMethod(nameof(Load), resourceUri); + TraceMethod(nameof(LoadAsync), resourceUri); Stopwatch sw = new Stopwatch(); sw.Start(); @@ -66,20 +73,20 @@ public async Task Load(Uri resourceUri, CancellationToken cancel } catch (Exception e) { - TraceException(nameof(Load), resourceUri, e); + TraceException(nameof(LoadAsync), resourceUri, e); throw; } sw.Stop(); - TraceExecutionTime(nameof(Load), resourceUri, sw.ElapsedMilliseconds); + TraceExecutionTime(nameof(LoadAsync), resourceUri, sw.ElapsedMilliseconds); return storageContent; } - public async Task Delete(Uri resourceUri, CancellationToken cancellationToken) + public async Task DeleteAsync(Uri resourceUri, CancellationToken cancellationToken) { DeleteCount++; - TraceMethod(nameof(Delete), resourceUri); + TraceMethod(nameof(DeleteAsync), resourceUri); Stopwatch sw = new Stopwatch(); sw.Start(); @@ -105,17 +112,17 @@ public async Task Delete(Uri resourceUri, CancellationToken cancellationToken) } catch (Exception e) { - TraceException(nameof(Delete), resourceUri, e); + TraceException(nameof(DeleteAsync), resourceUri, e); throw; } sw.Stop(); - TraceExecutionTime(nameof(Delete), resourceUri, sw.ElapsedMilliseconds); + TraceExecutionTime(nameof(DeleteAsync), resourceUri, sw.ElapsedMilliseconds); } - public async Task LoadString(Uri resourceUri, CancellationToken cancellationToken) + public async Task LoadStringAsync(Uri resourceUri, CancellationToken cancellationToken) { - StorageContent content = await Load(resourceUri, cancellationToken); + StorageContent content = await LoadAsync(resourceUri, cancellationToken); if (content == null) { return null; @@ -130,33 +137,9 @@ public async Task LoadString(Uri resourceUri, CancellationToken cancella } } - public Uri BaseAddress { get; protected set; } - public abstract bool Exists(string fileName); - public abstract Task> List(CancellationToken cancellationToken); + public abstract Task> ListAsync(CancellationToken cancellationToken); - public bool Verbose - { - get; - set; - } - - public int SaveCount - { - get; - protected set; - } - - public int LoadCount - { - get; - protected set; - } - - public int DeleteCount - { - get; - protected set; - } + public abstract bool Exists(string fileName); public void ResetStatistics() { @@ -235,4 +218,4 @@ private void TraceExecutionTime(string method, Uri resourceUri, long executionTi Trace.WriteLine(message); } } -} +} \ No newline at end of file diff --git a/src/Catalog/Registration/RecordingStorage.cs b/src/Catalog/Registration/RecordingStorage.cs index e5d11e90e..29b7ec391 100644 --- a/src/Catalog/Registration/RecordingStorage.cs +++ b/src/Catalog/Registration/RecordingStorage.cs @@ -1,10 +1,11 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using NuGet.Services.Metadata.Catalog.Persistence; + using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using NuGet.Services.Metadata.Catalog.Persistence; namespace NuGet.Services.Metadata.Catalog.Registration { @@ -23,28 +24,28 @@ public RecordingStorage(IStorage storage) public HashSet Loaded { get; private set; } public HashSet Saved { get; private set; } - public Task Save(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) + public Task SaveAsync(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) { - Task result = _innerStorage.Save(resourceUri, content, cancellationToken); + Task result = _innerStorage.SaveAsync(resourceUri, content, cancellationToken); Saved.Add(resourceUri); return result; } - public Task Load(Uri resourceUri, CancellationToken cancellationToken) + public Task LoadAsync(Uri resourceUri, CancellationToken cancellationToken) { - Task result = _innerStorage.Load(resourceUri, cancellationToken); + Task result = _innerStorage.LoadAsync(resourceUri, cancellationToken); Loaded.Add(resourceUri); return result; } - public Task Delete(Uri resourceUri, CancellationToken cancellationToken) + public Task DeleteAsync(Uri resourceUri, CancellationToken cancellationToken) { - return _innerStorage.Delete(resourceUri, cancellationToken); + return _innerStorage.DeleteAsync(resourceUri, cancellationToken); } - public Task LoadString(Uri resourceUri, CancellationToken cancellationToken) + public Task LoadStringAsync(Uri resourceUri, CancellationToken cancellationToken) { - Task result = _innerStorage.LoadString(resourceUri, cancellationToken); + Task result = _innerStorage.LoadStringAsync(resourceUri, cancellationToken); Loaded.Add(resourceUri); return result; } @@ -59,9 +60,9 @@ public Uri ResolveUri(string relativeUri) return _innerStorage.ResolveUri(relativeUri); } - public Task> List(CancellationToken cancellationToken) + public Task> ListAsync(CancellationToken cancellationToken) { - return _innerStorage.List(cancellationToken); + return _innerStorage.ListAsync(cancellationToken); } } -} +} \ No newline at end of file diff --git a/src/Catalog/Registration/RegistrationMakerCatalogWriter.cs b/src/Catalog/Registration/RegistrationMakerCatalogWriter.cs index 80a8f7ed8..3068e8d86 100644 --- a/src/Catalog/Registration/RegistrationMakerCatalogWriter.cs +++ b/src/Catalog/Registration/RegistrationMakerCatalogWriter.cs @@ -1,21 +1,22 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using Newtonsoft.Json.Linq; -using NuGet.Services.Metadata.Catalog.Persistence; -using NuGet.Versioning; + using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; +using Newtonsoft.Json.Linq; +using NuGet.Services.Metadata.Catalog.Persistence; +using NuGet.Versioning; using VDS.RDF; namespace NuGet.Services.Metadata.Catalog.Registration { public class RegistrationMakerCatalogWriter : CatalogWriterBase { - IList _cleanUpList; + private readonly IList _cleanUpList; public RegistrationMakerCatalogWriter(IStorage storage, int partitionSize = 100, IList cleanUpList = null, ICatalogGraphPersistence graphPersistence = null, CatalogContext context = null) : base(storage, graphPersistence, context) @@ -93,7 +94,7 @@ protected override ResourceSaveOperation CreateSaveOperationForItem(IStorage sto if (!registrationMakerCatalogItem.IsExistingItem && content != null) { - saveOperation.SaveTask = storage.Save(resourceUri, content, cancellationToken); + saveOperation.SaveTask = storage.SaveAsync(resourceUri, content, cancellationToken); } else { @@ -106,7 +107,7 @@ protected override ResourceSaveOperation CreateSaveOperationForItem(IStorage sto return base.CreateSaveOperationForItem(storage, context, item, cancellationToken); } - async Task> PartitionAndSavePages(Guid commitId, DateTime commitTimeStamp, SortedDictionary> versions, CancellationToken cancellationToken) + private async Task> PartitionAndSavePages(Guid commitId, DateTime commitTimeStamp, SortedDictionary> versions, CancellationToken cancellationToken) { IDictionary newPageEntries = new Dictionary(); @@ -133,7 +134,7 @@ protected override ResourceSaveOperation CreateSaveOperationForItem(IStorage sto return newPageEntries; } - static IGraph CreateExtraGraph(Uri pageUri, string lower, string upper) + private static IGraph CreateExtraGraph(Uri pageUri, string lower, string upper) { IGraph graph = new Graph(); INode resourceNode = graph.CreateUriNode(pageUri); @@ -142,7 +143,7 @@ static IGraph CreateExtraGraph(Uri pageUri, string lower, string upper) return graph; } - static NuGetVersion GetPackageVersion(Uri packageUri, IGraph pageContent) + private static NuGetVersion GetPackageVersion(Uri packageUri, IGraph pageContent) { Triple t1 = pageContent.GetTriplesWithSubjectPredicate( pageContent.CreateUriNode(packageUri), @@ -156,7 +157,7 @@ static NuGetVersion GetPackageVersion(Uri packageUri, IGraph pageContent) return NuGetVersion.Parse(s); } - static IGraph CreatePageSummary(Uri newPageUri, string lower, string upper) + private static IGraph CreatePageSummary(Uri newPageUri, string lower, string upper) { IGraph graph = new Graph(); @@ -174,4 +175,4 @@ protected override StorageContent CreateIndexContent(IGraph graph, Uri type) return new JTokenStorageContent(Utils.CreateJson(graph, frame), "application/json", "no-store"); } } -} +} \ No newline at end of file diff --git a/src/Catalog/Registration/RegistrationPersistence.cs b/src/Catalog/Registration/RegistrationPersistence.cs index 8ab136569..0dc3c8a35 100644 --- a/src/Catalog/Registration/RegistrationPersistence.cs +++ b/src/Catalog/Registration/RegistrationPersistence.cs @@ -48,7 +48,7 @@ public async Task Save(IDictionary> Load(IStorage storage, Uri resourceUri, CancellationToken cancellationToken) + private static async Task> Load(IStorage storage, Uri resourceUri, CancellationToken cancellationToken) { Trace.TraceInformation("RegistrationPersistence.Load: resourceUri = {0}", resourceUri); @@ -61,7 +61,7 @@ public async Task Save(IDictionary GetResources(IGraph graph) + private static IDictionary GetResources(IGraph graph) { IDictionary resources = new Dictionary(); @@ -78,7 +78,7 @@ public async Task Save(IDictionary ListExistingItems(TripleStore store) + private static IList ListExistingItems(TripleStore store) { string sparql = Utils.GetResource("sparql.SelectInlinePackage.rq"); @@ -96,7 +96,7 @@ static IList ListExistingItems(TripleStore store) return results; } - static void AddExistingItem(IDictionary resources, TripleStore store, Uri catalogEntry) + private static void AddExistingItem(IDictionary resources, TripleStore store, Uri catalogEntry) { Trace.TraceInformation("RegistrationPersistence.AddExistingItem: catalogEntry = {0}", catalogEntry); @@ -113,11 +113,11 @@ static void AddExistingItem(IDictionary LoadCatalog(IStorage storage, Uri resourceUri, CancellationToken cancellationToken) + private static async Task LoadCatalog(IStorage storage, Uri resourceUri, CancellationToken cancellationToken) { Trace.TraceInformation("RegistrationPersistence.LoadCatalog: resourceUri = {0}", resourceUri); - string json = await storage.LoadString(resourceUri, cancellationToken); + string json = await storage.LoadStringAsync(resourceUri, cancellationToken); IGraph graph = Utils.CreateGraph(json); @@ -151,16 +151,16 @@ static async Task LoadCatalog(IStorage storage, Uri resourceUri, Cancell return graph; } - static async Task LoadCatalogPage(IStorage storage, Uri pageUri, CancellationToken cancellationToken) + private static async Task LoadCatalogPage(IStorage storage, Uri pageUri, CancellationToken cancellationToken) { Trace.TraceInformation("RegistrationPersistence.LoadCatalogPage: pageUri = {0}", pageUri); - string json = await storage.LoadString(pageUri, cancellationToken); + string json = await storage.LoadStringAsync(pageUri, cancellationToken); IGraph graph = Utils.CreateGraph(json); return graph; } // Save implementation - static async Task Save(IStorage storage, Uri registrationBaseAddress, IDictionary registration, int partitionSize, int packageCountThreshold, Uri contentBaseAddress, CancellationToken cancellationToken) + private static async Task Save(IStorage storage, Uri registrationBaseAddress, IDictionary registration, int partitionSize, int packageCountThreshold, Uri contentBaseAddress, CancellationToken cancellationToken) { Trace.TraceInformation("RegistrationPersistence.Save"); @@ -181,7 +181,7 @@ static async Task Save(IStorage storage, Uri registrationBaseAddress, IDictionar } } - static async Task SaveSmallRegistration(IStorage storage, Uri registrationBaseAddress, IList items, int partitionSize, Uri contentBaseAddress, CancellationToken cancellationToken) + private static async Task SaveSmallRegistration(IStorage storage, Uri registrationBaseAddress, IList items, int partitionSize, Uri contentBaseAddress, CancellationToken cancellationToken) { Trace.TraceInformation("RegistrationPersistence.SaveSmallRegistration"); @@ -195,10 +195,10 @@ static async Task SaveSmallRegistration(IStorage storage, Uri registrationBaseAd JObject frame = (new CatalogContext()).GetJsonLdContext("context.Registration.json", graphPersistence.TypeUri); StorageContent content = new JTokenStorageContent(Utils.CreateJson(graphPersistence.Graph, frame), "application/json", "no-store"); - await storage.Save(graphPersistence.ResourceUri, content, cancellationToken); + await storage.SaveAsync(graphPersistence.ResourceUri, content, cancellationToken); } - static async Task SaveLargeRegistration(IStorage storage, Uri registrationBaseAddress, IList items, int partitionSize, Uri contentBaseAddress, CancellationToken cancellationToken) + private static async Task SaveLargeRegistration(IStorage storage, Uri registrationBaseAddress, IList items, int partitionSize, Uri contentBaseAddress, CancellationToken cancellationToken) { Trace.TraceInformation("RegistrationPersistence.SaveLargeRegistration: registrationBaseAddress = {0} items: {1}", registrationBaseAddress, items.Count); @@ -207,7 +207,7 @@ static async Task SaveLargeRegistration(IStorage storage, Uri registrationBaseAd await SaveRegistration(storage, registrationBaseAddress, items, cleanUpList, null, partitionSize, contentBaseAddress, cancellationToken); } - static async Task SaveRegistration(IStorage storage, Uri registrationBaseAddress, IList items, IList cleanUpList, SingleGraphPersistence graphPersistence, int partitionSize, Uri contentBaseAddress, CancellationToken cancellationToken) + private static async Task SaveRegistration(IStorage storage, Uri registrationBaseAddress, IList items, IList cleanUpList, SingleGraphPersistence graphPersistence, int partitionSize, Uri contentBaseAddress, CancellationToken cancellationToken) { Trace.TraceInformation("RegistrationPersistence.SaveRegistration: registrationBaseAddress = {0} items: {1}", registrationBaseAddress, items.Count); @@ -221,7 +221,7 @@ static async Task SaveRegistration(IStorage storage, Uri registrationBaseAddress } } - static async Task Cleanup(RecordingStorage storage, CancellationToken cancellationToken) + private static async Task Cleanup(RecordingStorage storage, CancellationToken cancellationToken) { Trace.TraceInformation("RegistrationPersistence.Cleanup"); @@ -230,7 +230,7 @@ static async Task Cleanup(RecordingStorage storage, CancellationToken cancellati { if (!storage.Saved.Contains(loaded)) { - tasks.Add(storage.Delete(loaded, cancellationToken)); + tasks.Add(storage.DeleteAsync(loaded, cancellationToken)); } } if (tasks.Count > 0) @@ -239,4 +239,4 @@ static async Task Cleanup(RecordingStorage storage, CancellationToken cancellati } } } -} +} \ No newline at end of file diff --git a/src/Catalog/SingleGraphPersistence.cs b/src/Catalog/SingleGraphPersistence.cs index e9592c5c0..51d011a16 100644 --- a/src/Catalog/SingleGraphPersistence.cs +++ b/src/Catalog/SingleGraphPersistence.cs @@ -1,10 +1,11 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using Newtonsoft.Json.Linq; -using NuGet.Services.Metadata.Catalog.Persistence; + using System; using System.Threading; using System.Threading.Tasks; +using Newtonsoft.Json.Linq; +using NuGet.Services.Metadata.Catalog.Persistence; using VDS.RDF; namespace NuGet.Services.Metadata.Catalog @@ -35,7 +36,7 @@ public async Task Initialize(CancellationToken cancellationToken) { Uri rootUri = _storage.ResolveUri("index.json"); - string json = await _storage.LoadString(rootUri, cancellationToken); + string json = await _storage.LoadStringAsync(rootUri, cancellationToken); if (json != null) { @@ -70,4 +71,4 @@ public Uri CreatePageUri(Uri baseAddress, string relativeAddress) return new Uri(_storage.BaseAddress, "index.json#" + relativeAddress); } } -} +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/PackageMonitoringStatusService.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/PackageMonitoringStatusService.cs index 0eea668cf..df347b6ec 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/PackageMonitoringStatusService.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/PackageMonitoringStatusService.cs @@ -54,7 +54,7 @@ public async Task> ListAsync(Cancel private async Task> GetListItems(string state, CancellationToken token) { var storage = GetStorage(state); - var list = await storage.List(token); + var list = await storage.ListAsync(token); return list.Select(item => { try @@ -106,7 +106,7 @@ public async Task> GetAsync(PackageState st var storage = GetStorage(state); var statusTasks = - (await storage.List(token)) + (await storage.ListAsync(token)) .Select(listItem => GetPackageAsync(storage, listItem.Uri, token)) .ToList(); @@ -141,7 +141,7 @@ private async Task SaveAsync(PackageMonitoringStatus status, CancellationToken t var packageUri = GetPackageUri(storage, status.Package); - await storage.Save(packageUri, storageContent, token); + await storage.SaveAsync(packageUri, storageContent, token); } private async Task DeleteAsync(FeedPackageIdentity package, PackageState state, CancellationToken token) @@ -152,7 +152,7 @@ private async Task DeleteAsync(FeedPackageIdentity package, PackageState state, return; } - await storage.Delete(GetPackageUri(storage, package), token); + await storage.DeleteAsync(GetPackageUri(storage, package), token); } private CatalogStorage GetStorage(PackageState state) @@ -218,7 +218,7 @@ private async Task GetPackageAsync(CatalogStorage stora { try { - var content = await storage.Load(packageUri, token); + var content = await storage.LoadAsync(packageUri, token); string statusString = null; using (var stream = content.GetContentStream()) { @@ -258,4 +258,4 @@ private async Task GetPackageAsync(CatalogStorage stora } } } -} +} \ No newline at end of file diff --git a/tests/CatalogTests/Helpers/FeedHelpersTests.cs b/tests/CatalogTests/Helpers/FeedHelpersTests.cs index 8bdc3efa1..436649398 100644 --- a/tests/CatalogTests/Helpers/FeedHelpersTests.cs +++ b/tests/CatalogTests/Helpers/FeedHelpersTests.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; -using System.IO; using System.Linq; using System.Net; using System.Net.Http; @@ -490,7 +489,7 @@ public async Task DownloadMetadata2CatalogAsync_WithOnePackage_UpdatesStorage(bo var blobs = new List(); - test.Storage.Setup(x => x.Save( + test.Storage.Setup(x => x.SaveAsync( It.IsNotNull(), It.IsNotNull(), It.IsAny())) @@ -500,7 +499,7 @@ public async Task DownloadMetadata2CatalogAsync_WithOnePackage_UpdatesStorage(bo }) .Returns(Task.FromResult(0)); - test.Storage.Setup(x => x.LoadString( + test.Storage.Setup(x => x.LoadStringAsync( It.Is(uri => uri == test.CatalogIndexUri), It.IsAny())) .ReturnsAsync(CatalogTestData.GetBeforeIndex(test.CatalogIndexUri).ToString()); @@ -779,7 +778,7 @@ private static Mock CreateStorageMock(string json) storage.Setup(x => x.ResolveUri(It.IsNotNull())) .Returns(new Uri("https://unit.test")) .Verifiable(); - storage.Setup(x => x.LoadString(It.IsNotNull(), It.IsAny())) + storage.Setup(x => x.LoadStringAsync(It.IsNotNull(), It.IsAny())) .ReturnsAsync(json) .Verifiable(); diff --git a/tests/CatalogTests/SortingGraphCollectorTests.cs b/tests/CatalogTests/SortingGraphCollectorTests.cs index 5f635fb19..dbef0e98e 100644 --- a/tests/CatalogTests/SortingGraphCollectorTests.cs +++ b/tests/CatalogTests/SortingGraphCollectorTests.cs @@ -41,7 +41,7 @@ public async Task AllGraphsAreReadOnly() Initialize(); var catalogStorage = Catalogs.CreateTestCatalogWithCommitThenTwoPackageCommit(); - await _mockServer.AddStorage(catalogStorage); + await _mockServer.AddStorageAsync(catalogStorage); // Act var result = await _collector.Run(CancellationToken.None); @@ -65,7 +65,7 @@ public async Task GraphsAreBatchedById() Initialize(); var catalogStorage = Catalogs.CreateTestCatalogWithCommitThenTwoPackageCommit(); - await _mockServer.AddStorage(catalogStorage); + await _mockServer.AddStorageAsync(catalogStorage); // Act var result = await _collector.Run(CancellationToken.None); @@ -98,4 +98,4 @@ protected override Task ProcessGraphs(KeyValuePair _catalogToDnxStorage.WithName(name)); - await _catalogToDnxStorage.Save( + await _catalogToDnxStorage.SaveAsync( new Uri("http://tempuri.org/listedpackage/index.json"), new StringStorageContent(GetExpectedIndexJsonContent("1.0.1")), CancellationToken.None); @@ -416,7 +416,7 @@ public async Task Run_WhenPackageIsAlreadySynchronized_SkipsPackage() }; var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete(); - await _mockServer.AddStorage(catalogStorage); + await _mockServer.AddStorageAsync(catalogStorage); _mockServer.SetAction( "/packages/listedpackage.1.0.1.nupkg", @@ -468,7 +468,7 @@ public async Task Run_WhenPackageIsAlreadySynchronizedButNotInIndex_ProcessesPac var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete(); - await _mockServer.AddStorage(catalogStorage); + await _mockServer.AddStorageAsync(catalogStorage); _mockServer.SetAction( "/packages/listedpackage.1.0.1.nupkg", @@ -503,7 +503,7 @@ public async Task Run_WhenDownloadingPackage_RejectsUnexpectedHttpStatusCode(Htt { var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete(); - await _mockServer.AddStorage(catalogStorage); + await _mockServer.AddStorageAsync(catalogStorage); _mockServer.Return404OnUnknownAction = true; @@ -527,7 +527,7 @@ public async Task Run_WhenDownloadingPackage_OnlyDownloadsNupkgOncePerCatalogLea { // Arrange var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete(); - await _mockServer.AddStorage(catalogStorage); + await _mockServer.AddStorageAsync(catalogStorage); _mockServer.SetAction( "/packages/listedpackage.1.0.0.nupkg", @@ -577,7 +577,7 @@ public async Task Run_WhenExceptionOccurs_DoesNotSkipPackage(string catalogUri, { // Arrange var catalogStorage = Catalogs.CreateTestCatalogWithCommitThenTwoPackageCommit(); - await _mockServer.AddStorage(catalogStorage); + await _mockServer.AddStorageAsync(catalogStorage); _mockServer.SetAction( "/packages/unlistedpackage.1.0.0.nupkg", @@ -638,7 +638,7 @@ public async Task Run_WhenMultipleEntriesWithSamePackageIdentityInSameBatch_Thro var expectedNupkg = GetStreamBytes(nupkgStream); var catalogStorage = Catalogs.CreateTestCatalogWithMultipleEntriesWithSamePackageIdentityInSameBatch(); - await _mockServer.AddStorage(catalogStorage); + await _mockServer.AddStorageAsync(catalogStorage); _mockServer.SetAction( "/packages/listedpackage.1.0.0.nupkg", diff --git a/tests/NgTests/DnxMakerTests.cs b/tests/NgTests/DnxMakerTests.cs index 354647dbe..6fff9092b 100644 --- a/tests/NgTests/DnxMakerTests.cs +++ b/tests/NgTests/DnxMakerTests.cs @@ -407,7 +407,7 @@ public async Task UpdatePackageVersionIndexAsync_WithValidVersion_CreatesIndex(s var storageForPackage = (MemoryStorage)catalogToDnxStorageFactory.Create(_packageId); var indexJsonUri = new Uri(storageForPackage.BaseAddress, "index.json"); - var indexJson = await storageForPackage.Load(indexJsonUri, CancellationToken.None); + var indexJson = await storageForPackage.LoadAsync(indexJsonUri, CancellationToken.None); var indexObject = JObject.Parse(indexJson.GetContentString()); var versions = indexObject["versions"].ToObject(); var expectedContent = GetExpectedIndexJsonContent(normalizedVersion); @@ -433,7 +433,7 @@ public async Task UpdatePackageVersionIndexAsync_WhenLastVersionRemoved_RemovesI var storageForPackage = (MemoryStorage)catalogToDnxStorageFactory.Create(_packageId); var indexJsonUri = new Uri(storageForPackage.BaseAddress, "index.json"); - var indexJson = await storageForPackage.Load(indexJsonUri, CancellationToken.None); + var indexJson = await storageForPackage.LoadAsync(indexJsonUri, CancellationToken.None); Assert.NotNull(indexJson); Assert.Equal(1, catalogToDnxStorage.Content.Count); @@ -441,7 +441,7 @@ public async Task UpdatePackageVersionIndexAsync_WhenLastVersionRemoved_RemovesI await maker.UpdatePackageVersionIndexAsync(_packageId, v => v.Remove(version), CancellationToken.None); - indexJson = await storageForPackage.Load(indexJsonUri, CancellationToken.None); + indexJson = await storageForPackage.LoadAsync(indexJsonUri, CancellationToken.None); Assert.Null(indexJson); Assert.Equal(0, catalogToDnxStorage.Content.Count); @@ -459,7 +459,7 @@ public async Task UpdatePackageVersionIndexAsync_WithNoVersions_DoesNotCreateInd var storageForPackage = (MemoryStorage)catalogToDnxStorageFactory.Create(_packageId); var indexJsonUri = new Uri(storageForPackage.BaseAddress, "index.json"); - var indexJson = await storageForPackage.Load(indexJsonUri, CancellationToken.None); + var indexJson = await storageForPackage.LoadAsync(indexJsonUri, CancellationToken.None); Assert.Null(indexJson); Assert.Equal(0, catalogToDnxStorage.Content.Count); @@ -486,7 +486,7 @@ public async Task UpdatePackageVersionIndexAsync_WithMultipleVersions_SortsVersi var storageForPackage = (MemoryStorage)catalogToDnxStorageFactory.Create(_packageId); var indexJsonUri = new Uri(storageForPackage.BaseAddress, "index.json"); - var indexJson = await storageForPackage.Load(indexJsonUri, CancellationToken.None); + var indexJson = await storageForPackage.LoadAsync(indexJsonUri, CancellationToken.None); var indexObject = JObject.Parse(indexJson.GetContentString()); var versions = indexObject["versions"].ToObject(); diff --git a/tests/NgTests/Feed2CatalogTests.cs b/tests/NgTests/Feed2CatalogTests.cs index 3fbce826b..91c6f0dad 100644 --- a/tests/NgTests/Feed2CatalogTests.cs +++ b/tests/NgTests/Feed2CatalogTests.cs @@ -780,7 +780,7 @@ public async Task RunInternal_CallsCatalogStorageLoadStringExactlyOnce() catalogStorage.Setup(x => x.ResolveUri(It.IsNotNull())) .Returns(new Uri(_feedBaseUri)); - catalogStorage.Setup(x => x.LoadString(It.IsNotNull(), It.IsAny())) + catalogStorage.Setup(x => x.LoadStringAsync(It.IsNotNull(), It.IsAny())) .ReturnsAsync(json); mockServer.SetAction("/", GetRootActionAsync); @@ -801,7 +801,7 @@ public async Task RunInternal_CallsCatalogStorageLoadStringExactlyOnce() await feed2catalogTestJob.RunOnce(CancellationToken.None); catalogStorage.Verify(x => x.ResolveUri(It.IsNotNull()), Times.AtLeastOnce()); - catalogStorage.Verify(x => x.LoadString(It.IsNotNull(), It.IsAny()), Times.Once()); + catalogStorage.Verify(x => x.LoadStringAsync(It.IsNotNull(), It.IsAny()), Times.Once()); } private static Task GetCreatedPackages(HttpRequestMessage request) diff --git a/tests/NgTests/Infrastructure/MemoryStorage.cs b/tests/NgTests/Infrastructure/MemoryStorage.cs index 5e2a1f594..e29bd8ecc 100644 --- a/tests/NgTests/Infrastructure/MemoryStorage.cs +++ b/tests/NgTests/Infrastructure/MemoryStorage.cs @@ -100,7 +100,7 @@ public override bool Exists(string fileName) return Content.Keys.Any(k => k.PathAndQuery.EndsWith(fileName)); } - public override Task> List(CancellationToken cancellationToken) + public override Task> ListAsync(CancellationToken cancellationToken) { return Task.FromResult(Content.Keys.AsEnumerable().Select(x => ListMock.ContainsKey(x) ? ListMock[x] : new StorageListItem(x, DateTime.UtcNow))); diff --git a/tests/NgTests/Infrastructure/MockServerHttpClientHandlerExtensions.cs b/tests/NgTests/Infrastructure/MockServerHttpClientHandlerExtensions.cs index 14b3e1939..f5eebdc4e 100644 --- a/tests/NgTests/Infrastructure/MockServerHttpClientHandlerExtensions.cs +++ b/tests/NgTests/Infrastructure/MockServerHttpClientHandlerExtensions.cs @@ -13,9 +13,9 @@ namespace NgTests.Infrastructure { public static class MockServerHttpClientHandlerExtensions { - public static async Task AddStorage(this MockServerHttpClientHandler handler, IStorage storage) + public static async Task AddStorageAsync(this MockServerHttpClientHandler handler, IStorage storage) { - var files = (await storage.List(CancellationToken.None)).Select(x => x.Uri); + var files = (await storage.ListAsync(CancellationToken.None)).Select(x => x.Uri); foreach (var file in files) { @@ -24,7 +24,7 @@ public static async Task AddStorage(this MockServerHttpClientHandler handler, IS handler.SetAction(relativeFileUrl, async message => { - var content = await storage.Load(storageFileUrl, CancellationToken.None); + var content = await storage.LoadAsync(storageFileUrl, CancellationToken.None); var response = new HttpResponseMessage(HttpStatusCode.OK); diff --git a/tests/NgTests/Package2CatalogJobTests.cs b/tests/NgTests/Package2CatalogJobTests.cs index bdf879d74..829cc0eb8 100644 --- a/tests/NgTests/Package2CatalogJobTests.cs +++ b/tests/NgTests/Package2CatalogJobTests.cs @@ -36,7 +36,7 @@ public async Task RunInternal_CallsCatalogStorageLoadStringExactlyOnce() catalogStorage.Setup(x => x.ResolveUri(It.IsNotNull())) .Returns(new Uri(_feedBaseUri)); - catalogStorage.Setup(x => x.LoadString(It.IsNotNull(), It.IsAny())) + catalogStorage.Setup(x => x.LoadStringAsync(It.IsNotNull(), It.IsAny())) .ReturnsAsync(json); messageHandler.SetAction("/", GetRootActionAsync); @@ -53,7 +53,7 @@ public async Task RunInternal_CallsCatalogStorageLoadStringExactlyOnce() await job.RunOnceAsync(CancellationToken.None); catalogStorage.Verify(x => x.ResolveUri(It.IsNotNull()), Times.AtLeastOnce()); - catalogStorage.Verify(x => x.LoadString(It.IsNotNull(), It.IsAny()), Times.Once()); + catalogStorage.Verify(x => x.LoadStringAsync(It.IsNotNull(), It.IsAny()), Times.Once()); } private static string GetPath(string packageId, string packageVersion) diff --git a/tests/NgTests/PackageMonitoringStatusServiceTests.cs b/tests/NgTests/PackageMonitoringStatusServiceTests.cs index c6cda020e..9db1ff76d 100644 --- a/tests/NgTests/PackageMonitoringStatusServiceTests.cs +++ b/tests/NgTests/PackageMonitoringStatusServiceTests.cs @@ -28,7 +28,7 @@ public class PackageMonitoringStatusServiceTests public PackageMonitoringStatusServiceTests() { Service = new PackageMonitoringStatusService( - new MemoryStorageFactory(), + new MemoryStorageFactory(), new Mock>().Object); } @@ -43,9 +43,9 @@ public async Task UpdateSavesNewStatus() // Arrange var feedPackageIdentity = new FeedPackageIdentity("howdy", "3.4.6"); var packageValidationResult = new PackageValidationResult( - new PackageIdentity(feedPackageIdentity.Id, new NuGetVersion(feedPackageIdentity.Version)), - null, - null, + new PackageIdentity(feedPackageIdentity.Id, new NuGetVersion(feedPackageIdentity.Version)), + null, + null, Enumerable.Empty()); var status = new PackageMonitoringStatus(packageValidationResult); @@ -89,7 +89,7 @@ public async Task UpdateDeletesOldStatuses() foreach (var state in Enum.GetNames(typeof(PackageState))) { var storage = storageFactory.Create(state.ToLowerInvariant()); - await storage.Save(storage.ResolveUri(packageFileName), new StringStorageContent("{}"), CancellationToken.None); + await storage.SaveAsync(storage.ResolveUri(packageFileName), new StringStorageContent("{}"), CancellationToken.None); Assert.True(storage.Exists(packageFileName)); } @@ -139,7 +139,7 @@ private static DeletionAuditEntry CreateDeletionAuditEntry(string id, string ver private static PackageMonitoringStatus CreateStatusWithPackageValidationResult(string packageId, string packageVersion, IEnumerable results) { var version = new NuGetVersion(packageVersion); - + var aggregateValidationResult = new DummyAggregateValidator(results).Validate(); var packageValidationResult = new PackageValidationResult( @@ -155,7 +155,7 @@ private static PackageMonitoringStatus CreateStatusWithPackageValidationResult(s CreateDeletionAuditEntry(packageId, packageVersion) }, new AggregateValidationResult[] { aggregateValidationResult }); - + return new PackageMonitoringStatus(packageValidationResult); } @@ -391,9 +391,9 @@ public async Task GetByPackageDeserializationException() var storageFactory = new MemoryStorageFactory(); var storage = storageFactory.Create(PackageState.Valid.ToString().ToLowerInvariant()); - await storage.Save( - storage.ResolveUri($"{desiredPackageId}/{desiredPackageId}.{desiredPackageVersion}.json"), - new StringStorageContent("this isn't json"), + await storage.SaveAsync( + storage.ResolveUri($"{desiredPackageId}/{desiredPackageId}.{desiredPackageVersion}.json"), + new StringStorageContent("this isn't json"), CancellationToken.None); var statusService = new PackageMonitoringStatusService( @@ -491,4 +491,4 @@ public async Task GetByState() AssertStatus); } } -} +} \ No newline at end of file diff --git a/tests/NgTests/PackageTimestampMetadataTests.cs b/tests/NgTests/PackageTimestampMetadataTests.cs index 38d5ce8ab..0a8a58f1b 100644 --- a/tests/NgTests/PackageTimestampMetadataTests.cs +++ b/tests/NgTests/PackageTimestampMetadataTests.cs @@ -134,9 +134,9 @@ private async Task CreateDummyClient(MemoryStorage catalogS var mockServer = new MockServerHttpClientHandler(); mockServer.SetAction("/", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK))); - await mockServer.AddStorage(catalogStorage); + await mockServer.AddStorageAsync(catalogStorage); return new CollectorHttpClient(mockServer); } } -} +} \ No newline at end of file diff --git a/tests/NgTests/RegistrationCollectorTests.cs b/tests/NgTests/RegistrationCollectorTests.cs index 1daeec8e9..8c18f8e2d 100644 --- a/tests/NgTests/RegistrationCollectorTests.cs +++ b/tests/NgTests/RegistrationCollectorTests.cs @@ -67,7 +67,7 @@ public async Task DoesNotSkipPackagesWhenExceptionOccurs(string catalogUri, stri SharedInit(useLegacy: true, useSemVer2: false); var catalogStorage = Catalogs.CreateTestCatalogWithCommitThenTwoPackageCommit(); - await _mockServer.AddStorage(catalogStorage); + await _mockServer.AddStorageAsync(catalogStorage); // Make the first request for a catalog leaf node fail. This will cause the registration collector // to fail the first time but pass the second time. @@ -131,7 +131,7 @@ public async Task CreatesRegistrationsAndRespectsDeletes() SharedInit(useLegacy: true, useSemVer2: false); var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete(); - await _mockServer.AddStorage(catalogStorage); + await _mockServer.AddStorageAsync(catalogStorage); ReadWriteCursor front = new DurableCursor(_legacyStorage.ResolveUri("cursor.json"), _legacyStorage, MemoryCursor.MinValue); ReadCursor back = MemoryCursor.CreateMax(); @@ -195,7 +195,7 @@ public async Task WhenPackageHasMultipleCommitsRespectsOrder(string pageContent) SharedInit(useLegacy: true, useSemVer2: false); var catalogStorage = Catalogs.CreateTestCatalogWithThreeItemsForSamePackage(pageContent); - await _mockServer.AddStorage(catalogStorage); + await _mockServer.AddStorageAsync(catalogStorage); ReadWriteCursor front = new DurableCursor(_legacyStorage.ResolveUri("cursor.json"), _legacyStorage, MemoryCursor.MinValue); ReadCursor back = MemoryCursor.CreateMax(); @@ -252,8 +252,8 @@ public async Task CreatesRegistrationsWithSemVer2() SharedInit(useLegacy: true, useSemVer2: true); var catalogStorage = Catalogs.CreateTestCatalogWithSemVer2Package(); - await _mockServer.AddStorage(catalogStorage); - + await _mockServer.AddStorageAsync(catalogStorage); + var front = new DurableCursor(_legacyStorage.ResolveUri("cursor.json"), _legacyStorage, MemoryCursor.MinValue); var back = MemoryCursor.CreateMax(); @@ -299,7 +299,7 @@ public async Task IgnoresSemVer2PackagesInLegacyStorageWhenSemVer2IsEnabled() SharedInit(useLegacy: true, useSemVer2: true); var catalogStorage = Catalogs.CreateTestCatalogWithSemVer2Package(); - await _mockServer.AddStorage(catalogStorage); + await _mockServer.AddStorageAsync(catalogStorage); var front = new DurableCursor(_legacyStorage.ResolveUri("cursor.json"), _legacyStorage, MemoryCursor.MinValue); var back = MemoryCursor.CreateMax(); @@ -332,7 +332,7 @@ public async Task PutsSemVer2PackagesInLegacyStorageWhenSemVer2IsDisabled() SharedInit(useLegacy: true, useSemVer2: false); var catalogStorage = Catalogs.CreateTestCatalogWithSemVer2Package(); - await _mockServer.AddStorage(catalogStorage); + await _mockServer.AddStorageAsync(catalogStorage); var front = new DurableCursor(_legacyStorage.ResolveUri("cursor.json"), _legacyStorage, MemoryCursor.MinValue); var back = MemoryCursor.CreateMax(); @@ -357,7 +357,7 @@ public async Task HandlesDeleteCatalogItemWithNonNormalizedVersion() SharedInit(useLegacy: true, useSemVer2: false); var catalogStorage = Catalogs.CreateTestCatalogWithNonNormalizedDelete(); - await _mockServer.AddStorage(catalogStorage); + await _mockServer.AddStorageAsync(catalogStorage); ReadWriteCursor front = new DurableCursor(_legacyStorage.ResolveUri("cursor.json"), _legacyStorage, MemoryCursor.MinValue); diff --git a/tests/NgTests/SearchIndexFromCatalogCollectorTests.cs b/tests/NgTests/SearchIndexFromCatalogCollectorTests.cs index a17ad3a06..f1c3d4fb8 100644 --- a/tests/NgTests/SearchIndexFromCatalogCollectorTests.cs +++ b/tests/NgTests/SearchIndexFromCatalogCollectorTests.cs @@ -37,7 +37,7 @@ public async Task DoesNotSkipPackagesWhenExceptionOccurs(string catalogUri, stri mockServer.SetAction("/", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK))); var catalogStorage = Catalogs.CreateTestCatalogWithCommitThenTwoPackageCommit(); - await mockServer.AddStorage(catalogStorage); + await mockServer.AddStorageAsync(catalogStorage); // Make the first request for a catalog leaf node fail. This will cause the registration collector // to fail the first time but pass the second time. @@ -123,4 +123,4 @@ private void FailFirstRequest(MockServerHttpClientHandler mockServer, string rel mockServer.SetAction(relativeUri, failFirst); } } -} +} \ No newline at end of file From d1c6f5f6089e81406e9fe7e13c63f963bcd6b0a0 Mon Sep 17 00:00:00 2001 From: Damon Tivel Date: Fri, 10 Aug 2018 14:44:37 -0700 Subject: [PATCH 03/15] Test: fix flaky basic search tests (#337) Progress on https://github.com/NuGet/NuGetGallery/issues/6292. --- .../AutocompleteFunctionalTests.cs | 2 +- .../NuGet.Services.BasicSearchTests.csproj | 1 + .../StartupFunctionalTests.cs | 3 ++- .../TestSupport/StartupTestCollection.cs | 13 +++++++++++++ .../V2SearchFunctionalTests.cs | 1 + .../V3SearchFunctionalTests.cs | 4 ++-- 6 files changed, 20 insertions(+), 4 deletions(-) create mode 100644 tests/NuGet.Services.BasicSearchTests/TestSupport/StartupTestCollection.cs diff --git a/tests/NuGet.Services.BasicSearchTests/AutocompleteFunctionalTests.cs b/tests/NuGet.Services.BasicSearchTests/AutocompleteFunctionalTests.cs index 5f968e06d..fc79d98e7 100644 --- a/tests/NuGet.Services.BasicSearchTests/AutocompleteFunctionalTests.cs +++ b/tests/NuGet.Services.BasicSearchTests/AutocompleteFunctionalTests.cs @@ -1,7 +1,6 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using System; using System.Net.Http; using System.Threading.Tasks; using NuGet.Services.BasicSearchTests.Models; @@ -10,6 +9,7 @@ namespace NuGet.Services.BasicSearchTests { + [Collection(StartupTestCollection.Name)] public class AutocompleteFunctionalTests { [Fact] diff --git a/tests/NuGet.Services.BasicSearchTests/NuGet.Services.BasicSearchTests.csproj b/tests/NuGet.Services.BasicSearchTests/NuGet.Services.BasicSearchTests.csproj index 5fd23ae8c..2656a1126 100644 --- a/tests/NuGet.Services.BasicSearchTests/NuGet.Services.BasicSearchTests.csproj +++ b/tests/NuGet.Services.BasicSearchTests/NuGet.Services.BasicSearchTests.csproj @@ -231,6 +231,7 @@ + diff --git a/tests/NuGet.Services.BasicSearchTests/StartupFunctionalTests.cs b/tests/NuGet.Services.BasicSearchTests/StartupFunctionalTests.cs index 1bf0179aa..42ea49f6c 100644 --- a/tests/NuGet.Services.BasicSearchTests/StartupFunctionalTests.cs +++ b/tests/NuGet.Services.BasicSearchTests/StartupFunctionalTests.cs @@ -8,6 +8,7 @@ namespace NuGet.Services.BasicSearchTests { + [Collection(StartupTestCollection.Name)] public class StartupFunctionalTests { [Fact] @@ -42,4 +43,4 @@ public async Task InvalidEndpoint() } } } -} +} \ No newline at end of file diff --git a/tests/NuGet.Services.BasicSearchTests/TestSupport/StartupTestCollection.cs b/tests/NuGet.Services.BasicSearchTests/TestSupport/StartupTestCollection.cs new file mode 100644 index 000000000..c559428ae --- /dev/null +++ b/tests/NuGet.Services.BasicSearchTests/TestSupport/StartupTestCollection.cs @@ -0,0 +1,13 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Xunit; + +namespace NuGet.Services.BasicSearchTests.TestSupport +{ + [CollectionDefinition(Name)] + public static class StartupTestCollection + { + public const string Name = "NuGet.Services.BasicSearch.Startup Test Collection"; + } +} \ No newline at end of file diff --git a/tests/NuGet.Services.BasicSearchTests/V2SearchFunctionalTests.cs b/tests/NuGet.Services.BasicSearchTests/V2SearchFunctionalTests.cs index 7cba0c3b6..2231bd92c 100644 --- a/tests/NuGet.Services.BasicSearchTests/V2SearchFunctionalTests.cs +++ b/tests/NuGet.Services.BasicSearchTests/V2SearchFunctionalTests.cs @@ -13,6 +13,7 @@ namespace NuGet.Services.BasicSearchTests { + [Collection(StartupTestCollection.Name)] public class V2SearchSkipFunctionalTests { [Fact] diff --git a/tests/NuGet.Services.BasicSearchTests/V3SearchFunctionalTests.cs b/tests/NuGet.Services.BasicSearchTests/V3SearchFunctionalTests.cs index 102eb87af..537e5fdc1 100644 --- a/tests/NuGet.Services.BasicSearchTests/V3SearchFunctionalTests.cs +++ b/tests/NuGet.Services.BasicSearchTests/V3SearchFunctionalTests.cs @@ -2,7 +2,6 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; -using System.Globalization; using System.IO; using System.Net.Http; using System.Threading.Tasks; @@ -14,6 +13,7 @@ namespace NuGet.Services.BasicSearchTests { + [Collection(StartupTestCollection.Name)] public class V3SearchFunctionalTests { [Fact] @@ -147,7 +147,7 @@ public async Task DoesNotIncludeUnlistedVersions() // Act var withPrereleaseResponse = await app.Client.GetAsync(new V3SearchBuilder { Query = query, Prerelease = true }.RequestUri); var withPrerelease = await withPrereleaseResponse.Content.ReadAsAsync(); - + // Assert Assert.Equal("3.1.3.42154", withPrerelease.GetPackageVersion("Antlr")); // the latest version is not prerelease Assert.False(withPrerelease.ContainsPackageVersion("Antlr", "3.4.1.9004-pre")); // the unlisted version is not in the result From 5741fe1c7c2183378e1d65463691a7487c4e41cb Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Wed, 15 Aug 2018 08:29:49 -0700 Subject: [PATCH 04/15] Add signing to search service (#338) Progress on https://github.com/NuGet/Engineering/issues/1644 --- src/Catalog/NuGet.Services.Metadata.Catalog.csproj | 6 ++++-- src/Ng/Ng.csproj | 6 ++++-- .../NuGet.ApplicationInsights.Owin.csproj | 6 ++++-- src/NuGet.Indexing/NuGet.Indexing.csproj | 6 ++++-- .../NuGet.Services.BasicSearch.csproj | 13 ++++++------- ...uGet.Services.Metadata.Catalog.Monitoring.csproj | 6 ++++-- .../NuGet.Services.Metadata.csproj | 13 ++++++------- src/Tools/MennoCopy/MennoCopy.csproj | 13 ++++++------- src/V3PerPackage/V3PerPackage.csproj | 6 ++++++ .../NuGet.IndexingTests/NuGet.IndexingTests.csproj | 6 ++++-- 10 files changed, 48 insertions(+), 33 deletions(-) diff --git a/src/Catalog/NuGet.Services.Metadata.Catalog.csproj b/src/Catalog/NuGet.Services.Metadata.Catalog.csproj index 2055e4873..a0d574452 100644 --- a/src/Catalog/NuGet.Services.Metadata.Catalog.csproj +++ b/src/Catalog/NuGet.Services.Metadata.Catalog.csproj @@ -319,7 +319,9 @@ - ..\..\build + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) - + \ No newline at end of file diff --git a/src/Ng/Ng.csproj b/src/Ng/Ng.csproj index c22aa9a82..20f84c4fe 100644 --- a/src/Ng/Ng.csproj +++ b/src/Ng/Ng.csproj @@ -414,9 +414,11 @@ - ..\..\build + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) - + diff --git a/src/NuGet.ApplicationInsights.Owin/NuGet.ApplicationInsights.Owin.csproj b/src/NuGet.ApplicationInsights.Owin/NuGet.ApplicationInsights.Owin.csproj index fb3bc4577..174f11091 100644 --- a/src/NuGet.ApplicationInsights.Owin/NuGet.ApplicationInsights.Owin.csproj +++ b/src/NuGet.ApplicationInsights.Owin/NuGet.ApplicationInsights.Owin.csproj @@ -81,8 +81,10 @@ - ..\..\build + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) - + \ No newline at end of file diff --git a/src/NuGet.Indexing/NuGet.Indexing.csproj b/src/NuGet.Indexing/NuGet.Indexing.csproj index 3d24cb6f2..744c15091 100644 --- a/src/NuGet.Indexing/NuGet.Indexing.csproj +++ b/src/NuGet.Indexing/NuGet.Indexing.csproj @@ -365,9 +365,11 @@ - ..\..\build + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) - + diff --git a/src/NuGet.Services.BasicSearch/NuGet.Services.BasicSearch.csproj b/src/NuGet.Services.BasicSearch/NuGet.Services.BasicSearch.csproj index ca65f44cc..10402eeb0 100644 --- a/src/NuGet.Services.BasicSearch/NuGet.Services.BasicSearch.csproj +++ b/src/NuGet.Services.BasicSearch/NuGet.Services.BasicSearch.csproj @@ -408,11 +408,10 @@ - + + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) + + \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj b/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj index 5be878f1c..b9643ef69 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj @@ -206,7 +206,9 @@ - ..\..\build + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) - + \ No newline at end of file diff --git a/src/NuGet.Services.Metadata/NuGet.Services.Metadata.csproj b/src/NuGet.Services.Metadata/NuGet.Services.Metadata.csproj index 2c22a0cec..375d78450 100644 --- a/src/NuGet.Services.Metadata/NuGet.Services.Metadata.csproj +++ b/src/NuGet.Services.Metadata/NuGet.Services.Metadata.csproj @@ -208,11 +208,10 @@ - + + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) + + \ No newline at end of file diff --git a/src/Tools/MennoCopy/MennoCopy.csproj b/src/Tools/MennoCopy/MennoCopy.csproj index cbef4e030..5bc9cd4c8 100644 --- a/src/Tools/MennoCopy/MennoCopy.csproj +++ b/src/Tools/MennoCopy/MennoCopy.csproj @@ -79,11 +79,10 @@ - + + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) + + \ No newline at end of file diff --git a/src/V3PerPackage/V3PerPackage.csproj b/src/V3PerPackage/V3PerPackage.csproj index 1a3a85743..a908a0550 100644 --- a/src/V3PerPackage/V3PerPackage.csproj +++ b/src/V3PerPackage/V3PerPackage.csproj @@ -134,4 +134,10 @@ + + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) + + \ No newline at end of file diff --git a/tests/NuGet.IndexingTests/NuGet.IndexingTests.csproj b/tests/NuGet.IndexingTests/NuGet.IndexingTests.csproj index 6fbdb4179..4f97c5e9e 100644 --- a/tests/NuGet.IndexingTests/NuGet.IndexingTests.csproj +++ b/tests/NuGet.IndexingTests/NuGet.IndexingTests.csproj @@ -222,9 +222,11 @@ - ..\..\build + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) - + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. From 1907ccb72ac1194f1b44cf6a804eb8ac3e2ae32f Mon Sep 17 00:00:00 2001 From: Damon Tivel Date: Mon, 20 Aug 2018 12:05:50 -0700 Subject: [PATCH 05/15] Test: add registration tests (#341) Progress on https://github.com/NuGet/NuGetGallery/issues/6317. --- .../Registration/RegistrationCatalogEntry.cs | 7 +- .../Registration/RegistrationCollector.cs | 10 +- src/Catalog/Registration/RegistrationKey.cs | 9 +- src/Catalog/Registration/RegistrationMaker.cs | 14 +- src/Catalog/SortingCollector.cs | 16 +- src/Catalog/SortingGraphCollector.cs | 25 +- src/Ng/Arguments.cs | 3 +- src/Ng/Jobs/Catalog2RegistrationJob.cs | 1 - src/Ng/Jobs/LightningJob.cs | 10 +- .../ValidationCollector.cs | 2 +- tests/CatalogTests/CatalogTests.csproj | 8 + .../Helpers/CatalogPackageDetails.cs | 202 ++++++++ .../Helpers/CatalogPackageEntry.cs | 41 ++ .../CatalogTests/Helpers/RegistrationIndex.cs | 46 ++ .../RegistrationIndexPackageDetails.cs | 45 ++ .../Helpers/RegistrationIndexPage.cs | 53 ++ .../Helpers/RegistrationPackage.cs | 50 ++ .../Helpers/RegistrationPackageDetails.cs | 89 ++++ .../Registration/RegistrationKeyTests.cs | 21 +- .../Registration/RegistrationMakerTests.cs | 459 ++++++++++++++++++ .../SortingGraphCollectorTests.cs | 8 +- tests/NgTests/CatalogConstants.cs | 164 ++++--- .../Infrastructure/MemoryStorageFactory.cs | 13 +- tests/NgTests/Infrastructure/TestPackage.cs | 17 +- tests/NgTests/Infrastructure/TestUtility.cs | 28 ++ tests/NgTests/NgTests.csproj | 1 + .../NgTests/SortingIdVersionCollectorTests.cs | 18 +- 27 files changed, 1213 insertions(+), 147 deletions(-) create mode 100644 tests/CatalogTests/Helpers/CatalogPackageDetails.cs create mode 100644 tests/CatalogTests/Helpers/CatalogPackageEntry.cs create mode 100644 tests/CatalogTests/Helpers/RegistrationIndex.cs create mode 100644 tests/CatalogTests/Helpers/RegistrationIndexPackageDetails.cs create mode 100644 tests/CatalogTests/Helpers/RegistrationIndexPage.cs create mode 100644 tests/CatalogTests/Helpers/RegistrationPackage.cs create mode 100644 tests/CatalogTests/Helpers/RegistrationPackageDetails.cs create mode 100644 tests/CatalogTests/Registration/RegistrationMakerTests.cs create mode 100644 tests/NgTests/Infrastructure/TestUtility.cs diff --git a/src/Catalog/Registration/RegistrationCatalogEntry.cs b/src/Catalog/Registration/RegistrationCatalogEntry.cs index a9b9a5a56..4c377cb5c 100644 --- a/src/Catalog/Registration/RegistrationCatalogEntry.cs +++ b/src/Catalog/Registration/RegistrationCatalogEntry.cs @@ -40,9 +40,10 @@ public RegistrationCatalogEntry(string resourceUri, IGraph graph, bool isExistin bool isExistingItem) { INode subject = graph.CreateUriNode(new Uri(resourceUri)); - string version = graph.GetTriplesWithSubjectPredicate(subject, graph.CreateUriNode(Schema.Predicates.Version)).First().Object.ToString(); + var triples = graph.GetTriplesWithSubjectPredicate(subject, graph.CreateUriNode(Schema.Predicates.Version)); + string version = triples.First().Object.ToString(); - RegistrationEntryKey registrationEntryKey = new RegistrationEntryKey(RegistrationKey.Promote(resourceUri, graph), version); + var registrationEntryKey = new RegistrationEntryKey(RegistrationKey.Promote(resourceUri, graph), version); RegistrationCatalogEntry registrationCatalogEntry = null; if (!IsDelete(subject, graph) && shouldInclude(registrationEntryKey, resourceUri, graph)) @@ -59,4 +60,4 @@ static bool IsDelete(INode subject, IGraph graph) || graph.ContainsTriple(new Triple(subject, graph.CreateUriNode(Schema.Predicates.Type), graph.CreateUriNode(Schema.DataTypes.PackageDelete))); } } -} +} \ No newline at end of file diff --git a/src/Catalog/Registration/RegistrationCollector.cs b/src/Catalog/Registration/RegistrationCollector.cs index 710515ca3..bf1ac3ad7 100644 --- a/src/Catalog/Registration/RegistrationCollector.cs +++ b/src/Catalog/Registration/RegistrationCollector.cs @@ -82,8 +82,8 @@ private string GetKey(JToken item) return item["nuget:id"].ToString(); } - protected override async Task ProcessGraphs( - KeyValuePair> sortedGraphs, + protected override async Task ProcessGraphsAsync( + KeyValuePair> sortedGraphs, CancellationToken cancellationToken) { var tasks = new List(); @@ -94,7 +94,7 @@ private string GetKey(JToken item) { TelemetryConstants.Id, sortedGraphs.Key.ToLowerInvariant() } })) { - var legacyTask = RegistrationMaker.Process( + var legacyTask = RegistrationMaker.ProcessAsync( registrationKey: new RegistrationKey(sortedGraphs.Key), newItems: sortedGraphs.Value, shouldInclude: _shouldIncludeSemVer2, @@ -108,7 +108,7 @@ private string GetKey(JToken item) if (_semVer2StorageFactory != null) { - var semVer2Task = RegistrationMaker.Process( + var semVer2Task = RegistrationMaker.ProcessAsync( registrationKey: new RegistrationKey(sortedGraphs.Key), newItems: sortedGraphs.Value, storageFactory: _semVer2StorageFactory, @@ -139,4 +139,4 @@ public static ShouldIncludeRegistrationPackage GetShouldIncludeRegistrationPacka } } } -} +} \ No newline at end of file diff --git a/src/Catalog/Registration/RegistrationKey.cs b/src/Catalog/Registration/RegistrationKey.cs index a62e77e46..e0d5e983c 100644 --- a/src/Catalog/Registration/RegistrationKey.cs +++ b/src/Catalog/Registration/RegistrationKey.cs @@ -1,5 +1,6 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + using System; using System.Linq; using VDS.RDF; @@ -10,11 +11,11 @@ public class RegistrationKey { public RegistrationKey(string id) { - Id = id; + Id = id ?? throw new ArgumentNullException(nameof(id)); ; } public string Id { get; } - + public override string ToString() { return Id.ToLowerInvariant(); @@ -34,7 +35,7 @@ public override bool Equals(object obj) return false; } - return StringComparer.OrdinalIgnoreCase.Equals(Id, rhs.Id); + return StringComparer.OrdinalIgnoreCase.Equals(Id, rhs.Id); } public static RegistrationKey Promote(string resourceUri, IGraph graph) @@ -45,4 +46,4 @@ public static RegistrationKey Promote(string resourceUri, IGraph graph) return new RegistrationKey(id); } } -} +} \ No newline at end of file diff --git a/src/Catalog/Registration/RegistrationMaker.cs b/src/Catalog/Registration/RegistrationMaker.cs index 503fc154b..75aa33379 100644 --- a/src/Catalog/Registration/RegistrationMaker.cs +++ b/src/Catalog/Registration/RegistrationMaker.cs @@ -13,9 +13,9 @@ namespace NuGet.Services.Metadata.Catalog.Registration { public static class RegistrationMaker { - public static async Task Process( + public static async Task ProcessAsync( RegistrationKey registrationKey, - IDictionary newItems, + IReadOnlyDictionary newItems, StorageFactory storageFactory, Uri contentBaseAddress, int partitionSize, @@ -23,7 +23,7 @@ public static class RegistrationMaker ITelemetryService telemetryService, CancellationToken cancellationToken) { - await Process( + await ProcessAsync( registrationKey, newItems, (k, u, g) => true, @@ -35,9 +35,9 @@ public static class RegistrationMaker cancellationToken); } - public static async Task Process( + public static async Task ProcessAsync( RegistrationKey registrationKey, - IDictionary newItems, + IReadOnlyDictionary newItems, ShouldIncludeRegistrationPackage shouldInclude, StorageFactory storageFactory, Uri contentBaseAddress, @@ -69,7 +69,7 @@ public static class RegistrationMaker } private static IDictionary PromoteRegistrationKey( - IDictionary newItems, + IReadOnlyDictionary newItems, ShouldIncludeRegistrationPackage shouldInclude) { IDictionary promoted = new Dictionary(); @@ -114,4 +114,4 @@ public static class RegistrationMaker return resulting; } } -} +} \ No newline at end of file diff --git a/src/Catalog/SortingCollector.cs b/src/Catalog/SortingCollector.cs index b6ad47fa5..e378f06bd 100644 --- a/src/Catalog/SortingCollector.cs +++ b/src/Catalog/SortingCollector.cs @@ -22,9 +22,9 @@ public SortingCollector(Uri index, ITelemetryService telemetryService, Func OnProcessBatch( - CollectorHttpClient client, + CollectorHttpClient client, IEnumerable items, - JToken context, + JToken context, DateTime commitTimeStamp, bool isLastBatch, CancellationToken cancellationToken) @@ -49,7 +49,7 @@ public SortingCollector(Uri index, ITelemetryService telemetryService, Func> sortedBatch in sortedItems) { - Task task = ProcessSortedBatch(client, sortedBatch, context, cancellationToken); + Task task = ProcessSortedBatchAsync(client, sortedBatch, context, cancellationToken); tasks.Add(task); @@ -66,10 +66,10 @@ public SortingCollector(Uri index, ITelemetryService telemetryService, Func> sortedBatch, - JToken context, + protected abstract Task ProcessSortedBatchAsync( + CollectorHttpClient client, + KeyValuePair> sortedBatch, + JToken context, CancellationToken cancellationToken); } -} +} \ No newline at end of file diff --git a/src/Catalog/SortingGraphCollector.cs b/src/Catalog/SortingGraphCollector.cs index 79c1e464a..77cf64ba7 100644 --- a/src/Catalog/SortingGraphCollector.cs +++ b/src/Catalog/SortingGraphCollector.cs @@ -14,29 +14,34 @@ namespace NuGet.Services.Metadata.Catalog { public abstract class SortingGraphCollector : SortingIdCollector { - Uri[] _types; + private readonly Uri[] _types; - public SortingGraphCollector(Uri index, Uri[] types, ITelemetryService telemetryService, Func handlerFunc = null) + public SortingGraphCollector( + Uri index, + Uri[] types, + ITelemetryService telemetryService, + Func handlerFunc = null) : base(index, telemetryService, handlerFunc) { _types = types; } - protected override async Task ProcessSortedBatch( + protected override async Task ProcessSortedBatchAsync( CollectorHttpClient client, KeyValuePair> sortedBatch, - JToken context, + JToken context, CancellationToken cancellationToken) { var graphs = new Dictionary(); var graphTasks = new Dictionary>(); - + foreach (var item in sortedBatch.Value) { if (Utils.IsType((JObject)context, item, _types)) { var itemUri = item["@id"].ToString(); + // Load package details from catalog. // Download the graph to a read-only container. This allows operations on each graph to be safely // parallelized. var task = client.GetGraphAsync(new Uri(itemUri), readOnly: true, token: cancellationToken); @@ -59,10 +64,14 @@ public SortingGraphCollector(Uri index, Uri[] types, ITelemetryService telemetry if (graphs.Count > 0) { - await ProcessGraphs(new KeyValuePair>(sortedBatch.Key, graphs), cancellationToken); + var sortedGraphs = new KeyValuePair>(sortedBatch.Key, graphs); + + await ProcessGraphsAsync(sortedGraphs, cancellationToken); } } - protected abstract Task ProcessGraphs(KeyValuePair> sortedGraphs, CancellationToken cancellationToken); + protected abstract Task ProcessGraphsAsync( + KeyValuePair> sortedGraphs, + CancellationToken cancellationToken); } -} +} \ No newline at end of file diff --git a/src/Ng/Arguments.cs b/src/Ng/Arguments.cs index 3b109cd69..d9b51a7c6 100644 --- a/src/Ng/Arguments.cs +++ b/src/Ng/Arguments.cs @@ -71,7 +71,6 @@ public static class Arguments public const string SemVer2StorageKeyValue = "semVer2StorageKeyValue"; public const string SemVer2StoragePath = "semVer2StoragePath"; - public const string UnlistShouldDelete = "unlistShouldDelete"; public const string UseCompressedStorage = "useCompressedStorage"; public const string UseSemVer2Storage = "useSemVer2Storage"; @@ -168,4 +167,4 @@ public static class Arguments public const string CursorFile = "cursorFile"; #endregion } -} +} \ No newline at end of file diff --git a/src/Ng/Jobs/Catalog2RegistrationJob.cs b/src/Ng/Jobs/Catalog2RegistrationJob.cs index 1214f34f9..0c3820652 100644 --- a/src/Ng/Jobs/Catalog2RegistrationJob.cs +++ b/src/Ng/Jobs/Catalog2RegistrationJob.cs @@ -67,7 +67,6 @@ public override string GetUsage() protected override void Init(IDictionary arguments, CancellationToken cancellationToken) { var source = arguments.GetOrThrow(Arguments.Source); - var unlistShouldDelete = arguments.GetOrDefault(Arguments.UnlistShouldDelete, false); var verbose = arguments.GetOrDefault(Arguments.Verbose, false); var contentBaseAddress = arguments.GetOrDefault(Arguments.ContentBaseAddress); diff --git a/src/Ng/Jobs/LightningJob.cs b/src/Ng/Jobs/LightningJob.cs index 7f78d9d99..1d9ec9f8e 100644 --- a/src/Ng/Jobs/LightningJob.cs +++ b/src/Ng/Jobs/LightningJob.cs @@ -291,7 +291,7 @@ private async Task PrepareAsync() // Write command files _log.WriteLine("Start preparing lightning reindex command files..."); - + string templateFileContents; using (var templateStreamReader = new StreamReader(_templateFile)) { @@ -333,7 +333,7 @@ private async Task PrepareAsync() //the not required arguments need to be added only if they were passed in //they cannot be hardcoded in the template string optionalArguments = string.Empty; - if(_arguments.ContainsKey(Arguments.StorageSuffix)) + if (_arguments.ContainsKey(Arguments.StorageSuffix)) { optionalArguments = optionalArguments + $" -{Arguments.StorageSuffix} {_arguments[Arguments.StorageSuffix]}"; } @@ -475,9 +475,9 @@ private static Task UpdateCursorFileAsync(string cursorFileName, int startIndex, return Task.FromResult(true); } - private async Task ProcessGraphsAsync(string packageId, IDictionary sortedGraphs) + private async Task ProcessGraphsAsync(string packageId, IReadOnlyDictionary sortedGraphs) { - await RegistrationMaker.Process( + await RegistrationMaker.ProcessAsync( new RegistrationKey(packageId), sortedGraphs, _shouldIncludeSemVer2, @@ -491,7 +491,7 @@ private async Task ProcessGraphsAsync(string packageId, IDictionary> sortedBatch, JToken context, CancellationToken cancellationToken) + protected override async Task ProcessSortedBatchAsync(CollectorHttpClient client, KeyValuePair> sortedBatch, JToken context, CancellationToken cancellationToken) { var packageId = sortedBatch.Key.Id; var packageVersion = sortedBatch.Key.Version; diff --git a/tests/CatalogTests/CatalogTests.csproj b/tests/CatalogTests/CatalogTests.csproj index 65031e8cc..799bf08b6 100644 --- a/tests/CatalogTests/CatalogTests.csproj +++ b/tests/CatalogTests/CatalogTests.csproj @@ -187,11 +187,18 @@ + + + + + + + @@ -203,6 +210,7 @@ + diff --git a/tests/CatalogTests/Helpers/CatalogPackageDetails.cs b/tests/CatalogTests/Helpers/CatalogPackageDetails.cs new file mode 100644 index 000000000..a74f88362 --- /dev/null +++ b/tests/CatalogTests/Helpers/CatalogPackageDetails.cs @@ -0,0 +1,202 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Security.Cryptography; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NgTests; +using NgTests.Infrastructure; + +namespace CatalogTests.Helpers +{ + internal sealed class CatalogPackageDetails + { + private static readonly JObject _context = JObject.Parse( +@"{ + ""@vocab"": ""http://schema.nuget.org/schema#"", + ""catalog"": ""http://schema.nuget.org/catalog#"", + ""xsd"": ""http://www.w3.org/2001/XMLSchema#"", + ""dependencies"": { + ""@id"": ""dependency"", + ""@container"": ""@set"" + }, + ""dependencyGroups"": { + ""@id"": ""dependencyGroup"", + ""@container"": ""@set"" + }, + ""packageEntries"": { + ""@id"": ""packageEntry"", + ""@container"": ""@set"" + }, + ""supportedFrameworks"": { + ""@id"": ""supportedFramework"", + ""@container"": ""@set"" + }, + ""tags"": { + ""@id"": ""tag"", + ""@container"": ""@set"" + }, + ""published"": { + ""@type"": ""xsd:dateTime"" + }, + ""created"": { + ""@type"": ""xsd:dateTime"" + }, + ""lastEdited"": { + ""@type"": ""xsd:dateTime"" + }, + ""catalog:commitTimeStamp"": { + ""@type"": ""xsd:dateTime"" + } + }"); + + [JsonProperty(CatalogConstants.IdKeyword)] + internal string IdKeyword { get; } + [JsonProperty(CatalogConstants.TypeKeyword)] + internal string[] TypeKeyword { get; } + [JsonProperty(CatalogConstants.Authors)] + internal string Authors { get; } + [JsonProperty(CatalogConstants.CatalogCommitId)] + internal string CommitId { get; } + [JsonProperty(CatalogConstants.CatalogCommitTimeStamp)] + internal string CommitTimeStamp { get; } + [JsonProperty(CatalogConstants.Created)] + internal string Created { get; } + [JsonProperty(CatalogConstants.Description)] + internal string Description { get; } + [JsonProperty(CatalogConstants.Id)] + internal string Id { get; } + [JsonProperty(CatalogConstants.IsPrerelease)] + internal bool IsPrerelease { get; } + [JsonProperty(CatalogConstants.LastEdited)] + internal string LastEdited { get; } + [JsonProperty(CatalogConstants.Listed)] + internal bool Listed { get; } + [JsonProperty(CatalogConstants.PackageHash)] + internal string PackageHash { get; } + [JsonProperty(CatalogConstants.PackageHashAlgorithm)] + internal string PackageHashAlgorithm { get; } + [JsonProperty(CatalogConstants.PackageSize)] + internal int PackageSize { get; } + [JsonProperty(CatalogConstants.Published)] + internal string Published { get; } + [JsonProperty(CatalogConstants.RequireLicenseAcceptance)] + internal bool RequireLicenseAcceptance { get; } + [JsonProperty(CatalogConstants.VerbatimVersion)] + internal string VerbatimVersion { get; } + [JsonProperty(CatalogConstants.Version)] + internal string Version { get; } + [JsonProperty(CatalogConstants.PackageEntries)] + internal CatalogPackageEntry[] PackageEntries { get; } + [JsonProperty(CatalogConstants.ContextKeyword)] + internal JObject ContextKeyword { get; } + + internal CatalogPackageDetails(string id = null, string version = null) + { + var now = DateTimeOffset.UtcNow; + + Id = id ?? TestUtility.CreateRandomAlphanumericString(); + + var build = (int)(now.Ticks & 0xff); // random number + + VerbatimVersion = version ?? $"1.0.{build}"; + Version = version ?? VerbatimVersion; + + IdKeyword = $"https://nuget.test/v3-catalog0/data/{now.ToString("yyyy.MM.dd.HH.mm.ss")}/{Id}.{Version}.json"; + TypeKeyword = new[] { CatalogConstants.PackageDetails, CatalogConstants.CatalogPermalink }; + Authors = TestUtility.CreateRandomAlphanumericString(); + CommitId = Guid.NewGuid().ToString("D"); + + const string datetimeFormat = "yyyy-MM-ddTHH:mm:ss.FFFZ"; + + CommitTimeStamp = now.ToString("yyyy-MM-ddTHH:mm:ss.FFFFFFFZ"); + Created = now.AddHours(-2).ToString(datetimeFormat); + Description = TestUtility.CreateRandomAlphanumericString(); + LastEdited = now.AddHours(-1).ToString(datetimeFormat); + Listed = true; + PackageHash = CreateFakePackageHash(); + PackageHashAlgorithm = TestUtility.CreateRandomAlphanumericString(); + PackageSize = (int)(now.Ticks & 0xffffff); // random number + Published = Created; + RequireLicenseAcceptance = now.Ticks % 2 == 0; + + PackageEntries = new[] + { + new CatalogPackageEntry( + idKeyword: $"{IdKeyword}#{Id}.nuspec", + typeKeyword: CatalogConstants.PackageEntry, + compressedLength: (int)(now.Ticks & 0xffff) + 1, + fullName: $"{Id}.nuspec", + length: (int)(now.Ticks & 0xfff) + 1, + name: $"{Id}.nuspec"), + new CatalogPackageEntry( + idKeyword: $"{IdKeyword}#.signature.p7s", + typeKeyword: CatalogConstants.PackageEntry, + compressedLength: (int)(now.Ticks & 0xffff), + fullName: ".signature.p7s", + length: (int)(now.Ticks & 0xfff), + name: ".signature.p7s") + }; + + ContextKeyword = _context; + } + + [JsonConstructor] + internal CatalogPackageDetails( + string idKeyword, + string[] typeKeyword, + string authors, + string commitId, + string commitTimeStamp, + string created, + string description, + string id, + bool isPrerelease, + string lastEdited, + bool listed, + string packageHash, + string packageHashAlgorithm, + int packageSize, + string published, + bool requireLicenseAcceptance, + string verbatimVersion, + string version, + CatalogPackageEntry[] packageEntries, + JObject contextKeyword) + { + IdKeyword = idKeyword; + TypeKeyword = typeKeyword; + Authors = authors; + CommitId = commitId; + CommitTimeStamp = commitTimeStamp; + Created = created; + Description = description; + Id = id; + IsPrerelease = isPrerelease; + LastEdited = lastEdited; + Listed = listed; + PackageHash = packageHash; + PackageHashAlgorithm = packageHashAlgorithm; + PackageSize = packageSize; + Published = published; + RequireLicenseAcceptance = requireLicenseAcceptance; + VerbatimVersion = verbatimVersion; + Version = version; + PackageEntries = packageEntries; + ContextKeyword = contextKeyword; + } + + private static string CreateFakePackageHash() + { + using (var rng = RandomNumberGenerator.Create()) + { + var bytes = new byte[64]; + + rng.GetBytes(bytes); + + return Convert.ToBase64String(bytes); + } + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Helpers/CatalogPackageEntry.cs b/tests/CatalogTests/Helpers/CatalogPackageEntry.cs new file mode 100644 index 000000000..b14c5eca5 --- /dev/null +++ b/tests/CatalogTests/Helpers/CatalogPackageEntry.cs @@ -0,0 +1,41 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; +using NgTests; + +namespace CatalogTests.Helpers +{ + internal sealed class CatalogPackageEntry + { + [JsonProperty(CatalogConstants.IdKeyword)] + internal string IdKeyword { get; } + [JsonProperty(CatalogConstants.TypeKeyword)] + internal string TypeKeyword { get; } + [JsonProperty(CatalogConstants.CompressedLength)] + internal long CompressedLength { get; } + [JsonProperty(CatalogConstants.FullName)] + internal string FullName { get; } + [JsonProperty(CatalogConstants.Length)] + internal long Length { get; } + [JsonProperty(CatalogConstants.Name)] + internal string Name { get; } + + [JsonConstructor] + internal CatalogPackageEntry( + string idKeyword, + string typeKeyword, + long compressedLength, + string fullName, + long length, + string name) + { + IdKeyword = idKeyword; + TypeKeyword = typeKeyword; + CompressedLength = compressedLength; + FullName = fullName; + Length = length; + Name = name; + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Helpers/RegistrationIndex.cs b/tests/CatalogTests/Helpers/RegistrationIndex.cs new file mode 100644 index 000000000..7d478ce90 --- /dev/null +++ b/tests/CatalogTests/Helpers/RegistrationIndex.cs @@ -0,0 +1,46 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NgTests; + +namespace CatalogTests.Helpers +{ + internal sealed class RegistrationIndex + { + [JsonProperty(CatalogConstants.IdKeyword)] + internal string IdKeyword { get; } + [JsonProperty(CatalogConstants.TypeKeyword)] + internal string[] TypeKeyword { get; } + [JsonProperty(CatalogConstants.CommitId)] + internal string CommitId { get; } + [JsonProperty(CatalogConstants.CommitTimeStamp)] + internal string CommitTimeStamp { get; } + [JsonProperty(CatalogConstants.Count)] + internal int Count { get; } + [JsonProperty(CatalogConstants.Items)] + internal RegistrationIndexPage[] Items { get; } + [JsonProperty(CatalogConstants.ContextKeyword)] + internal JObject ContextKeyword { get; } + + [JsonConstructor] + internal RegistrationIndex( + string idKeyword, + string[] typeKeyword, + string commitId, + string commitTimeStamp, + int count, + RegistrationIndexPage[] items, + JObject contextKeyword) + { + IdKeyword = idKeyword; + TypeKeyword = typeKeyword; + CommitId = commitId; + CommitTimeStamp = commitTimeStamp; + Count = count; + Items = items; + ContextKeyword = contextKeyword; + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Helpers/RegistrationIndexPackageDetails.cs b/tests/CatalogTests/Helpers/RegistrationIndexPackageDetails.cs new file mode 100644 index 000000000..0a17e37ee --- /dev/null +++ b/tests/CatalogTests/Helpers/RegistrationIndexPackageDetails.cs @@ -0,0 +1,45 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; +using NgTests; + +namespace CatalogTests.Helpers +{ + internal sealed class RegistrationIndexPackageDetails + { + [JsonProperty(CatalogConstants.IdKeyword)] + internal string IdKeyword { get; } + [JsonProperty(CatalogConstants.TypeKeyword)] + internal string TypeKeyword { get; } + [JsonProperty(CatalogConstants.CommitId)] + internal string CommitId { get; } + [JsonProperty(CatalogConstants.CommitTimeStamp)] + internal string CommitTimeStamp { get; } + [JsonProperty(CatalogConstants.CatalogEntry)] + internal RegistrationPackageDetails CatalogEntry { get; } + [JsonProperty(CatalogConstants.PackageContent)] + internal string PackageContent { get; } + [JsonProperty(CatalogConstants.Registration)] + internal string Registration { get; } + + [JsonConstructor] + internal RegistrationIndexPackageDetails( + string idKeyword, + string typeKeyword, + string commitId, + string commitTimeStamp, + RegistrationPackageDetails catalogEntry, + string packageContent, + string registration) + { + IdKeyword = idKeyword; + TypeKeyword = typeKeyword; + CommitId = commitId; + CommitTimeStamp = commitTimeStamp; + CatalogEntry = catalogEntry; + PackageContent = packageContent; + Registration = registration; + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Helpers/RegistrationIndexPage.cs b/tests/CatalogTests/Helpers/RegistrationIndexPage.cs new file mode 100644 index 000000000..f6cca2d61 --- /dev/null +++ b/tests/CatalogTests/Helpers/RegistrationIndexPage.cs @@ -0,0 +1,53 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; +using NgTests; + +namespace CatalogTests.Helpers +{ + internal sealed class RegistrationIndexPage + { + [JsonProperty(CatalogConstants.IdKeyword)] + internal string IdKeyword { get; } + [JsonProperty(CatalogConstants.TypeKeyword)] + internal string TypeKeyword { get; } + [JsonProperty(CatalogConstants.CommitId)] + internal string CommitId { get; } + [JsonProperty(CatalogConstants.CommitTimeStamp)] + internal string CommitTimeStamp { get; } + [JsonProperty(CatalogConstants.Count)] + internal int Count { get; } + [JsonProperty(CatalogConstants.Items)] + internal RegistrationIndexPackageDetails[] Items { get; } + [JsonProperty(CatalogConstants.Parent)] + internal string Parent { get; } + [JsonProperty(CatalogConstants.Lower)] + internal string Lower { get; } + [JsonProperty(CatalogConstants.Upper)] + internal string Upper { get; } + + [JsonConstructor] + internal RegistrationIndexPage( + string idKeyword, + string typeKeyword, + string commitId, + string commitTimeStamp, + int count, + RegistrationIndexPackageDetails[] items, + string parent, + string lower, + string upper) + { + IdKeyword = idKeyword; + TypeKeyword = typeKeyword; + CommitId = commitId; + CommitTimeStamp = commitTimeStamp; + Count = count; + Items = items; + Parent = parent; + Lower = lower; + Upper = upper; + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Helpers/RegistrationPackage.cs b/tests/CatalogTests/Helpers/RegistrationPackage.cs new file mode 100644 index 000000000..d5ba456f8 --- /dev/null +++ b/tests/CatalogTests/Helpers/RegistrationPackage.cs @@ -0,0 +1,50 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NgTests; + +namespace CatalogTests.Helpers +{ + internal sealed class RegistrationPackage + { + [JsonProperty(CatalogConstants.IdKeyword)] + internal string IdKeyword { get; } + [JsonProperty(CatalogConstants.TypeKeyword)] + internal string[] TypeKeyword { get; } + [JsonProperty(CatalogConstants.CatalogEntry)] + internal string CatalogEntry { get; } + [JsonProperty(CatalogConstants.Listed)] + internal bool Listed { get; } + [JsonProperty(CatalogConstants.PackageContent)] + internal string PackageContent { get; } + [JsonProperty(CatalogConstants.Published)] + internal string Published { get; } + [JsonProperty(CatalogConstants.Registration)] + internal string Registration { get; } + [JsonProperty(CatalogConstants.ContextKeyword)] + internal JObject ContextKeyword { get; } + + [JsonConstructor] + internal RegistrationPackage( + string idKeyword, + string[] typeKeyword, + string catalogEntry, + bool listed, + string packageContent, + string published, + string registration, + JObject contextKeyword) + { + IdKeyword = idKeyword; + TypeKeyword = typeKeyword; + CatalogEntry = catalogEntry; + Listed = listed; + PackageContent = packageContent; + Published = published; + Registration = registration; + ContextKeyword = contextKeyword; + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Helpers/RegistrationPackageDetails.cs b/tests/CatalogTests/Helpers/RegistrationPackageDetails.cs new file mode 100644 index 000000000..6e4bd82f5 --- /dev/null +++ b/tests/CatalogTests/Helpers/RegistrationPackageDetails.cs @@ -0,0 +1,89 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; +using NgTests; + +namespace CatalogTests.Helpers +{ + internal sealed class RegistrationPackageDetails + { + [JsonProperty(CatalogConstants.IdKeyword)] + internal string IdKeyword { get; } + [JsonProperty(CatalogConstants.TypeKeyword)] + internal string TypeKeyword { get; } + [JsonProperty(CatalogConstants.Authors)] + internal string Authors { get; } + [JsonProperty(CatalogConstants.Description)] + internal string Description { get; } + [JsonProperty(CatalogConstants.IconUrl)] + internal string IconUrl { get; } + [JsonProperty(CatalogConstants.Id)] + internal string Id { get; } + [JsonProperty(CatalogConstants.Language)] + internal string Language { get; } + [JsonProperty(CatalogConstants.LicenseUrl)] + internal string LicenseUrl { get; } + [JsonProperty(CatalogConstants.Listed)] + internal bool Listed { get; } + [JsonProperty(CatalogConstants.MinClientVersion)] + internal string MinClientVersion { get; } + [JsonProperty(CatalogConstants.PackageContent)] + internal string PackageContent { get; } + [JsonProperty(CatalogConstants.ProjectUrl)] + internal string ProjectUrl { get; } + [JsonProperty(CatalogConstants.Published)] + internal string Published { get; } + [JsonProperty(CatalogConstants.RequireLicenseAcceptance)] + internal bool RequireLicenseAcceptance { get; } + [JsonProperty(CatalogConstants.Summary)] + internal string Summary { get; } + [JsonProperty(CatalogConstants.Tags)] + internal string[] Tags { get; } + [JsonProperty(CatalogConstants.Title)] + internal string Title { get; } + [JsonProperty(CatalogConstants.Version)] + internal string Version { get; } + + [JsonConstructor] + internal RegistrationPackageDetails( + string idKeyword, + string typeKeyword, + string authors, + string description, + string iconUrl, + string id, + string language, + string licenseUrl, + bool listed, + string minClientVersion, + string packageContent, + string projectUrl, + string published, + bool requireLicenseAcceptance, + string summary, + string[] tags, + string title, + string version) + { + IdKeyword = idKeyword; + TypeKeyword = typeKeyword; + Authors = authors; + Description = description; + IconUrl = iconUrl; + Id = id; + Language = language; + LicenseUrl = licenseUrl; + Listed = listed; + MinClientVersion = minClientVersion; + PackageContent = packageContent; + ProjectUrl = projectUrl; + Published = published; + RequireLicenseAcceptance = requireLicenseAcceptance; + Summary = summary; + Tags = tags; + Title = title; + Version = version; + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Registration/RegistrationKeyTests.cs b/tests/CatalogTests/Registration/RegistrationKeyTests.cs index ea6a8a4f7..fc71a467a 100644 --- a/tests/CatalogTests/Registration/RegistrationKeyTests.cs +++ b/tests/CatalogTests/Registration/RegistrationKeyTests.cs @@ -1,6 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; using System.Collections.Generic; using System.Linq; using NuGet.Services.Metadata.Catalog.Registration; @@ -10,6 +11,14 @@ namespace CatalogTests.Registration { public class RegistrationKeyTests { + [Fact] + public void Constructor_WhenIdIsNull_Throws() + { + var exception = Assert.Throws(() => new RegistrationKey(id: null)); + + Assert.Equal("id", exception.ParamName); + } + [Theory] [MemberData(nameof(RegistrationKeys))] public void EqualsIgnoresCase(string idA, string idB, bool expectedEquals) @@ -43,6 +52,16 @@ public void GetHashCodeIgnoresCaseAndUsesNormalized(string idA, string idB, bool Assert.Equal(hashCodeA, hashCodeB); } + [Theory] + [InlineData("a")] + [InlineData("A")] + public void ToString_Always_ReturnsLowerCase(string id) + { + var key = new RegistrationKey(id); + + Assert.Equal(id.ToLowerInvariant(), key.ToString()); + } + public static IEnumerable RegistrationKeys { get @@ -55,4 +74,4 @@ public static IEnumerable RegistrationKeys public static IEnumerable EqualRegistrationKeys => RegistrationKeys.Where(a => (bool)a[2]); } -} +} \ No newline at end of file diff --git a/tests/CatalogTests/Registration/RegistrationMakerTests.cs b/tests/CatalogTests/Registration/RegistrationMakerTests.cs new file mode 100644 index 000000000..250a10c3e --- /dev/null +++ b/tests/CatalogTests/Registration/RegistrationMakerTests.cs @@ -0,0 +1,459 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using CatalogTests.Helpers; +using Moq; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NgTests; +using NgTests.Infrastructure; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Persistence; +using NuGet.Services.Metadata.Catalog.Registration; +using NuGet.Versioning; +using VDS.RDF; +using Xunit; + +namespace CatalogTests.Registration +{ + public class RegistrationMakerTests + { + private const string _cacheControl = "no-store"; + private const string _contentType = "application/json"; + private const int _partitionSize = 1; + private const int _packageCountThreshold = 128; + + private static readonly Uri _contentBaseAddress = new Uri("https://nuget.test/"); + private static readonly JsonSerializerSettings _jsonSettings = new JsonSerializerSettings() + { + DateParseHandling = DateParseHandling.None, + NullValueHandling = NullValueHandling.Ignore + }; + + private readonly MemoryStorageFactory _storageFactory = new MemoryStorageFactory(new Uri("https://nuget.test/v3-registration3/")); + private readonly Mock _telemetryService = new Mock(); + + public RegistrationMakerTests() + { + RegistrationMakerCatalogItem.PackagePathProvider = new PackagesFolderPackagePathProvider(); + } + + [Fact] + public async Task ProcessAsync_WithEmptyStorageAndEmptyNewItems_DoesNotCreateAnything() + { + var emptyItems = new Dictionary(); + var storage = await ProcessAsync(emptyItems, packageId: null); + + Assert.Empty(storage.Content); + Assert.Empty(storage.ContentBytes); + } + + [Fact] + public async Task ProcessAsync_WithEmptyStorageAndOneNewItem_CreatesIndexAndLeaf() + { + var packageDetails = new CatalogPackageDetails(); + var newItem = CreateNewItem(packageDetails); + var storage = await ProcessAsync(newItem, packageDetails.Id); + + Verify( + storage, + expectedStorageContentCount: 2, + expectedPage: new ExpectedPage(packageDetails)); + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + public async Task ProcessAsync_WithCustomPartitionSize_FillsPages(int partitionSize) + { + // The idea here is that we should see pages filled before a new pages is created. + // Ultimately, we should have 2 filled pages plus one unfilled page. + var packageVersionCount = partitionSize * 2 + 1; + var pages = new List(); + + for (var i = 0; i < packageVersionCount; ++i) + { + var packageDetails = new CatalogPackageDetails(id: "A", version: $"1.0.{i}"); + var newItem = CreateNewItem(packageDetails); + var storage = await ProcessAsync(newItem, packageDetails.Id, partitionSize); + + pages.Add(new ExpectedPage(packageDetails)); + + var expectedPages = Repaginate(pages, partitionSize); + + Verify( + storage, + expectedStorageContentCount: pages.Count + 1 /*index*/, + expectedPages: expectedPages); + } + } + + [Fact] + public async Task ProcessAsync_WithVariablePartitionSize_RepaginatesOlderPages() + { + var partitionSize = 2; + var packageVersionCount = partitionSize * 2; + var pages = new List(); + CatalogPackageDetails packageDetails; + IReadOnlyDictionary newItem; + MemoryStorage storage; + IReadOnlyList expectedPages; + + for (var i = 0; i < packageVersionCount; ++i) + { + packageDetails = new CatalogPackageDetails(id: "a", version: $"1.0.{i}"); + newItem = CreateNewItem(packageDetails); + storage = await ProcessAsync(newItem, packageDetails.Id, partitionSize); + + pages.Add(new ExpectedPage(packageDetails)); + + expectedPages = Repaginate(pages, partitionSize); + + Verify( + storage, + expectedStorageContentCount: pages.Count + 1 /*index*/, + expectedPages: expectedPages); + } + + partitionSize = 3; + ++packageVersionCount; + + packageDetails = new CatalogPackageDetails(id: "a", version: $"1.0.{packageVersionCount}"); + newItem = CreateNewItem(packageDetails); + storage = await ProcessAsync(newItem, packageDetails.Id, partitionSize); + + pages.Add(new ExpectedPage(packageDetails)); + + expectedPages = Repaginate(pages, partitionSize); + + Verify( + storage, + expectedStorageContentCount: pages.Count + 1 /*index*/, + expectedPages: expectedPages); + } + + private static IReadOnlyDictionary CreateNewItem(CatalogPackageDetails packageDetails) + { + var json = JsonConvert.SerializeObject(packageDetails, _jsonSettings); + + using (var reader = new JsonTextReader(new StringReader(json))) + { + reader.DateParseHandling = DateParseHandling.DateTimeOffset; // make sure we always preserve timezone info + + var jObject = JObject.Load(reader); + var graph = Utils.CreateGraph(jObject, readOnly: true); + + return new Dictionary() + { + { packageDetails.IdKeyword, graph } + }; + } + } + + private async Task ProcessAsync( + IReadOnlyDictionary newItems, + string packageId, + int? partitionSize = null) + { + var registrationKey = new RegistrationKey(packageId?.ToLowerInvariant() ?? string.Empty); + + await RegistrationMaker.ProcessAsync( + registrationKey, + newItems, + _storageFactory, + _contentBaseAddress, + partitionSize ?? _partitionSize, + _packageCountThreshold, + _telemetryService.Object, + CancellationToken.None); + + return (MemoryStorage)_storageFactory.Create(registrationKey.ToString()); + } + + private void Verify( + MemoryStorage registrationStorage, + int expectedStorageContentCount, + ExpectedPage expectedPage) + { + Verify(registrationStorage, expectedStorageContentCount, new[] { expectedPage }); + } + + private void Verify( + MemoryStorage registrationStorage, + int expectedStorageContentCount, + IReadOnlyList expectedPages) + { + Assert.Equal(expectedStorageContentCount, registrationStorage.Content.Count); + + var firstPage = expectedPages.First(); + var packageId = firstPage.Details.First().Id.ToLowerInvariant(); + var indexUri = new Uri(GetRegistrationPackageIndexUri(packageId)); + var index = GetStorageContent(registrationStorage, indexUri); + + VerifyIndex(index, indexUri, packageId, expectedPages); + } + + private void VerifyIndex( + RegistrationIndex index, + Uri indexUri, + string packageId, + IReadOnlyList expectedPages) + { + Assert.Equal(indexUri.AbsoluteUri, index.IdKeyword); + Assert.Equal( + new[] { "catalog:CatalogRoot", "PackageRegistration", CatalogConstants.CatalogPermalink }, + index.TypeKeyword); + + Assert.True(Guid.TryParse(index.CommitId, out var guid)); + Assert.True(DateTime.TryParse(index.CommitTimeStamp, out var datetime)); + Assert.Equal(expectedPages.Count, index.Count); + Assert.Equal(index.Count, index.Items.Length); + + for (var i = 0; i < index.Count; ++i) + { + var page = index.Items[i]; + var expectedPageVersionRange = expectedPages[i]; + + VerifyIndexItems( + page, + indexUri, + packageId, + expectedPageVersionRange, + index.CommitId, + index.CommitTimeStamp); + } + + var expectedContext = new JObject( + new JProperty(CatalogConstants.VocabKeyword, CatalogConstants.NuGetSchemaUri), + new JProperty(CatalogConstants.Catalog, CatalogConstants.NuGetCatalogSchemaUri), + new JProperty(CatalogConstants.Xsd, CatalogConstants.XmlSchemaUri), + new JProperty(CatalogConstants.Items, + new JObject( + new JProperty(CatalogConstants.IdKeyword, CatalogConstants.CatalogItem), + new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword))), + new JProperty(CatalogConstants.CommitTimeStamp, + new JObject( + new JProperty(CatalogConstants.IdKeyword, CatalogConstants.CatalogCommitTimeStamp), + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime))), + new JProperty(CatalogConstants.CommitId, + new JObject( + new JProperty(CatalogConstants.IdKeyword, CatalogConstants.CatalogCommitId))), + new JProperty(CatalogConstants.Count, + new JObject( + new JProperty(CatalogConstants.IdKeyword, CatalogConstants.CatalogCount))), + new JProperty(CatalogConstants.Parent, + new JObject( + new JProperty(CatalogConstants.IdKeyword, CatalogConstants.CatalogParent), + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.IdKeyword))), + new JProperty(CatalogConstants.Tags, + new JObject( + new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword), + new JProperty(CatalogConstants.IdKeyword, CatalogConstants.Tag))), + new JProperty(CatalogConstants.PackageTargetFrameworks, + new JObject( + new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword), + new JProperty(CatalogConstants.IdKeyword, CatalogConstants.PackageTargetFramework))), + new JProperty(CatalogConstants.DependencyGroups, + new JObject( + new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword), + new JProperty(CatalogConstants.IdKeyword, CatalogConstants.DependencyGroup))), + new JProperty(CatalogConstants.Dependencies, + new JObject( + new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword), + new JProperty(CatalogConstants.IdKeyword, CatalogConstants.Dependency))), + new JProperty(CatalogConstants.PackageContent, + new JObject( + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.IdKeyword))), + new JProperty(CatalogConstants.Published, + new JObject( + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime))), + new JProperty(CatalogConstants.Registration, + new JObject( + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.IdKeyword)))); + + Assert.Equal(expectedContext.ToString(), index.ContextKeyword.ToString()); + } + + private void VerifyIndexItems( + RegistrationIndexPage items, + Uri indexUri, + string packageId, + ExpectedPage expectedPage, + string commitId, + string commitTimeStamp) + { + Assert.Equal( + GetRegistrationPackageIndexUri(packageId) + $"#page/{expectedPage.LowerVersion}/{expectedPage.UpperVersion}", + items.IdKeyword); + Assert.Equal(CatalogConstants.CatalogCatalogPage, items.TypeKeyword); + Assert.Equal(commitId, items.CommitId); + Assert.Equal(commitTimeStamp, items.CommitTimeStamp); + Assert.Equal(items.Count, items.Items.Length); + Assert.Equal(GetRegistrationPackageIndexUri(packageId), items.Parent); + Assert.Equal(expectedPage.LowerVersion, items.Lower); + Assert.Equal(expectedPage.UpperVersion, items.Upper); + + for (var i = 0; i < items.Count; ++i) + { + var item = items.Items[i]; + var packageDetails = expectedPage.Details[i]; + + VerifyIndexItem(item, packageDetails, commitId, commitTimeStamp); + } + } + + private void VerifyIndexItem( + RegistrationIndexPackageDetails item, + CatalogPackageDetails packageDetails, + string commitId, + string commitTimeStamp) + { + var packageId = packageDetails.Id.ToLowerInvariant(); + var packageVersion = packageDetails.Version.ToLowerInvariant(); + + Assert.Equal(GetRegistrationPackageVersionUri(packageId, packageVersion), item.IdKeyword); + Assert.Equal(CatalogConstants.Package, item.TypeKeyword); + Assert.Equal(commitId, item.CommitId); + Assert.Equal(commitTimeStamp, item.CommitTimeStamp); + Assert.Equal(packageDetails.IdKeyword, item.CatalogEntry.IdKeyword); + Assert.Equal(CatalogConstants.PackageDetails, item.CatalogEntry.TypeKeyword); + Assert.Equal(packageDetails.Authors, item.CatalogEntry.Authors); + Assert.Equal(packageDetails.Description, item.CatalogEntry.Description); + Assert.Empty(item.CatalogEntry.IconUrl); + Assert.Equal(packageDetails.Id, item.CatalogEntry.Id); + Assert.Empty(item.CatalogEntry.Language); + Assert.Empty(item.CatalogEntry.LicenseUrl); + Assert.Equal(packageDetails.Listed, item.CatalogEntry.Listed); + Assert.Empty(item.CatalogEntry.MinClientVersion); + Assert.Equal(GetPackageContentUri(packageId, packageVersion), item.CatalogEntry.PackageContent); + Assert.Empty(item.CatalogEntry.ProjectUrl); + Assert.Equal(GetRegistrationDateTime(packageDetails.Published), item.CatalogEntry.Published); + Assert.Equal(packageDetails.RequireLicenseAcceptance, item.CatalogEntry.RequireLicenseAcceptance); + Assert.Empty(item.CatalogEntry.Summary); + Assert.Equal(new[] { string.Empty }, item.CatalogEntry.Tags); + Assert.Empty(item.CatalogEntry.Title); + Assert.Equal(packageDetails.Version, item.CatalogEntry.Version); + + var leafUri = new Uri(GetRegistrationPackageVersionUri(packageId, packageVersion)); + var registrationStorage = (MemoryStorage)_storageFactory.Create(packageId); + var leaf = GetStorageContent(registrationStorage, leafUri); + + VerifyLeaf(leaf, leafUri, packageDetails, packageId, packageVersion); + } + + private void VerifyLeaf( + RegistrationPackage leaf, + Uri expectedIdKeyword, + CatalogPackageDetails packageDetails, + string packageId, + string packageVersion) + { + Assert.Equal(expectedIdKeyword.AbsoluteUri, leaf.IdKeyword); + Assert.Equal( + new[] { CatalogConstants.Package, CatalogConstants.NuGetCatalogSchemaPermalinkUri }, + leaf.TypeKeyword); + Assert.Equal(packageDetails.Listed, leaf.Listed); + Assert.Equal(GetPackageContentUri(packageId, packageVersion), leaf.PackageContent); + Assert.Equal(GetRegistrationDateTime(packageDetails.Published), leaf.Published); + Assert.Equal(GetRegistrationPackageIndexUri(packageId), leaf.Registration); + + var expectedContext = new JObject( + new JProperty(CatalogConstants.VocabKeyword, CatalogConstants.NuGetSchemaUri), + new JProperty(CatalogConstants.Xsd, CatalogConstants.XmlSchemaUri), + new JProperty(CatalogConstants.CatalogEntry, + new JObject( + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.IdKeyword))), + new JProperty(CatalogConstants.Registration, + new JObject( + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.IdKeyword))), + new JProperty(CatalogConstants.PackageContent, + new JObject( + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.IdKeyword))), + new JProperty(CatalogConstants.Published, + new JObject( + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime)))); + + Assert.Equal(expectedContext.ToString(), leaf.ContextKeyword.ToString()); + } + + private static string GetRegistrationDateTime(string catalogDateTime) + { + return DateTimeOffset.Parse(catalogDateTime) + .ToLocalTime() + .ToString("yyyy-MM-ddTHH:mm:ss.FFFzzz"); + } + + private static T GetStorageContent(MemoryStorage registrationStorage, Uri contentUri) + { + Assert.True(registrationStorage.Content.TryGetValue(contentUri, out var content)); + + var jTokenStorageContent = content as JTokenStorageContent; + + Assert.NotNull(jTokenStorageContent); + Assert.Equal(_cacheControl, jTokenStorageContent.CacheControl); + Assert.Equal(_contentType, jTokenStorageContent.ContentType); + + var properties = typeof(T).GetProperties(BindingFlags.NonPublic | BindingFlags.Instance); + + // Verify that no new properties were added unexpectedly. + Assert.Equal(properties.Length, ((JObject)jTokenStorageContent.Content).Count); + + var json = jTokenStorageContent.Content.ToString(); + + using (var stringReader = new StringReader(json)) + { + return JsonConvert.DeserializeObject(json, _jsonSettings); + } + } + + private static string GetPackageContentUri(string packageId, string packageVersion) + { + return $"{_contentBaseAddress.AbsoluteUri}packages/{packageId}.{packageVersion}.nupkg"; + } + + private string GetRegistrationPackageIndexUri(string packageId) + { + return $"{_storageFactory.BaseAddress.AbsoluteUri}{packageId}/index.json"; + } + + private string GetRegistrationPackageVersionUri(string packageId, string packageVersion) + { + return $"{_storageFactory.BaseAddress.AbsoluteUri}{packageId}/{packageVersion}.json"; + } + + private static IReadOnlyList Repaginate( + IReadOnlyList pages, + int partitionSize) + { + return pages.Select((page, index) => new { Page = page, Index = index }) + .GroupBy(x => x.Index / partitionSize) + .Select(group => new ExpectedPage(group.SelectMany(x => x.Page.Details).ToArray())) + .ToList(); + } + + private sealed class ExpectedPage + { + internal IReadOnlyList Details { get; } + internal string LowerVersion { get; } + internal string UpperVersion { get; } + + internal ExpectedPage(params CatalogPackageDetails[] packageDetails) + { + Details = packageDetails; + + var versions = packageDetails.Select(details => new NuGetVersion(details.Version)).ToArray(); + + LowerVersion = versions.Min().ToNormalizedString().ToLowerInvariant(); + UpperVersion = versions.Max().ToNormalizedString().ToLowerInvariant(); + } + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/SortingGraphCollectorTests.cs b/tests/CatalogTests/SortingGraphCollectorTests.cs index dbef0e98e..e11a4b8b4 100644 --- a/tests/CatalogTests/SortingGraphCollectorTests.cs +++ b/tests/CatalogTests/SortingGraphCollectorTests.cs @@ -78,7 +78,7 @@ public async Task GraphsAreBatchedById() private class TestSortingGraphCollector : SortingGraphCollector { - private readonly ConcurrentBag>> _allSortedGraphs; + private readonly ConcurrentBag>> _allSortedGraphs; public TestSortingGraphCollector(Uri index, Uri[] types, Func handlerFunc) : base( index, @@ -86,12 +86,12 @@ private class TestSortingGraphCollector : SortingGraphCollector new Mock().Object, handlerFunc) { - _allSortedGraphs = new ConcurrentBag>>(); + _allSortedGraphs = new ConcurrentBag>>(); } - public IEnumerable>> AllSortedGraphs => _allSortedGraphs; + public IEnumerable>> AllSortedGraphs => _allSortedGraphs; - protected override Task ProcessGraphs(KeyValuePair> sortedGraphs, CancellationToken cancellationToken) + protected override Task ProcessGraphsAsync(KeyValuePair> sortedGraphs, CancellationToken cancellationToken) { _allSortedGraphs.Add(sortedGraphs); return Task.FromResult(0); diff --git a/tests/NgTests/CatalogConstants.cs b/tests/NgTests/CatalogConstants.cs index ea573f35c..3fac8a046 100644 --- a/tests/NgTests/CatalogConstants.cs +++ b/tests/NgTests/CatalogConstants.cs @@ -3,78 +3,98 @@ namespace NgTests { - internal static class CatalogConstants + public static class CatalogConstants { - internal const string AppendOnlyCatalog = "AppendOnlyCatalog"; - internal const string Authors = "authors"; - internal const string Catalog = "catalog"; - internal const string CatalogCommitId = "catalog:commitId"; - internal const string CatalogCommitTimeStamp = "catalog:commitTimeStamp"; - internal const string CatalogDetails = "catalog:details"; - internal const string CatalogPage = "CatalogPage"; - internal const string CatalogPermalink = "catalog:Permalink"; - internal const string CatalogRoot = "CatalogRoot"; - internal const string Categories = "categories"; - internal const string CommitId = "commitId"; - internal const string CommitTimeStamp = "commitTimeStamp"; - internal const string CompressedLength = "compressedLength"; - internal const string ContainerKeyword = "@container"; - internal const string ContextKeyword = "@context"; - internal const string Count = "count"; - internal const string Created = "created"; - internal const string Dependencies = "dependencies"; - internal const string Dependency = "dependency"; - internal const string DependencyGroup = "dependencyGroup"; - internal const string DependencyGroups = "dependencyGroups"; - internal const string Description = "description"; - internal const string Details = "details"; - internal const string Entries = "entries"; - internal const string FullName = "fullName"; - internal const string Id = "id"; - internal const string IdKeyword = "@id"; - internal const string IsPrerelease = "isPrerelease"; - internal const string Item = "item"; - internal const string Items = "items"; - internal const string LastEdited = "lastEdited"; - internal const string Length = "length"; - internal const string Links = "links"; - internal const string Listed = "listed"; - internal const string Name = "name"; - internal const string NuGet = "nuget"; - internal const string NuGetCatalogSchemaUri = "http://schema.nuget.org/catalog#"; - internal const string NuGetId = "nuget:id"; - internal const string NuGetLastCreated = "nuget:lastCreated"; - internal const string NuGetLastDeleted = "nuget:lastDeleted"; - internal const string NuGetLastEdited = "nuget:lastEdited"; - internal const string NuGetPackageDelete = "nuget:PackageDelete"; - internal const string NuGetPackageDetails = "nuget:PackageDetails"; - internal const string NuGetSchemaUri = "http://schema.nuget.org/schema#"; - internal const string NuGetVersion = "nuget:version"; - internal const string OriginalId = "originalId"; - internal const string PackageContent = "packageContent"; - internal const string PackageDelete = "PackageDelete"; - internal const string PackageDetails = "PackageDetails"; - internal const string PackageEntries = "packageEntries"; - internal const string PackageEntry = "PackageEntry"; - internal const string PackageEntryUncapitalized = "packageEntry"; - internal const string PackageHash = "packageHash"; - internal const string PackageHashAlgorithm = "packageHashAlgorithm"; - internal const string PackageSize = "packageSize"; - internal const string Parent = "parent"; - internal const string Permalink = "Permalink"; - internal const string Published = "published"; - internal const string SetKeyword = "@set"; - internal const string SupportedFramework = "supportedFramework"; - internal const string SupportedFrameworks = "supportedFrameworks"; - internal const string Tag = "tag"; - internal const string Tags = "tags"; - internal const string TypeKeyword = "@type"; - internal const string VerbatimVersion = "verbatimVersion"; - internal const string Version = "version"; - internal const string VocabKeyword = "@vocab"; - internal const string XmlDateTimeSchemaUri = "http://www.w3.org/2001/XMLSchema#dateTime"; - internal const string XmlSchemaUri = "http://www.w3.org/2001/XMLSchema#"; - internal const string Xsd = "xsd"; - internal const string XsdDateTime = "xsd:dateTime"; + public const string AppendOnlyCatalog = "AppendOnlyCatalog"; + public const string Authors = "authors"; + public const string Catalog = "catalog"; + public const string CatalogCatalogPage = "catalog:CatalogPage"; + public const string CatalogCommitId = "catalog:commitId"; + public const string CatalogCommitTimeStamp = "catalog:commitTimeStamp"; + public const string CatalogCount = "catalog:count"; + public const string CatalogDetails = "catalog:details"; + public const string CatalogEntry = "catalogEntry"; + public const string CatalogItem = "catalog:item"; + public const string CatalogPage = "CatalogPage"; + public const string CatalogParent = "catalog:parent"; + public const string CatalogPermalink = "catalog:Permalink"; + public const string CatalogRoot = "CatalogRoot"; + public const string Categories = "categories"; + public const string CommitId = "commitId"; + public const string CommitTimeStamp = "commitTimeStamp"; + public const string CompressedLength = "compressedLength"; + public const string ContainerKeyword = "@container"; + public const string ContextKeyword = "@context"; + public const string Count = "count"; + public const string Created = "created"; + public const string Dependencies = "dependencies"; + public const string Dependency = "dependency"; + public const string DependencyGroup = "dependencyGroup"; + public const string DependencyGroups = "dependencyGroups"; + public const string Description = "description"; + public const string Details = "details"; + public const string Entries = "entries"; + public const string FullName = "fullName"; + public const string IconUrl = "iconUrl"; + public const string Id = "id"; + public const string IdKeyword = "@id"; + public const string IsPrerelease = "isPrerelease"; + public const string Item = "item"; + public const string Items = "items"; + public const string Language = "language"; + public const string LastEdited = "lastEdited"; + public const string Length = "length"; + public const string LicenseUrl = "licenseUrl"; + public const string Links = "links"; + public const string Listed = "listed"; + public const string Lower = "lower"; + public const string MinClientVersion = "minClientVersion"; + public const string Name = "name"; + public const string NuGet = "nuget"; + public const string NuGetCatalogSchemaPermalinkUri = "http://schema.nuget.org/catalog#Permalink"; + public const string NuGetCatalogSchemaUri = "http://schema.nuget.org/catalog#"; + public const string NuGetId = "nuget:id"; + public const string NuGetLastCreated = "nuget:lastCreated"; + public const string NuGetLastDeleted = "nuget:lastDeleted"; + public const string NuGetLastEdited = "nuget:lastEdited"; + public const string NuGetPackageDelete = "nuget:PackageDelete"; + public const string NuGetPackageDetails = "nuget:PackageDetails"; + public const string NuGetSchemaUri = "http://schema.nuget.org/schema#"; + public const string NuGetVersion = "nuget:version"; + public const string OriginalId = "originalId"; + public const string Package = "Package"; + public const string PackageContent = "packageContent"; + public const string PackageDelete = "PackageDelete"; + public const string PackageDetails = "PackageDetails"; + public const string PackageEntries = "packageEntries"; + public const string PackageEntry = "PackageEntry"; + public const string PackageEntryUncapitalized = "packageEntry"; + public const string PackageHash = "packageHash"; + public const string PackageHashAlgorithm = "packageHashAlgorithm"; + public const string PackageSize = "packageSize"; + public const string PackageTargetFramework = "packageTargetFramework"; + public const string PackageTargetFrameworks = "packageTargetFrameworks"; + public const string Parent = "parent"; + public const string Permalink = "Permalink"; + public const string ProjectUrl = "projectUrl"; + public const string Published = "published"; + public const string RequireLicenseAcceptance = "requireLicenseAcceptance"; + public const string Registration = "registration"; + public const string SetKeyword = "@set"; + public const string SupportedFramework = "supportedFramework"; + public const string SupportedFrameworks = "supportedFrameworks"; + public const string Summary = "summary"; + public const string Tag = "tag"; + public const string Tags = "tags"; + public const string Title = "title"; + public const string TypeKeyword = "@type"; + public const string Upper = "upper"; + public const string VerbatimVersion = "verbatimVersion"; + public const string Version = "version"; + public const string VocabKeyword = "@vocab"; + public const string XmlDateTimeSchemaUri = "http://www.w3.org/2001/XMLSchema#dateTime"; + public const string XmlSchemaUri = "http://www.w3.org/2001/XMLSchema#"; + public const string Xsd = "xsd"; + public const string XsdDateTime = "xsd:dateTime"; } } \ No newline at end of file diff --git a/tests/NgTests/Infrastructure/MemoryStorageFactory.cs b/tests/NgTests/Infrastructure/MemoryStorageFactory.cs index f81702701..92192f0ec 100644 --- a/tests/NgTests/Infrastructure/MemoryStorageFactory.cs +++ b/tests/NgTests/Infrastructure/MemoryStorageFactory.cs @@ -9,10 +9,17 @@ namespace NgTests.Infrastructure { public class MemoryStorageFactory : StorageFactory { - private Dictionary _cachedStorages; + private readonly Dictionary _cachedStorages; public MemoryStorageFactory() + : this(new Uri("https://nuget.test")) { + } + + public MemoryStorageFactory(Uri baseAddress) + { + BaseAddress = baseAddress ?? throw new ArgumentNullException(nameof(baseAddress)); + _cachedStorages = new Dictionary(); } @@ -20,10 +27,10 @@ public override Storage Create(string name = null) { if (!_cachedStorages.ContainsKey(name)) { - _cachedStorages[name] = (MemoryStorage)new MemoryStorage().WithName(name); + _cachedStorages[name] = (MemoryStorage)new MemoryStorage(BaseAddress).WithName(name); } return _cachedStorages[name]; } } -} +} \ No newline at end of file diff --git a/tests/NgTests/Infrastructure/TestPackage.cs b/tests/NgTests/Infrastructure/TestPackage.cs index ae1837860..e4f647761 100644 --- a/tests/NgTests/Infrastructure/TestPackage.cs +++ b/tests/NgTests/Infrastructure/TestPackage.cs @@ -4,7 +4,6 @@ using System; using System.IO; using System.IO.Compression; -using System.Linq; using System.Security.Cryptography; using NuGet.Versioning; @@ -43,10 +42,10 @@ public void Dispose() internal static TestPackage Create(Random random) { - var id = CreateRandomAlphanumericString(random); + var id = TestUtility.CreateRandomAlphanumericString(random); var version = CreateRandomVersion(random); - var author = CreateRandomAlphanumericString(random); - var description = CreateRandomAlphanumericString(random); + var author = TestUtility.CreateRandomAlphanumericString(random); + var description = TestUtility.CreateRandomAlphanumericString(random); var nuspec = CreateNuspec(id, version, author, description); using (var rng = RandomNumberGenerator.Create()) @@ -57,16 +56,6 @@ internal static TestPackage Create(Random random) } } - private static string CreateRandomAlphanumericString(Random random) - { - const string characters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; - - return new string( - Enumerable.Repeat(characters, count: 16) - .Select(s => s[random.Next(s.Length)]) - .ToArray()); - } - private static string CreateNuspec(string id, NuGetVersion version, string author, string description) { return $@" diff --git a/tests/NgTests/Infrastructure/TestUtility.cs b/tests/NgTests/Infrastructure/TestUtility.cs new file mode 100644 index 000000000..752978d1e --- /dev/null +++ b/tests/NgTests/Infrastructure/TestUtility.cs @@ -0,0 +1,28 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; + +namespace NgTests.Infrastructure +{ + public static class TestUtility + { + private static readonly Random _random = new Random(); + + public static string CreateRandomAlphanumericString() + { + return CreateRandomAlphanumericString(_random); + } + + public static string CreateRandomAlphanumericString(Random random) + { + const string characters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + + return new string( + Enumerable.Repeat(characters, count: 16) + .Select(s => s[random.Next(s.Length)]) + .ToArray()); + } + } +} \ No newline at end of file diff --git a/tests/NgTests/NgTests.csproj b/tests/NgTests/NgTests.csproj index 1b7c68da6..af928906f 100644 --- a/tests/NgTests/NgTests.csproj +++ b/tests/NgTests/NgTests.csproj @@ -279,6 +279,7 @@ + diff --git a/tests/NgTests/SortingIdVersionCollectorTests.cs b/tests/NgTests/SortingIdVersionCollectorTests.cs index a92cc6bfa..fce5cf2bd 100644 --- a/tests/NgTests/SortingIdVersionCollectorTests.cs +++ b/tests/NgTests/SortingIdVersionCollectorTests.cs @@ -72,12 +72,12 @@ public async Task OnProcessBatch_BatchesCorrectly(IEnumerable items) collectorMock .Setup(x => x.OverridableProcessSortedBatch(It.IsAny>>())) .Returns>>( - (pair) => + (pair) => { // Assert Assert.DoesNotContain( - seenPackages, - (p) => + seenPackages, + (p) => { return p.Id == pair.Key.Id && p.Version == pair.Key.Version; }); @@ -102,7 +102,7 @@ private static JToken CreatePackage(string id, string version) public class TestableSortingIdVersionCollector : SortingIdVersionCollector { - public TestableSortingIdVersionCollector() + public TestableSortingIdVersionCollector() : base( new Uri("https://www.microsoft.com"), new Mock().Object, @@ -115,10 +115,10 @@ public Task OnProcessBatch(IEnumerable items) return base.OnProcessBatch(null, items, null, DateTime.MinValue, false, CancellationToken.None); } - protected override Task ProcessSortedBatch( - CollectorHttpClient client, - KeyValuePair> sortedBatch, - JToken context, + protected override Task ProcessSortedBatchAsync( + CollectorHttpClient client, + KeyValuePair> sortedBatch, + JToken context, CancellationToken cancellationToken) { return OverridableProcessSortedBatch(sortedBatch); @@ -131,4 +131,4 @@ public Task OnProcessBatch(IEnumerable items) } } } -} +} \ No newline at end of file From 58678f62f9d02d6bc5be2f720bdbe6f95909732d Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Tue, 21 Aug 2018 10:10:43 -0700 Subject: [PATCH 06/15] remove unused scripts (#340) --- .../NuGet.Services.BasicSearch.csproj | 2 - .../scripts/ai.0.22.9-build00167.js | 3524 ----------------- .../scripts/ai.0.22.9-build00167.min.js | 1 - 3 files changed, 3527 deletions(-) delete mode 100644 src/NuGet.Services.BasicSearch/scripts/ai.0.22.9-build00167.js delete mode 100644 src/NuGet.Services.BasicSearch/scripts/ai.0.22.9-build00167.min.js diff --git a/src/NuGet.Services.BasicSearch/NuGet.Services.BasicSearch.csproj b/src/NuGet.Services.BasicSearch/NuGet.Services.BasicSearch.csproj index 10402eeb0..4b9213138 100644 --- a/src/NuGet.Services.BasicSearch/NuGet.Services.BasicSearch.csproj +++ b/src/NuGet.Services.BasicSearch/NuGet.Services.BasicSearch.csproj @@ -332,8 +332,6 @@ - - Designer diff --git a/src/NuGet.Services.BasicSearch/scripts/ai.0.22.9-build00167.js b/src/NuGet.Services.BasicSearch/scripts/ai.0.22.9-build00167.js deleted file mode 100644 index 6a79fefd8..000000000 --- a/src/NuGet.Services.BasicSearch/scripts/ai.0.22.9-build00167.js +++ /dev/null @@ -1,3524 +0,0 @@ -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - (function (LoggingSeverity) { - LoggingSeverity[LoggingSeverity["CRITICAL"] = 0] = "CRITICAL"; - LoggingSeverity[LoggingSeverity["WARNING"] = 1] = "WARNING"; - })(ApplicationInsights.LoggingSeverity || (ApplicationInsights.LoggingSeverity = {})); - var LoggingSeverity = ApplicationInsights.LoggingSeverity; - (function (_InternalMessageId) { - _InternalMessageId[_InternalMessageId["NONUSRACT_BrowserDoesNotSupportLocalStorage"] = 0] = "NONUSRACT_BrowserDoesNotSupportLocalStorage"; - _InternalMessageId[_InternalMessageId["NONUSRACT_BrowserCannotReadLocalStorage"] = 1] = "NONUSRACT_BrowserCannotReadLocalStorage"; - _InternalMessageId[_InternalMessageId["NONUSRACT_BrowserCannotReadSessionStorage"] = 2] = "NONUSRACT_BrowserCannotReadSessionStorage"; - _InternalMessageId[_InternalMessageId["NONUSRACT_BrowserCannotWriteLocalStorage"] = 3] = "NONUSRACT_BrowserCannotWriteLocalStorage"; - _InternalMessageId[_InternalMessageId["NONUSRACT_BrowserCannotWriteSessionStorage"] = 4] = "NONUSRACT_BrowserCannotWriteSessionStorage"; - _InternalMessageId[_InternalMessageId["NONUSRACT_BrowserFailedRemovalFromLocalStorage"] = 5] = "NONUSRACT_BrowserFailedRemovalFromLocalStorage"; - _InternalMessageId[_InternalMessageId["NONUSRACT_BrowserFailedRemovalFromSessionStorage"] = 6] = "NONUSRACT_BrowserFailedRemovalFromSessionStorage"; - _InternalMessageId[_InternalMessageId["NONUSRACT_CannotSendEmptyTelemetry"] = 7] = "NONUSRACT_CannotSendEmptyTelemetry"; - _InternalMessageId[_InternalMessageId["NONUSRACT_ClientPerformanceMathError"] = 8] = "NONUSRACT_ClientPerformanceMathError"; - _InternalMessageId[_InternalMessageId["NONUSRACT_ErrorParsingAISessionCookie"] = 9] = "NONUSRACT_ErrorParsingAISessionCookie"; - _InternalMessageId[_InternalMessageId["NONUSRACT_ErrorPVCalc"] = 10] = "NONUSRACT_ErrorPVCalc"; - _InternalMessageId[_InternalMessageId["NONUSRACT_ExceptionWhileLoggingError"] = 11] = "NONUSRACT_ExceptionWhileLoggingError"; - _InternalMessageId[_InternalMessageId["NONUSRACT_FailedAddingTelemetryToBuffer"] = 12] = "NONUSRACT_FailedAddingTelemetryToBuffer"; - _InternalMessageId[_InternalMessageId["NONUSRACT_FailedMonitorAjaxAbort"] = 13] = "NONUSRACT_FailedMonitorAjaxAbort"; - _InternalMessageId[_InternalMessageId["NONUSRACT_FailedMonitorAjaxDur"] = 14] = "NONUSRACT_FailedMonitorAjaxDur"; - _InternalMessageId[_InternalMessageId["NONUSRACT_FailedMonitorAjaxOpen"] = 15] = "NONUSRACT_FailedMonitorAjaxOpen"; - _InternalMessageId[_InternalMessageId["NONUSRACT_FailedMonitorAjaxRSC"] = 16] = "NONUSRACT_FailedMonitorAjaxRSC"; - _InternalMessageId[_InternalMessageId["NONUSRACT_FailedMonitorAjaxSend"] = 17] = "NONUSRACT_FailedMonitorAjaxSend"; - _InternalMessageId[_InternalMessageId["NONUSRACT_FailedToAddHandlerForOnBeforeUnload"] = 18] = "NONUSRACT_FailedToAddHandlerForOnBeforeUnload"; - _InternalMessageId[_InternalMessageId["NONUSRACT_FailedToSendQueuedTelemetry"] = 19] = "NONUSRACT_FailedToSendQueuedTelemetry"; - _InternalMessageId[_InternalMessageId["NONUSRACT_FailedToReportDataLoss"] = 20] = "NONUSRACT_FailedToReportDataLoss"; - _InternalMessageId[_InternalMessageId["NONUSRACT_FlushFailed"] = 21] = "NONUSRACT_FlushFailed"; - _InternalMessageId[_InternalMessageId["NONUSRACT_MessageLimitPerPVExceeded"] = 22] = "NONUSRACT_MessageLimitPerPVExceeded"; - _InternalMessageId[_InternalMessageId["NONUSRACT_MissingRequiredFieldSpecification"] = 23] = "NONUSRACT_MissingRequiredFieldSpecification"; - _InternalMessageId[_InternalMessageId["NONUSRACT_NavigationTimingNotSupported"] = 24] = "NONUSRACT_NavigationTimingNotSupported"; - _InternalMessageId[_InternalMessageId["NONUSRACT_OnError"] = 25] = "NONUSRACT_OnError"; - _InternalMessageId[_InternalMessageId["NONUSRACT_SessionRenewalDateIsZero"] = 26] = "NONUSRACT_SessionRenewalDateIsZero"; - _InternalMessageId[_InternalMessageId["NONUSRACT_SenderNotInitialized"] = 27] = "NONUSRACT_SenderNotInitialized"; - _InternalMessageId[_InternalMessageId["NONUSRACT_StartTrackEventFailed"] = 28] = "NONUSRACT_StartTrackEventFailed"; - _InternalMessageId[_InternalMessageId["NONUSRACT_StopTrackEventFailed"] = 29] = "NONUSRACT_StopTrackEventFailed"; - _InternalMessageId[_InternalMessageId["NONUSRACT_StartTrackFailed"] = 30] = "NONUSRACT_StartTrackFailed"; - _InternalMessageId[_InternalMessageId["NONUSRACT_StopTrackFailed"] = 31] = "NONUSRACT_StopTrackFailed"; - _InternalMessageId[_InternalMessageId["NONUSRACT_TelemetrySampledAndNotSent"] = 32] = "NONUSRACT_TelemetrySampledAndNotSent"; - _InternalMessageId[_InternalMessageId["NONUSRACT_TrackEventFailed"] = 33] = "NONUSRACT_TrackEventFailed"; - _InternalMessageId[_InternalMessageId["NONUSRACT_TrackExceptionFailed"] = 34] = "NONUSRACT_TrackExceptionFailed"; - _InternalMessageId[_InternalMessageId["NONUSRACT_TrackMetricFailed"] = 35] = "NONUSRACT_TrackMetricFailed"; - _InternalMessageId[_InternalMessageId["NONUSRACT_TrackPVFailed"] = 36] = "NONUSRACT_TrackPVFailed"; - _InternalMessageId[_InternalMessageId["NONUSRACT_TrackPVFailedCalc"] = 37] = "NONUSRACT_TrackPVFailedCalc"; - _InternalMessageId[_InternalMessageId["NONUSRACT_TrackTraceFailed"] = 38] = "NONUSRACT_TrackTraceFailed"; - _InternalMessageId[_InternalMessageId["NONUSRACT_TransmissionFailed"] = 39] = "NONUSRACT_TransmissionFailed"; - _InternalMessageId[_InternalMessageId["USRACT_CannotSerializeObject"] = 40] = "USRACT_CannotSerializeObject"; - _InternalMessageId[_InternalMessageId["USRACT_CannotSerializeObjectNonSerializable"] = 41] = "USRACT_CannotSerializeObjectNonSerializable"; - _InternalMessageId[_InternalMessageId["USRACT_CircularReferenceDetected"] = 42] = "USRACT_CircularReferenceDetected"; - _InternalMessageId[_InternalMessageId["USRACT_ClearAuthContextFailed"] = 43] = "USRACT_ClearAuthContextFailed"; - _InternalMessageId[_InternalMessageId["USRACT_ExceptionTruncated"] = 44] = "USRACT_ExceptionTruncated"; - _InternalMessageId[_InternalMessageId["USRACT_IllegalCharsInName"] = 45] = "USRACT_IllegalCharsInName"; - _InternalMessageId[_InternalMessageId["USRACT_ItemNotInArray"] = 46] = "USRACT_ItemNotInArray"; - _InternalMessageId[_InternalMessageId["USRACT_MaxAjaxPerPVExceeded"] = 47] = "USRACT_MaxAjaxPerPVExceeded"; - _InternalMessageId[_InternalMessageId["USRACT_MessageTruncated"] = 48] = "USRACT_MessageTruncated"; - _InternalMessageId[_InternalMessageId["USRACT_NameTooLong"] = 49] = "USRACT_NameTooLong"; - _InternalMessageId[_InternalMessageId["USRACT_SampleRateOutOfRange"] = 50] = "USRACT_SampleRateOutOfRange"; - _InternalMessageId[_InternalMessageId["USRACT_SetAuthContextFailed"] = 51] = "USRACT_SetAuthContextFailed"; - _InternalMessageId[_InternalMessageId["USRACT_SetAuthContextFailedAccountName"] = 52] = "USRACT_SetAuthContextFailedAccountName"; - _InternalMessageId[_InternalMessageId["USRACT_StringValueTooLong"] = 53] = "USRACT_StringValueTooLong"; - _InternalMessageId[_InternalMessageId["USRACT_StartCalledMoreThanOnce"] = 54] = "USRACT_StartCalledMoreThanOnce"; - _InternalMessageId[_InternalMessageId["USRACT_StopCalledWithoutStart"] = 55] = "USRACT_StopCalledWithoutStart"; - _InternalMessageId[_InternalMessageId["USRACT_TelemetryInitializerFailed"] = 56] = "USRACT_TelemetryInitializerFailed"; - _InternalMessageId[_InternalMessageId["USRACT_TrackArgumentsNotSpecified"] = 57] = "USRACT_TrackArgumentsNotSpecified"; - _InternalMessageId[_InternalMessageId["USRACT_UrlTooLong"] = 58] = "USRACT_UrlTooLong"; - })(ApplicationInsights._InternalMessageId || (ApplicationInsights._InternalMessageId = {})); - var _InternalMessageId = ApplicationInsights._InternalMessageId; - var _InternalLogMessage = (function () { - function _InternalLogMessage(msgId, msg, properties) { - this.message = _InternalMessageId[msgId].toString(); - this.messageId = msgId; - var diagnosticText = (msg ? " message:" + _InternalLogMessage.sanitizeDiagnosticText(msg) : "") + - (properties ? " props:" + _InternalLogMessage.sanitizeDiagnosticText(JSON.stringify(properties)) : ""); - this.message += diagnosticText; - } - _InternalLogMessage.sanitizeDiagnosticText = function (text) { - return "\"" + text.replace(/\"/g, "") + "\""; - }; - return _InternalLogMessage; - })(); - ApplicationInsights._InternalLogMessage = _InternalLogMessage; - var _InternalLogging = (function () { - function _InternalLogging() { - } - _InternalLogging.throwInternalNonUserActionable = function (severity, message) { - if (this.enableDebugExceptions()) { - throw message; - } - else { - if (typeof (message) !== "undefined" && !!message) { - if (typeof (message.message) !== "undefined") { - message.message = this.AiNonUserActionablePrefix + message.message; - this.warnToConsole(message.message); - this.logInternalMessage(severity, message); - } - } - } - }; - _InternalLogging.throwInternalUserActionable = function (severity, message) { - if (this.enableDebugExceptions()) { - throw message; - } - else { - if (typeof (message) !== "undefined" && !!message) { - if (typeof (message.message) !== "undefined") { - message.message = this.AiUserActionablePrefix + message.message; - this.warnToConsole(message.message); - this.logInternalMessage(severity, message); - } - } - } - }; - _InternalLogging.warnToConsole = function (message) { - if (typeof console !== "undefined" && !!console) { - if (typeof console.warn === "function") { - console.warn(message); - } - else if (typeof console.log === "function") { - console.log(message); - } - } - }; - _InternalLogging.resetInternalMessageCount = function () { - this._messageCount = 0; - }; - _InternalLogging.clearInternalMessageLoggedTypes = function () { - if (ApplicationInsights.Util.canUseSessionStorage()) { - var sessionStorageKeys = ApplicationInsights.Util.getSessionStorageKeys(); - for (var i = 0; i < sessionStorageKeys.length; i++) { - if (sessionStorageKeys[i].indexOf(_InternalLogging.AIInternalMessagePrefix) === 0) { - ApplicationInsights.Util.removeSessionStorage(sessionStorageKeys[i]); - } - } - } - }; - _InternalLogging.setMaxInternalMessageLimit = function (limit) { - if (!limit) { - throw new Error('limit cannot be undefined.'); - } - this.MAX_INTERNAL_MESSAGE_LIMIT = limit; - }; - _InternalLogging.logInternalMessage = function (severity, message) { - if (this._areInternalMessagesThrottled()) { - return; - } - var logMessage = true; - if (ApplicationInsights.Util.canUseSessionStorage()) { - var storageMessageKey = _InternalLogging.AIInternalMessagePrefix + _InternalMessageId[message.messageId]; - var internalMessageTypeLogRecord = ApplicationInsights.Util.getSessionStorage(storageMessageKey); - if (internalMessageTypeLogRecord) { - logMessage = false; - } - else { - ApplicationInsights.Util.setSessionStorage(storageMessageKey, "1"); - } - } - if (logMessage) { - if (this.verboseLogging() || severity === LoggingSeverity.CRITICAL) { - this.queue.push(message); - this._messageCount++; - } - if (this._messageCount == this.MAX_INTERNAL_MESSAGE_LIMIT) { - var throttleLimitMessage = "Internal events throttle limit per PageView reached for this app."; - var throttleMessage = new _InternalLogMessage(_InternalMessageId.NONUSRACT_MessageLimitPerPVExceeded, throttleLimitMessage); - this.queue.push(throttleMessage); - this.warnToConsole(throttleLimitMessage); - } - } - }; - _InternalLogging._areInternalMessagesThrottled = function () { - return this._messageCount >= this.MAX_INTERNAL_MESSAGE_LIMIT; - }; - _InternalLogging.AiUserActionablePrefix = "AI: "; - _InternalLogging.AIInternalMessagePrefix = "AITR_"; - _InternalLogging.AiNonUserActionablePrefix = "AI (Internal): "; - _InternalLogging.enableDebugExceptions = function () { return false; }; - _InternalLogging.verboseLogging = function () { return false; }; - _InternalLogging.queue = []; - _InternalLogging.MAX_INTERNAL_MESSAGE_LIMIT = 25; - _InternalLogging._messageCount = 0; - return _InternalLogging; - })(); - ApplicationInsights._InternalLogging = _InternalLogging; - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -/// -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - var StorageType; - (function (StorageType) { - StorageType[StorageType["LocalStorage"] = 0] = "LocalStorage"; - StorageType[StorageType["SessionStorage"] = 1] = "SessionStorage"; - })(StorageType || (StorageType = {})); - var Util = (function () { - function Util() { - } - Util._getLocalStorageObject = function () { - return Util._getVerifiedStorageObject(StorageType.LocalStorage); - }; - Util._getVerifiedStorageObject = function (storageType) { - var storage = null; - var fail; - var uid; - try { - uid = new Date; - storage = storageType === StorageType.LocalStorage ? window.localStorage : window.sessionStorage; - storage.setItem(uid, uid); - fail = storage.getItem(uid) != uid; - storage.removeItem(uid); - if (fail) { - storage = null; - } - } - catch (exception) { - storage = null; - } - return storage; - }; - Util.canUseLocalStorage = function () { - return !!Util._getLocalStorageObject(); - }; - Util.getStorage = function (name) { - var storage = Util._getLocalStorageObject(); - if (storage !== null) { - try { - return storage.getItem(name); - } - catch (e) { - var message = new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_BrowserCannotReadLocalStorage, "Browser failed read of local storage. " + Util.getExceptionName(e), { exception: Util.dump(e) }); - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.WARNING, message); - } - } - return null; - }; - Util.setStorage = function (name, data) { - var storage = Util._getLocalStorageObject(); - if (storage !== null) { - try { - storage.setItem(name, data); - return true; - } - catch (e) { - var message = new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_BrowserCannotWriteLocalStorage, "Browser failed write to local storage. " + Util.getExceptionName(e), { exception: Util.dump(e) }); - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.WARNING, message); - } - } - return false; - }; - Util.removeStorage = function (name) { - var storage = Util._getLocalStorageObject(); - if (storage !== null) { - try { - storage.removeItem(name); - return true; - } - catch (e) { - var message = new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_BrowserFailedRemovalFromLocalStorage, "Browser failed removal of local storage item. " + Util.getExceptionName(e), { exception: Util.dump(e) }); - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.WARNING, message); - } - } - return false; - }; - Util._getSessionStorageObject = function () { - return Util._getVerifiedStorageObject(StorageType.SessionStorage); - }; - Util.canUseSessionStorage = function () { - return !!Util._getSessionStorageObject(); - }; - Util.getSessionStorageKeys = function () { - var keys = []; - if (Util.canUseSessionStorage()) { - for (var key in window.sessionStorage) { - keys.push(key); - } - } - return keys; - }; - Util.getSessionStorage = function (name) { - var storage = Util._getSessionStorageObject(); - if (storage !== null) { - try { - return storage.getItem(name); - } - catch (e) { - var message = new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_BrowserCannotReadSessionStorage, "Browser failed read of session storage. " + Util.getExceptionName(e), { exception: Util.dump(e) }); - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.CRITICAL, message); - } - } - return null; - }; - Util.setSessionStorage = function (name, data) { - var storage = Util._getSessionStorageObject(); - if (storage !== null) { - try { - storage.setItem(name, data); - return true; - } - catch (e) { - var message = new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_BrowserCannotWriteSessionStorage, "Browser failed write to session storage. " + Util.getExceptionName(e), { exception: Util.dump(e) }); - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.CRITICAL, message); - } - } - return false; - }; - Util.removeSessionStorage = function (name) { - var storage = Util._getSessionStorageObject(); - if (storage !== null) { - try { - storage.removeItem(name); - return true; - } - catch (e) { - var message = new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_BrowserFailedRemovalFromSessionStorage, "Browser failed removal of session storage item. " + Util.getExceptionName(e), { exception: Util.dump(e) }); - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.CRITICAL, message); - } - } - return false; - }; - Util.setCookie = function (name, value, domain) { - var domainAttrib = ""; - if (domain) { - domainAttrib = ";domain=" + domain; - } - Util.document.cookie = name + "=" + value + domainAttrib + ";path=/"; - }; - Util.stringToBoolOrDefault = function (str) { - if (!str) { - return false; - } - return str.toString().toLowerCase() === "true"; - }; - Util.getCookie = function (name) { - var value = ""; - if (name && name.length) { - var cookieName = name + "="; - var cookies = Util.document.cookie.split(";"); - for (var i = 0; i < cookies.length; i++) { - var cookie = cookies[i]; - cookie = Util.trim(cookie); - if (cookie && cookie.indexOf(cookieName) === 0) { - value = cookie.substring(cookieName.length, cookies[i].length); - break; - } - } - } - return value; - }; - Util.deleteCookie = function (name) { - Util.document.cookie = name + "=;path=/;expires=Thu, 01 Jan 1970 00:00:01 GMT;"; - }; - Util.trim = function (str) { - if (typeof str !== "string") - return str; - return str.replace(/^\s+|\s+$/g, ""); - }; - Util.newId = function () { - var base64chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; - var result = ""; - var random = Math.random() * 1073741824; - while (random > 0) { - var char = base64chars.charAt(random % 64); - result += char; - random = Math.floor(random / 64); - } - return result; - }; - Util.isArray = function (obj) { - return Object.prototype.toString.call(obj) === "[object Array]"; - }; - Util.isError = function (obj) { - return Object.prototype.toString.call(obj) === "[object Error]"; - }; - Util.isDate = function (obj) { - return Object.prototype.toString.call(obj) === "[object Date]"; - }; - Util.toISOStringForIE8 = function (date) { - if (Util.isDate(date)) { - if (Date.prototype.toISOString) { - return date.toISOString(); - } - else { - function pad(number) { - var r = String(number); - if (r.length === 1) { - r = "0" + r; - } - return r; - } - return date.getUTCFullYear() - + "-" + pad(date.getUTCMonth() + 1) - + "-" + pad(date.getUTCDate()) - + "T" + pad(date.getUTCHours()) - + ":" + pad(date.getUTCMinutes()) - + ":" + pad(date.getUTCSeconds()) - + "." + String((date.getUTCMilliseconds() / 1000).toFixed(3)).slice(2, 5) - + "Z"; - } - } - }; - Util.getIEVersion = function (userAgentStr) { - if (userAgentStr === void 0) { userAgentStr = null; } - var myNav = userAgentStr ? userAgentStr.toLowerCase() : navigator.userAgent.toLowerCase(); - return (myNav.indexOf('msie') != -1) ? parseInt(myNav.split('msie')[1]) : null; - }; - Util.msToTimeSpan = function (totalms) { - if (isNaN(totalms) || totalms < 0) { - totalms = 0; - } - var ms = "" + totalms % 1000; - var sec = "" + Math.floor(totalms / 1000) % 60; - var min = "" + Math.floor(totalms / (1000 * 60)) % 60; - var hour = "" + Math.floor(totalms / (1000 * 60 * 60)) % 24; - ms = ms.length === 1 ? "00" + ms : ms.length === 2 ? "0" + ms : ms; - sec = sec.length < 2 ? "0" + sec : sec; - min = min.length < 2 ? "0" + min : min; - hour = hour.length < 2 ? "0" + hour : hour; - return hour + ":" + min + ":" + sec + "." + ms; - }; - Util.isCrossOriginError = function (message, url, lineNumber, columnNumber, error) { - return (message === "Script error." || message === "Script error") && error === null; - }; - Util.dump = function (object) { - var objectTypeDump = Object.prototype.toString.call(object); - var propertyValueDump = JSON.stringify(object); - if (objectTypeDump === "[object Error]") { - propertyValueDump = "{ stack: '" + object.stack + "', message: '" + object.message + "', name: '" + object.name + "'"; - } - return objectTypeDump + propertyValueDump; - }; - Util.getExceptionName = function (object) { - var objectTypeDump = Object.prototype.toString.call(object); - if (objectTypeDump === "[object Error]") { - return object.name; - } - return ""; - }; - Util.addEventHandler = function (eventName, callback) { - if (!window || typeof eventName !== 'string' || typeof callback !== 'function') { - return false; - } - var verbEventName = 'on' + eventName; - if (window.addEventListener) { - window.addEventListener(eventName, callback, false); - } - else if (window["attachEvent"]) { - window["attachEvent"].call(verbEventName, callback); - } - else { - return false; - } - return true; - }; - Util.document = typeof document !== "undefined" ? document : {}; - Util.NotSpecified = "not_specified"; - return Util; - })(); - ApplicationInsights.Util = Util; - var UrlHelper = (function () { - function UrlHelper() { - } - UrlHelper.parseUrl = function (url) { - if (!UrlHelper.htmlAnchorElement) { - UrlHelper.htmlAnchorElement = UrlHelper.document.createElement('a'); - } - UrlHelper.htmlAnchorElement.href = url; - return UrlHelper.htmlAnchorElement; - }; - UrlHelper.getAbsoluteUrl = function (url) { - var result; - var a = UrlHelper.parseUrl(url); - if (a) { - result = a.href; - } - return result; - }; - UrlHelper.getPathName = function (url) { - var result; - var a = UrlHelper.parseUrl(url); - if (a) { - result = a.pathname; - } - return result; - }; - UrlHelper.document = typeof document !== "undefined" ? document : {}; - return UrlHelper; - })(); - ApplicationInsights.UrlHelper = UrlHelper; - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -/// -/// -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - "use strict"; - var extensions = (function () { - function extensions() { - } - extensions.IsNullOrUndefined = function (obj) { - return typeof (obj) === "undefined" || obj === null; - }; - return extensions; - })(); - ApplicationInsights.extensions = extensions; - var stringUtils = (function () { - function stringUtils() { - } - stringUtils.GetLength = function (strObject) { - var res = 0; - if (!extensions.IsNullOrUndefined(strObject)) { - var stringified = ""; - try { - stringified = strObject.toString(); - } - catch (ex) { - } - res = stringified.length; - res = isNaN(res) ? 0 : res; - } - return res; - }; - return stringUtils; - })(); - ApplicationInsights.stringUtils = stringUtils; - var dateTime = (function () { - function dateTime() { - } - dateTime.Now = (window.performance && window.performance.now) ? - function () { - return performance.now(); - } - : - function () { - return new Date().getTime(); - }; - dateTime.GetDuration = function (start, end) { - var result = null; - if (start !== 0 && end !== 0 && !extensions.IsNullOrUndefined(start) && !extensions.IsNullOrUndefined(end)) { - result = end - start; - } - return result; - }; - return dateTime; - })(); - ApplicationInsights.dateTime = dateTime; - var EventHelper = (function () { - function EventHelper() { - } - EventHelper.AttachEvent = function (obj, eventNameWithoutOn, handlerRef) { - var result = false; - if (!extensions.IsNullOrUndefined(obj)) { - if (!extensions.IsNullOrUndefined(obj.attachEvent)) { - obj.attachEvent("on" + eventNameWithoutOn, handlerRef); - result = true; - } - else { - if (!extensions.IsNullOrUndefined(obj.addEventListener)) { - obj.addEventListener(eventNameWithoutOn, handlerRef, false); - result = true; - } - } - } - return result; - }; - EventHelper.DetachEvent = function (obj, eventNameWithoutOn, handlerRef) { - if (!extensions.IsNullOrUndefined(obj)) { - if (!extensions.IsNullOrUndefined(obj.detachEvent)) { - obj.detachEvent("on" + eventNameWithoutOn, handlerRef); - } - else { - if (!extensions.IsNullOrUndefined(obj.removeEventListener)) { - obj.removeEventListener(eventNameWithoutOn, handlerRef, false); - } - } - } - }; - return EventHelper; - })(); - ApplicationInsights.EventHelper = EventHelper; - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -/// -/// -/// -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - "use strict"; - var XHRMonitoringState = (function () { - function XHRMonitoringState() { - this.openDone = false; - this.setRequestHeaderDone = false; - this.sendDone = false; - this.abortDone = false; - this.onreadystatechangeCallbackAttached = false; - } - return XHRMonitoringState; - })(); - ApplicationInsights.XHRMonitoringState = XHRMonitoringState; - var ajaxRecord = (function () { - function ajaxRecord(id) { - this.completed = false; - this.requestHeadersSize = null; - this.ttfb = null; - this.responseReceivingDuration = null; - this.callbackDuration = null; - this.ajaxTotalDuration = null; - this.aborted = null; - this.pageUrl = null; - this.requestUrl = null; - this.requestSize = 0; - this.method = null; - this.status = null; - this.requestSentTime = null; - this.responseStartedTime = null; - this.responseFinishedTime = null; - this.callbackFinishedTime = null; - this.endTime = null; - this.originalOnreadystatechage = null; - this.xhrMonitoringState = new XHRMonitoringState(); - this.clientFailure = 0; - this.CalculateMetrics = function () { - var self = this; - self.ajaxTotalDuration = ApplicationInsights.dateTime.GetDuration(self.requestSentTime, self.responseFinishedTime); - }; - this.id = id; - } - ajaxRecord.prototype.getAbsoluteUrl = function () { - return this.requestUrl ? ApplicationInsights.UrlHelper.getAbsoluteUrl(this.requestUrl) : null; - }; - ajaxRecord.prototype.getPathName = function () { - return this.requestUrl ? ApplicationInsights.UrlHelper.getPathName(this.requestUrl) : null; - }; - return ajaxRecord; - })(); - ApplicationInsights.ajaxRecord = ajaxRecord; - ; - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -; -/// -/// -/// -/// -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - "use strict"; - var AjaxMonitor = (function () { - function AjaxMonitor(appInsights) { - this.currentWindowHost = window.location.host; - this.appInsights = appInsights; - this.initialized = false; - this.Init(); - } - AjaxMonitor.prototype.Init = function () { - if (this.supportsMonitoring()) { - this.instrumentOpen(); - this.instrumentSend(); - this.instrumentAbort(); - this.initialized = true; - } - }; - AjaxMonitor.prototype.isMonitoredInstance = function (xhr, excludeAjaxDataValidation) { - return this.initialized - && (excludeAjaxDataValidation === true || !ApplicationInsights.extensions.IsNullOrUndefined(xhr.ajaxData)) - && xhr[AjaxMonitor.DisabledPropertyName] !== true; - }; - AjaxMonitor.prototype.supportsMonitoring = function () { - var result = false; - if (!ApplicationInsights.extensions.IsNullOrUndefined(XMLHttpRequest)) { - result = true; - } - return result; - }; - AjaxMonitor.prototype.instrumentOpen = function () { - var originalOpen = XMLHttpRequest.prototype.open; - var ajaxMonitorInstance = this; - XMLHttpRequest.prototype.open = function (method, url, async) { - try { - if (ajaxMonitorInstance.isMonitoredInstance(this, true) && - (!this.ajaxData || - !this.ajaxData.xhrMonitoringState.openDone)) { - ajaxMonitorInstance.openHandler(this, method, url, async); - } - } - catch (e) { - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.CRITICAL, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_FailedMonitorAjaxOpen, "Failed to monitor XMLHttpRequest.open, monitoring data for this ajax call may be incorrect.", { - ajaxDiagnosticsMessage: AjaxMonitor.getFailedAjaxDiagnosticsMessage(this), - exception: Microsoft.ApplicationInsights.Util.dump(e) - })); - } - return originalOpen.apply(this, arguments); - }; - }; - AjaxMonitor.prototype.openHandler = function (xhr, method, url, async) { - var ajaxData = new ApplicationInsights.ajaxRecord(ApplicationInsights.Util.newId()); - ajaxData.method = method; - ajaxData.requestUrl = url; - ajaxData.xhrMonitoringState.openDone = true; - xhr.ajaxData = ajaxData; - this.attachToOnReadyStateChange(xhr); - }; - AjaxMonitor.getFailedAjaxDiagnosticsMessage = function (xhr) { - var result = ""; - try { - if (!ApplicationInsights.extensions.IsNullOrUndefined(xhr) && - !ApplicationInsights.extensions.IsNullOrUndefined(xhr.ajaxData) && - !ApplicationInsights.extensions.IsNullOrUndefined(xhr.ajaxData.requestUrl)) { - result += "(url: '" + xhr.ajaxData.requestUrl + "')"; - } - } - catch (e) { } - return result; - }; - AjaxMonitor.prototype.instrumentSend = function () { - var originalSend = XMLHttpRequest.prototype.send; - var ajaxMonitorInstance = this; - XMLHttpRequest.prototype.send = function (content) { - try { - if (ajaxMonitorInstance.isMonitoredInstance(this) && !this.ajaxData.xhrMonitoringState.sendDone) { - ajaxMonitorInstance.sendHandler(this, content); - } - } - catch (e) { - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.CRITICAL, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_FailedMonitorAjaxSend, "Failed to monitor XMLHttpRequest, monitoring data for this ajax call may be incorrect.", { - ajaxDiagnosticsMessage: AjaxMonitor.getFailedAjaxDiagnosticsMessage(this), - exception: Microsoft.ApplicationInsights.Util.dump(e) - })); - } - return originalSend.apply(this, arguments); - }; - }; - AjaxMonitor.prototype.sendHandler = function (xhr, content) { - xhr.ajaxData.requestSentTime = ApplicationInsights.dateTime.Now(); - if (!this.appInsights.config.disableCorrelationHeaders && (ApplicationInsights.UrlHelper.parseUrl(xhr.ajaxData.getAbsoluteUrl()).host == this.currentWindowHost)) { - xhr.setRequestHeader("x-ms-request-id", xhr.ajaxData.id); - } - xhr.ajaxData.xhrMonitoringState.sendDone = true; - }; - AjaxMonitor.prototype.instrumentAbort = function () { - var originalAbort = XMLHttpRequest.prototype.abort; - var ajaxMonitorInstance = this; - XMLHttpRequest.prototype.abort = function () { - try { - if (ajaxMonitorInstance.isMonitoredInstance(this) && !this.ajaxData.xhrMonitoringState.abortDone) { - this.ajaxData.aborted = 1; - this.ajaxData.xhrMonitoringState.abortDone = true; - } - } - catch (e) { - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.CRITICAL, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_FailedMonitorAjaxAbort, "Failed to monitor XMLHttpRequest.abort, monitoring data for this ajax call may be incorrect.", { - ajaxDiagnosticsMessage: AjaxMonitor.getFailedAjaxDiagnosticsMessage(this), - exception: Microsoft.ApplicationInsights.Util.dump(e) - })); - } - return originalAbort.apply(this, arguments); - }; - }; - AjaxMonitor.prototype.attachToOnReadyStateChange = function (xhr) { - var ajaxMonitorInstance = this; - xhr.ajaxData.xhrMonitoringState.onreadystatechangeCallbackAttached = ApplicationInsights.EventHelper.AttachEvent(xhr, "readystatechange", function () { - try { - if (ajaxMonitorInstance.isMonitoredInstance(xhr)) { - if (xhr.readyState === 4) { - ajaxMonitorInstance.onAjaxComplete(xhr); - } - } - } - catch (e) { - var exceptionText = Microsoft.ApplicationInsights.Util.dump(e); - if (!exceptionText || exceptionText.toLowerCase().indexOf("c00c023f") == -1) { - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.CRITICAL, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_FailedMonitorAjaxRSC, "Failed to monitor XMLHttpRequest 'readystatechange' event handler, monitoring data for this ajax call may be incorrect.", { - ajaxDiagnosticsMessage: AjaxMonitor.getFailedAjaxDiagnosticsMessage(xhr), - exception: Microsoft.ApplicationInsights.Util.dump(e) - })); - } - } - }); - }; - AjaxMonitor.prototype.onAjaxComplete = function (xhr) { - xhr.ajaxData.responseFinishedTime = ApplicationInsights.dateTime.Now(); - xhr.ajaxData.status = xhr.status; - xhr.ajaxData.CalculateMetrics(); - if (xhr.ajaxData.ajaxTotalDuration < 0) { - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.WARNING, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_FailedMonitorAjaxDur, "Failed to calculate the duration of the ajax call, monitoring data for this ajax call won't be sent.", { - ajaxDiagnosticsMessage: AjaxMonitor.getFailedAjaxDiagnosticsMessage(xhr), - requestSentTime: xhr.ajaxData.requestSentTime, - responseFinishedTime: xhr.ajaxData.responseFinishedTime - })); - } - else { - this.appInsights.trackAjax(xhr.ajaxData.id, xhr.ajaxData.getAbsoluteUrl(), xhr.ajaxData.getPathName(), xhr.ajaxData.ajaxTotalDuration, (+(xhr.ajaxData.status)) >= 200 && (+(xhr.ajaxData.status)) < 400, +xhr.ajaxData.status); - xhr.ajaxData = null; - } - }; - AjaxMonitor.instrumentedByAppInsightsName = "InstrumentedByAppInsights"; - AjaxMonitor.DisabledPropertyName = "Microsoft_ApplicationInsights_BypassAjaxInstrumentation"; - return AjaxMonitor; - })(); - ApplicationInsights.AjaxMonitor = AjaxMonitor; - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - var HashCodeScoreGenerator = (function () { - function HashCodeScoreGenerator() { - } - HashCodeScoreGenerator.prototype.getHashCodeScore = function (key) { - var score = this.getHashCode(key) / HashCodeScoreGenerator.INT_MAX_VALUE; - return score * 100; - }; - HashCodeScoreGenerator.prototype.getHashCode = function (input) { - if (input == "") { - return 0; - } - while (input.length < HashCodeScoreGenerator.MIN_INPUT_LENGTH) { - input = input.concat(input); - } - var hash = 5381; - for (var i = 0; i < input.length; ++i) { - hash = ((hash << 5) + hash) + input.charCodeAt(i); - hash = hash & hash; - } - return Math.abs(hash); - }; - HashCodeScoreGenerator.INT_MAX_VALUE = 2147483647; - HashCodeScoreGenerator.MIN_INPUT_LENGTH = 8; - return HashCodeScoreGenerator; - })(); - ApplicationInsights.HashCodeScoreGenerator = HashCodeScoreGenerator; - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -/// -/// -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - "use strict"; - (function (FieldType) { - FieldType[FieldType["Default"] = 0] = "Default"; - FieldType[FieldType["Required"] = 1] = "Required"; - FieldType[FieldType["Array"] = 2] = "Array"; - FieldType[FieldType["Hidden"] = 4] = "Hidden"; - })(ApplicationInsights.FieldType || (ApplicationInsights.FieldType = {})); - var FieldType = ApplicationInsights.FieldType; - ; - var Serializer = (function () { - function Serializer() { - } - Serializer.serialize = function (input) { - var output = Serializer._serializeObject(input, "root"); - return JSON.stringify(output); - }; - Serializer._serializeObject = function (source, name) { - var circularReferenceCheck = "__aiCircularRefCheck"; - var output = {}; - if (!source) { - ApplicationInsights._InternalLogging.throwInternalUserActionable(ApplicationInsights.LoggingSeverity.CRITICAL, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.USRACT_CannotSerializeObject, "cannot serialize object because it is null or undefined", { name: name })); - return output; - } - if (source[circularReferenceCheck]) { - ApplicationInsights._InternalLogging.throwInternalUserActionable(ApplicationInsights.LoggingSeverity.WARNING, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.USRACT_CircularReferenceDetected, "Circular reference detected while serializing object", { name: name })); - return output; - } - if (!source.aiDataContract) { - if (name === "measurements") { - output = Serializer._serializeStringMap(source, "number", name); - } - else if (name === "properties") { - output = Serializer._serializeStringMap(source, "string", name); - } - else if (name === "tags") { - output = Serializer._serializeStringMap(source, "string", name); - } - else if (ApplicationInsights.Util.isArray(source)) { - output = Serializer._serializeArray(source, name); - } - else { - ApplicationInsights._InternalLogging.throwInternalUserActionable(ApplicationInsights.LoggingSeverity.WARNING, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.USRACT_CannotSerializeObjectNonSerializable, "Attempting to serialize an object which does not implement ISerializable", { name: name })); - try { - JSON.stringify(source); - output = source; - } - catch (e) { - ApplicationInsights._InternalLogging.throwInternalUserActionable(ApplicationInsights.LoggingSeverity.CRITICAL, e && typeof e.toString === 'function' ? e.toString() : "Error serializing object"); - } - } - return output; - } - source[circularReferenceCheck] = true; - for (var field in source.aiDataContract) { - var contract = source.aiDataContract[field]; - var isRequired = (typeof contract === "function") ? (contract() & FieldType.Required) : (contract & FieldType.Required); - var isHidden = (typeof contract === "function") ? (contract() & FieldType.Hidden) : (contract & FieldType.Hidden); - var isArray = contract & FieldType.Array; - var isPresent = source[field] !== undefined; - var isObject = typeof source[field] === "object" && source[field] !== null; - if (isRequired && !isPresent && !isArray) { - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.CRITICAL, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_MissingRequiredFieldSpecification, "Missing required field specification. The field is required but not present on source", { field: field, name: name })); - continue; - } - if (isHidden) { - continue; - } - var value; - if (isObject) { - if (isArray) { - value = Serializer._serializeArray(source[field], field); - } - else { - value = Serializer._serializeObject(source[field], field); - } - } - else { - value = source[field]; - } - if (value !== undefined) { - output[field] = value; - } - } - delete source[circularReferenceCheck]; - return output; - }; - Serializer._serializeArray = function (sources, name) { - var output = undefined; - if (!!sources) { - if (!ApplicationInsights.Util.isArray(sources)) { - ApplicationInsights._InternalLogging.throwInternalUserActionable(ApplicationInsights.LoggingSeverity.CRITICAL, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.USRACT_ItemNotInArray, "This field was specified as an array in the contract but the item is not an array.\r\n", { name: name })); - } - else { - output = []; - for (var i = 0; i < sources.length; i++) { - var source = sources[i]; - var item = Serializer._serializeObject(source, name + "[" + i + "]"); - output.push(item); - } - } - } - return output; - }; - Serializer._serializeStringMap = function (map, expectedType, name) { - var output = undefined; - if (map) { - output = {}; - for (var field in map) { - var value = map[field]; - if (expectedType === "string") { - if (value === undefined) { - output[field] = "undefined"; - } - else if (value === null) { - output[field] = "null"; - } - else if (!value.toString) { - output[field] = "invalid field: toString() is not defined."; - } - else { - output[field] = value.toString(); - } - } - else if (expectedType === "number") { - if (value === undefined) { - output[field] = "undefined"; - } - else if (value === null) { - output[field] = "null"; - } - else { - var num = parseFloat(value); - if (isNaN(num)) { - output[field] = "NaN"; - } - else { - output[field] = num; - } - } - } - else { - output[field] = "invalid field: " + name + " is of unknown type."; - ApplicationInsights._InternalLogging.throwInternalUserActionable(ApplicationInsights.LoggingSeverity.CRITICAL, output[field]); - } - } - } - return output; - }; - return Serializer; - })(); - ApplicationInsights.Serializer = Serializer; - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -var Microsoft; -(function (Microsoft) { - var Telemetry; - (function (Telemetry) { - "use strict"; - var Base = (function () { - function Base() { - } - return Base; - })(); - Telemetry.Base = Base; - })(Telemetry = Microsoft.Telemetry || (Microsoft.Telemetry = {})); -})(Microsoft || (Microsoft = {})); -/// -var Microsoft; -(function (Microsoft) { - var Telemetry; - (function (Telemetry) { - "use strict"; - var Envelope = (function () { - function Envelope() { - this.ver = 1; - this.sampleRate = 100.0; - this.tags = {}; - } - return Envelope; - })(); - Telemetry.Envelope = Envelope; - })(Telemetry = Microsoft.Telemetry || (Microsoft.Telemetry = {})); -})(Microsoft || (Microsoft = {})); -/// -/// -/// -var __extends = (this && this.__extends) || function (d, b) { - for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p]; - function __() { this.constructor = d; } - __.prototype = b.prototype; - d.prototype = new __(); -}; -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - var Telemetry; - (function (Telemetry) { - var Common; - (function (Common) { - "use strict"; - var Envelope = (function (_super) { - __extends(Envelope, _super); - function Envelope(data, name) { - var _this = this; - _super.call(this); - this.name = name; - this.data = data; - this.time = ApplicationInsights.Util.toISOStringForIE8(new Date()); - this.aiDataContract = { - time: ApplicationInsights.FieldType.Required, - iKey: ApplicationInsights.FieldType.Required, - name: ApplicationInsights.FieldType.Required, - sampleRate: function () { - return (_this.sampleRate == 100) ? ApplicationInsights.FieldType.Hidden : ApplicationInsights.FieldType.Required; - }, - tags: ApplicationInsights.FieldType.Required, - data: ApplicationInsights.FieldType.Required - }; - } - return Envelope; - })(Microsoft.Telemetry.Envelope); - Common.Envelope = Envelope; - })(Common = Telemetry.Common || (Telemetry.Common = {})); - })(Telemetry = ApplicationInsights.Telemetry || (ApplicationInsights.Telemetry = {})); - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -/// -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - var Telemetry; - (function (Telemetry) { - var Common; - (function (Common) { - "use strict"; - var Base = (function (_super) { - __extends(Base, _super); - function Base() { - _super.apply(this, arguments); - this.aiDataContract = {}; - } - return Base; - })(Microsoft.Telemetry.Base); - Common.Base = Base; - })(Common = Telemetry.Common || (Telemetry.Common = {})); - })(Telemetry = ApplicationInsights.Telemetry || (ApplicationInsights.Telemetry = {})); - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -var AI; -(function (AI) { - "use strict"; - var ContextTagKeys = (function () { - function ContextTagKeys() { - this.applicationVersion = "ai.application.ver"; - this.applicationBuild = "ai.application.build"; - this.applicationTypeId = "ai.application.typeId"; - this.applicationId = "ai.application.applicationId"; - this.deviceId = "ai.device.id"; - this.deviceIp = "ai.device.ip"; - this.deviceLanguage = "ai.device.language"; - this.deviceLocale = "ai.device.locale"; - this.deviceModel = "ai.device.model"; - this.deviceNetwork = "ai.device.network"; - this.deviceNetworkName = "ai.device.networkName"; - this.deviceOEMName = "ai.device.oemName"; - this.deviceOS = "ai.device.os"; - this.deviceOSVersion = "ai.device.osVersion"; - this.deviceRoleInstance = "ai.device.roleInstance"; - this.deviceRoleName = "ai.device.roleName"; - this.deviceScreenResolution = "ai.device.screenResolution"; - this.deviceType = "ai.device.type"; - this.deviceMachineName = "ai.device.machineName"; - this.deviceVMName = "ai.device.vmName"; - this.locationIp = "ai.location.ip"; - this.operationId = "ai.operation.id"; - this.operationName = "ai.operation.name"; - this.operationParentId = "ai.operation.parentId"; - this.operationRootId = "ai.operation.rootId"; - this.operationSyntheticSource = "ai.operation.syntheticSource"; - this.operationIsSynthetic = "ai.operation.isSynthetic"; - this.operationCorrelationVector = "ai.operation.correlationVector"; - this.sessionId = "ai.session.id"; - this.sessionIsFirst = "ai.session.isFirst"; - this.sessionIsNew = "ai.session.isNew"; - this.userAccountAcquisitionDate = "ai.user.accountAcquisitionDate"; - this.userAccountId = "ai.user.accountId"; - this.userAgent = "ai.user.userAgent"; - this.userId = "ai.user.id"; - this.userStoreRegion = "ai.user.storeRegion"; - this.userAuthUserId = "ai.user.authUserId"; - this.userAnonymousUserAcquisitionDate = "ai.user.anonUserAcquisitionDate"; - this.userAuthenticatedUserAcquisitionDate = "ai.user.authUserAcquisitionDate"; - this.sampleRate = "ai.sample.sampleRate"; - this.cloudName = "ai.cloud.name"; - this.cloudRoleVer = "ai.cloud.roleVer"; - this.cloudEnvironment = "ai.cloud.environment"; - this.cloudLocation = "ai.cloud.location"; - this.cloudDeploymentUnit = "ai.cloud.deploymentUnit"; - this.serverDeviceOS = "ai.serverDevice.os"; - this.serverDeviceOSVer = "ai.serverDevice.osVer"; - this.internalSdkVersion = "ai.internal.sdkVersion"; - this.internalAgentVersion = "ai.internal.agentVersion"; - this.internalDataCollectorReceivedTime = "ai.internal.dataCollectorReceivedTime"; - this.internalProfileId = "ai.internal.profileId"; - this.internalProfileClassId = "ai.internal.profileClassId"; - this.internalAccountId = "ai.internal.accountId"; - this.internalApplicationName = "ai.internal.applicationName"; - this.internalInstrumentationKey = "ai.internal.instrumentationKey"; - this.internalTelemetryItemId = "ai.internal.telemetryItemId"; - this.internalApplicationType = "ai.internal.applicationType"; - this.internalRequestSource = "ai.internal.requestSource"; - this.internalFlowType = "ai.internal.flowType"; - this.internalIsAudit = "ai.internal.isAudit"; - this.internalTrackingSourceId = "ai.internal.trackingSourceId"; - this.internalTrackingType = "ai.internal.trackingType"; - this.internalIsDiagnosticExample = "ai.internal.isDiagnosticExample"; - } - return ContextTagKeys; - })(); - AI.ContextTagKeys = ContextTagKeys; -})(AI || (AI = {})); -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - var Context; - (function (Context) { - "use strict"; - var Application = (function () { - function Application() { - } - return Application; - })(); - Context.Application = Application; - })(Context = ApplicationInsights.Context || (ApplicationInsights.Context = {})); - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - var Context; - (function (Context) { - "use strict"; - var Device = (function () { - function Device() { - this.id = "browser"; - this.type = "Browser"; - } - return Device; - })(); - Context.Device = Device; - })(Context = ApplicationInsights.Context || (ApplicationInsights.Context = {})); - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - var Context; - (function (Context) { - "use strict"; - var Internal = (function () { - function Internal() { - this.sdkVersion = "JavaScript:" + ApplicationInsights.Version; - } - return Internal; - })(); - Context.Internal = Internal; - })(Context = ApplicationInsights.Context || (ApplicationInsights.Context = {})); - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - var Context; - (function (Context) { - "use strict"; - var Location = (function () { - function Location() { - } - return Location; - })(); - Context.Location = Location; - })(Context = ApplicationInsights.Context || (ApplicationInsights.Context = {})); - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -/// -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - var Context; - (function (Context) { - "use strict"; - var Operation = (function () { - function Operation() { - this.id = ApplicationInsights.Util.newId(); - if (window && window.location && window.location.pathname) { - this.name = window.location.pathname; - } - } - return Operation; - })(); - Context.Operation = Operation; - })(Context = ApplicationInsights.Context || (ApplicationInsights.Context = {})); - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -/// -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - var SamplingScoreGenerator = (function () { - function SamplingScoreGenerator() { - this.hashCodeGeneragor = new ApplicationInsights.HashCodeScoreGenerator(); - } - SamplingScoreGenerator.prototype.getSamplingScore = function (envelope) { - var tagKeys = new AI.ContextTagKeys(); - var score = 0; - if (envelope.tags[tagKeys.userId]) { - score = this.hashCodeGeneragor.getHashCodeScore(envelope.tags[tagKeys.userId]); - } - else if (envelope.tags[tagKeys.operationId]) { - score = this.hashCodeGeneragor.getHashCodeScore(envelope.tags[tagKeys.operationId]); - } - else { - score = Math.random(); - } - return score; - }; - return SamplingScoreGenerator; - })(); - ApplicationInsights.SamplingScoreGenerator = SamplingScoreGenerator; - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -/// -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - var Context; - (function (Context) { - "use strict"; - var Sample = (function () { - function Sample(sampleRate) { - this.INT_MAX_VALUE = 2147483647; - if (sampleRate > 100 || sampleRate < 0) { - ApplicationInsights._InternalLogging.throwInternalUserActionable(ApplicationInsights.LoggingSeverity.WARNING, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.USRACT_SampleRateOutOfRange, "Sampling rate is out of range (0..100). Sampling will be disabled, you may be sending too much data which may affect your AI service level.", { samplingRate: sampleRate })); - this.sampleRate = 100; - } - this.sampleRate = sampleRate; - this.samplingScoreGenerator = new ApplicationInsights.SamplingScoreGenerator(); - } - Sample.prototype.isSampledIn = function (envelope) { - if (this.sampleRate == 100) - return true; - var score = this.samplingScoreGenerator.getSamplingScore(envelope); - return score < this.sampleRate; - }; - return Sample; - })(); - Context.Sample = Sample; - })(Context = ApplicationInsights.Context || (ApplicationInsights.Context = {})); - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -var AI; -(function (AI) { - "use strict"; - (function (SessionState) { - SessionState[SessionState["Start"] = 0] = "Start"; - SessionState[SessionState["End"] = 1] = "End"; - })(AI.SessionState || (AI.SessionState = {})); - var SessionState = AI.SessionState; -})(AI || (AI = {})); -/// -/// -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - var Context; - (function (Context) { - "use strict"; - var Session = (function () { - function Session() { - } - return Session; - })(); - Context.Session = Session; - var _SessionManager = (function () { - function _SessionManager(config) { - if (!config) { - config = {}; - } - if (!(typeof config.sessionExpirationMs === "function")) { - config.sessionExpirationMs = function () { return _SessionManager.acquisitionSpan; }; - } - if (!(typeof config.sessionRenewalMs === "function")) { - config.sessionRenewalMs = function () { return _SessionManager.renewalSpan; }; - } - this.config = config; - this.automaticSession = new Session(); - } - _SessionManager.prototype.update = function () { - if (!this.automaticSession.id) { - this.initializeAutomaticSession(); - } - var now = +new Date; - var acquisitionExpired = now - this.automaticSession.acquisitionDate > this.config.sessionExpirationMs(); - var renewalExpired = now - this.automaticSession.renewalDate > this.config.sessionRenewalMs(); - if (acquisitionExpired || renewalExpired) { - this.automaticSession.isFirst = undefined; - this.renew(); - } - else { - this.automaticSession.renewalDate = +new Date; - this.setCookie(this.automaticSession.id, this.automaticSession.acquisitionDate, this.automaticSession.renewalDate); - } - }; - _SessionManager.prototype.backup = function () { - this.setStorage(this.automaticSession.id, this.automaticSession.acquisitionDate, this.automaticSession.renewalDate); - }; - _SessionManager.prototype.initializeAutomaticSession = function () { - var cookie = ApplicationInsights.Util.getCookie('ai_session'); - if (cookie && typeof cookie.split === "function") { - this.initializeAutomaticSessionWithData(cookie); - } - else { - var storage = ApplicationInsights.Util.getStorage('ai_session'); - if (storage) { - this.initializeAutomaticSessionWithData(storage); - } - } - if (!this.automaticSession.id) { - this.automaticSession.isFirst = true; - this.renew(); - } - }; - _SessionManager.prototype.initializeAutomaticSessionWithData = function (sessionData) { - var params = sessionData.split("|"); - if (params.length > 0) { - this.automaticSession.id = params[0]; - } - try { - if (params.length > 1) { - var acq = +params[1]; - this.automaticSession.acquisitionDate = +new Date(acq); - this.automaticSession.acquisitionDate = this.automaticSession.acquisitionDate > 0 ? this.automaticSession.acquisitionDate : 0; - } - if (params.length > 2) { - var renewal = +params[2]; - this.automaticSession.renewalDate = +new Date(renewal); - this.automaticSession.renewalDate = this.automaticSession.renewalDate > 0 ? this.automaticSession.renewalDate : 0; - } - } - catch (e) { - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.CRITICAL, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_ErrorParsingAISessionCookie, "Error parsing ai_session cookie, session will be reset: " + ApplicationInsights.Util.getExceptionName(e), { exception: ApplicationInsights.Util.dump(e) })); - } - if (this.automaticSession.renewalDate == 0) { - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.WARNING, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_SessionRenewalDateIsZero, "AI session renewal date is 0, session will be reset.")); - } - }; - _SessionManager.prototype.renew = function () { - var now = +new Date; - this.automaticSession.id = ApplicationInsights.Util.newId(); - this.automaticSession.acquisitionDate = now; - this.automaticSession.renewalDate = now; - this.setCookie(this.automaticSession.id, this.automaticSession.acquisitionDate, this.automaticSession.renewalDate); - if (!ApplicationInsights.Util.canUseLocalStorage()) { - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.WARNING, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_BrowserDoesNotSupportLocalStorage, "Browser does not support local storage. Session durations will be inaccurate.")); - } - }; - _SessionManager.prototype.setCookie = function (guid, acq, renewal) { - var acquisitionExpiry = acq + this.config.sessionExpirationMs(); - var renewalExpiry = renewal + this.config.sessionRenewalMs(); - var cookieExpiry = new Date(); - var cookie = [guid, acq, renewal]; - if (acquisitionExpiry < renewalExpiry) { - cookieExpiry.setTime(acquisitionExpiry); - } - else { - cookieExpiry.setTime(renewalExpiry); - } - var cookieDomnain = this.config.cookieDomain ? this.config.cookieDomain() : null; - ApplicationInsights.Util.setCookie('ai_session', cookie.join('|') + ';expires=' + cookieExpiry.toUTCString(), cookieDomnain); - }; - _SessionManager.prototype.setStorage = function (guid, acq, renewal) { - ApplicationInsights.Util.setStorage('ai_session', [guid, acq, renewal].join('|')); - }; - _SessionManager.acquisitionSpan = 86400000; - _SessionManager.renewalSpan = 1800000; - return _SessionManager; - })(); - Context._SessionManager = _SessionManager; - })(Context = ApplicationInsights.Context || (ApplicationInsights.Context = {})); - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -/// -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - var Context; - (function (Context) { - "use strict"; - var User = (function () { - function User(config) { - var cookie = ApplicationInsights.Util.getCookie(User.userCookieName); - if (cookie) { - var params = cookie.split(User.cookieSeparator); - if (params.length > 0) { - this.id = params[0]; - } - } - this.config = config; - if (!this.id) { - this.id = ApplicationInsights.Util.newId(); - var date = new Date(); - var acqStr = ApplicationInsights.Util.toISOStringForIE8(date); - this.accountAcquisitionDate = acqStr; - date.setTime(date.getTime() + 31536000000); - var newCookie = [this.id, acqStr]; - var cookieDomain = this.config.cookieDomain ? this.config.cookieDomain() : undefined; - ApplicationInsights.Util.setCookie(User.userCookieName, newCookie.join(User.cookieSeparator) + ';expires=' + date.toUTCString(), cookieDomain); - ApplicationInsights.Util.removeStorage('ai_session'); - } - this.accountId = config.accountId ? config.accountId() : undefined; - var authCookie = ApplicationInsights.Util.getCookie(User.authUserCookieName); - if (authCookie) { - authCookie = decodeURI(authCookie); - var authCookieString = authCookie.split(User.cookieSeparator); - if (authCookieString[0]) { - this.authenticatedId = authCookieString[0]; - } - if (authCookieString.length > 1 && authCookieString[1]) { - this.accountId = authCookieString[1]; - } - } - } - User.prototype.setAuthenticatedUserContext = function (authenticatedUserId, accountId) { - var isInvalidInput = !this.validateUserInput(authenticatedUserId) || (accountId && !this.validateUserInput(accountId)); - if (isInvalidInput) { - ApplicationInsights._InternalLogging.throwInternalUserActionable(ApplicationInsights.LoggingSeverity.WARNING, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.USRACT_SetAuthContextFailedAccountName, "Setting auth user context failed. " + - "User auth/account id should be of type string, and not contain commas, semi-colons, equal signs, spaces, or vertical-bars.")); - return; - } - this.authenticatedId = authenticatedUserId; - var authCookie = this.authenticatedId; - if (accountId) { - this.accountId = accountId; - authCookie = [this.authenticatedId, this.accountId].join(User.cookieSeparator); - } - ApplicationInsights.Util.setCookie(User.authUserCookieName, encodeURI(authCookie), this.config.cookieDomain()); - }; - User.prototype.clearAuthenticatedUserContext = function () { - this.authenticatedId = null; - this.accountId = null; - ApplicationInsights.Util.deleteCookie(User.authUserCookieName); - }; - User.prototype.validateUserInput = function (id) { - if (typeof id !== 'string' || - !id || - id.match(/,|;|=| |\|/)) { - return false; - } - return true; - }; - User.cookieSeparator = '|'; - User.userCookieName = 'ai_user'; - User.authUserCookieName = 'ai_authUser'; - return User; - })(); - Context.User = User; - })(Context = ApplicationInsights.Context || (ApplicationInsights.Context = {})); - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - "use strict"; - var DataLossAnalyzer = (function () { - function DataLossAnalyzer() { - } - DataLossAnalyzer.reset = function () { - if (DataLossAnalyzer.isEnabled()) { - ApplicationInsights.Util.setSessionStorage(DataLossAnalyzer.ITEMS_QUEUED_KEY, "0"); - } - }; - DataLossAnalyzer.isEnabled = function () { - return DataLossAnalyzer.enabled && - DataLossAnalyzer.appInsights != null && - ApplicationInsights.Util.canUseSessionStorage(); - }; - DataLossAnalyzer.getIssuesReported = function () { - var result = (!DataLossAnalyzer.isEnabled() || isNaN(+ApplicationInsights.Util.getSessionStorage(DataLossAnalyzer.ISSUES_REPORTED_KEY))) ? - 0 : - +ApplicationInsights.Util.getSessionStorage(DataLossAnalyzer.ISSUES_REPORTED_KEY); - return result; - }; - DataLossAnalyzer.incrementItemsQueued = function () { - try { - if (DataLossAnalyzer.isEnabled()) { - var itemsQueued = DataLossAnalyzer.getNumberOfLostItems(); - ++itemsQueued; - ApplicationInsights.Util.setSessionStorage(DataLossAnalyzer.ITEMS_QUEUED_KEY, itemsQueued.toString()); - } - } - catch (e) { } - }; - DataLossAnalyzer.decrementItemsQueued = function (countOfItemsSentSuccessfully) { - try { - if (DataLossAnalyzer.isEnabled()) { - var itemsQueued = DataLossAnalyzer.getNumberOfLostItems(); - itemsQueued -= countOfItemsSentSuccessfully; - if (itemsQueued < 0) - itemsQueued = 0; - ApplicationInsights.Util.setSessionStorage(DataLossAnalyzer.ITEMS_QUEUED_KEY, itemsQueued.toString()); - } - } - catch (e) { } - }; - DataLossAnalyzer.getNumberOfLostItems = function () { - var result = 0; - try { - if (DataLossAnalyzer.isEnabled()) { - result = isNaN(+ApplicationInsights.Util.getSessionStorage(DataLossAnalyzer.ITEMS_QUEUED_KEY)) ? - 0 : - +ApplicationInsights.Util.getSessionStorage(DataLossAnalyzer.ITEMS_QUEUED_KEY); - } - } - catch (e) { - result = 0; - } - return result; - }; - DataLossAnalyzer.reportLostItems = function () { - try { - if (DataLossAnalyzer.isEnabled() && - DataLossAnalyzer.getIssuesReported() < DataLossAnalyzer.LIMIT_PER_SESSION && - DataLossAnalyzer.getNumberOfLostItems() > 0) { - DataLossAnalyzer.appInsights.trackTrace("AI (Internal): Internal report DATALOSS: " - + DataLossAnalyzer.getNumberOfLostItems(), null); - DataLossAnalyzer.appInsights.flush(); - var issuesReported = DataLossAnalyzer.getIssuesReported(); - ++issuesReported; - ApplicationInsights.Util.setSessionStorage(DataLossAnalyzer.ISSUES_REPORTED_KEY, issuesReported.toString()); - } - } - catch (e) { - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.CRITICAL, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_FailedToReportDataLoss, "Failed to report data loss: " + ApplicationInsights.Util.getExceptionName(e), { exception: ApplicationInsights.Util.dump(e) })); - } - finally { - try { - DataLossAnalyzer.reset(); - } - catch (e) { } - } - }; - DataLossAnalyzer.enabled = false; - DataLossAnalyzer.LIMIT_PER_SESSION = 10; - DataLossAnalyzer.ITEMS_QUEUED_KEY = "AI_itemsQueued"; - DataLossAnalyzer.ISSUES_REPORTED_KEY = "AI_lossIssuesReported"; - return DataLossAnalyzer; - })(); - ApplicationInsights.DataLossAnalyzer = DataLossAnalyzer; - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -/// -/// -/// -/// -/// -/// -/// -/// -/// -/// -/// -/// -/// -/// -; -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - "use strict"; - var Sender = (function () { - function Sender(config) { - this._buffer = []; - this._lastSend = 0; - this._config = config; - this._sender = null; - if (typeof XMLHttpRequest != "undefined") { - var testXhr = new XMLHttpRequest(); - if ("withCredentials" in testXhr) { - this._sender = this._xhrSender; - } - else if (typeof XDomainRequest !== "undefined") { - this._sender = this._xdrSender; - } - } - } - Sender.prototype.send = function (envelope) { - var _this = this; - try { - if (this._config.disableTelemetry()) { - return; - } - if (!envelope) { - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.CRITICAL, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_CannotSendEmptyTelemetry, "Cannot send empty telemetry")); - return; - } - if (!this._sender) { - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.CRITICAL, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_SenderNotInitialized, "Sender was not initialized")); - return; - } - var payload = ApplicationInsights.Serializer.serialize(envelope); - if (this._getSizeInBytes(this._buffer) + payload.length > this._config.maxBatchSizeInBytes()) { - this.triggerSend(); - } - this._buffer.push(payload); - if (!this._timeoutHandle) { - this._timeoutHandle = setTimeout(function () { - _this._timeoutHandle = null; - _this.triggerSend(); - }, this._config.maxBatchInterval()); - } - ApplicationInsights.DataLossAnalyzer.incrementItemsQueued(); - } - catch (e) { - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.CRITICAL, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_FailedAddingTelemetryToBuffer, "Failed adding telemetry to the sender's buffer, some telemetry will be lost: " + ApplicationInsights.Util.getExceptionName(e), { exception: ApplicationInsights.Util.dump(e) })); - } - }; - Sender.prototype._getSizeInBytes = function (list) { - var size = 0; - if (list && list.length) { - for (var i = 0; i < list.length; i++) { - var item = list[i]; - if (item && item.length) { - size += item.length; - } - } - } - return size; - }; - Sender.prototype.triggerSend = function (async) { - var isAsync = true; - if (typeof async === 'boolean') { - isAsync = async; - } - try { - if (!this._config.disableTelemetry()) { - if (this._buffer.length) { - var batch = this._config.emitLineDelimitedJson() ? - this._buffer.join("\n") : - "[" + this._buffer.join(",") + "]"; - this._sender(batch, isAsync, this._buffer.length); - } - this._lastSend = +new Date; - } - this._buffer.length = 0; - clearTimeout(this._timeoutHandle); - this._timeoutHandle = null; - } - catch (e) { - if (!ApplicationInsights.Util.getIEVersion() || ApplicationInsights.Util.getIEVersion() > 9) { - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.CRITICAL, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_TransmissionFailed, "Telemetry transmission failed, some telemetry will be lost: " + ApplicationInsights.Util.getExceptionName(e), { exception: ApplicationInsights.Util.dump(e) })); - } - } - }; - Sender.prototype._xhrSender = function (payload, isAsync, countOfItemsInPayload) { - var xhr = new XMLHttpRequest(); - xhr[ApplicationInsights.AjaxMonitor.DisabledPropertyName] = true; - xhr.open("POST", this._config.endpointUrl(), isAsync); - xhr.setRequestHeader("Content-type", "application/json"); - xhr.onreadystatechange = function () { return Sender._xhrReadyStateChange(xhr, payload, countOfItemsInPayload); }; - xhr.onerror = function (event) { return Sender._onError(payload, xhr.responseText || xhr.response || "", event); }; - xhr.send(payload); - }; - Sender.prototype._xdrSender = function (payload, isAsync) { - var xdr = new XDomainRequest(); - xdr.onload = function () { return Sender._xdrOnLoad(xdr, payload); }; - xdr.onerror = function (event) { return Sender._onError(payload, xdr.responseText || "", event); }; - xdr.open('POST', this._config.endpointUrl()); - xdr.send(payload); - }; - Sender._xhrReadyStateChange = function (xhr, payload, countOfItemsInPayload) { - if (xhr.readyState === 4) { - if ((xhr.status < 200 || xhr.status >= 300) && xhr.status !== 0) { - Sender._onError(payload, xhr.responseText || xhr.response || ""); - } - else { - Sender._onSuccess(payload, countOfItemsInPayload); - } - } - }; - Sender._xdrOnLoad = function (xdr, payload) { - if (xdr && (xdr.responseText + "" === "200" || xdr.responseText === "")) { - Sender._onSuccess(payload, 0); - } - else { - Sender._onError(payload, xdr && xdr.responseText || ""); - } - }; - Sender._onError = function (payload, message, event) { - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.WARNING, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_OnError, "Failed to send telemetry.", { message: message })); - }; - Sender._onSuccess = function (payload, countOfItemsInPayload) { - ApplicationInsights.DataLossAnalyzer.decrementItemsQueued(countOfItemsInPayload); - }; - return Sender; - })(); - ApplicationInsights.Sender = Sender; - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -/// -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - "use strict"; - var SplitTest = (function () { - function SplitTest() { - this.hashCodeGeneragor = new ApplicationInsights.HashCodeScoreGenerator(); - } - SplitTest.prototype.isEnabled = function (key, percentEnabled) { - return this.hashCodeGeneragor.getHashCodeScore(key) < percentEnabled; - }; - return SplitTest; - })(); - ApplicationInsights.SplitTest = SplitTest; - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -var Microsoft; -(function (Microsoft) { - var Telemetry; - (function (Telemetry) { - "use strict"; - var Domain = (function () { - function Domain() { - } - return Domain; - })(); - Telemetry.Domain = Domain; - })(Telemetry = Microsoft.Telemetry || (Microsoft.Telemetry = {})); -})(Microsoft || (Microsoft = {})); -var AI; -(function (AI) { - "use strict"; - (function (SeverityLevel) { - SeverityLevel[SeverityLevel["Verbose"] = 0] = "Verbose"; - SeverityLevel[SeverityLevel["Information"] = 1] = "Information"; - SeverityLevel[SeverityLevel["Warning"] = 2] = "Warning"; - SeverityLevel[SeverityLevel["Error"] = 3] = "Error"; - SeverityLevel[SeverityLevel["Critical"] = 4] = "Critical"; - })(AI.SeverityLevel || (AI.SeverityLevel = {})); - var SeverityLevel = AI.SeverityLevel; -})(AI || (AI = {})); -/// -/// -var AI; -(function (AI) { - "use strict"; - var MessageData = (function (_super) { - __extends(MessageData, _super); - function MessageData() { - this.ver = 2; - this.properties = {}; - _super.call(this); - } - return MessageData; - })(Microsoft.Telemetry.Domain); - AI.MessageData = MessageData; -})(AI || (AI = {})); -/// -/// -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - var Telemetry; - (function (Telemetry) { - var Common; - (function (Common) { - "use strict"; - var DataSanitizer = (function () { - function DataSanitizer() { - } - DataSanitizer.sanitizeKeyAndAddUniqueness = function (key, map) { - var origLength = key.length; - var field = DataSanitizer.sanitizeKey(key); - if (field.length !== origLength) { - var i = 0; - var uniqueField = field; - while (map[uniqueField] !== undefined) { - i++; - uniqueField = field.substring(0, DataSanitizer.MAX_NAME_LENGTH - 3) + DataSanitizer.padNumber(i); - } - field = uniqueField; - } - return field; - }; - DataSanitizer.sanitizeKey = function (name) { - if (name) { - name = ApplicationInsights.Util.trim(name.toString()); - if (name.search(/[^0-9a-zA-Z-._()\/ ]/g) >= 0) { - name = name.replace(/[^0-9a-zA-Z-._()\/ ]/g, "_"); - ApplicationInsights._InternalLogging.throwInternalUserActionable(ApplicationInsights.LoggingSeverity.WARNING, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.USRACT_IllegalCharsInName, "name contains illegal characters. Illegal characters have been replaced with '_'.", { newName: name })); - } - if (name.length > DataSanitizer.MAX_NAME_LENGTH) { - name = name.substring(0, DataSanitizer.MAX_NAME_LENGTH); - ApplicationInsights._InternalLogging.throwInternalUserActionable(ApplicationInsights.LoggingSeverity.WARNING, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.USRACT_NameTooLong, "name is too long. It has been truncated to " + DataSanitizer.MAX_NAME_LENGTH + " characters.", { name: name })); - } - } - return name; - }; - DataSanitizer.sanitizeString = function (value) { - if (value) { - value = ApplicationInsights.Util.trim(value); - if (value.toString().length > DataSanitizer.MAX_STRING_LENGTH) { - value = value.toString().substring(0, DataSanitizer.MAX_STRING_LENGTH); - ApplicationInsights._InternalLogging.throwInternalUserActionable(ApplicationInsights.LoggingSeverity.WARNING, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.USRACT_StringValueTooLong, "string value is too long. It has been truncated to " + DataSanitizer.MAX_STRING_LENGTH + " characters.", { value: value })); - } - } - return value; - }; - DataSanitizer.sanitizeUrl = function (url) { - if (url) { - if (url.length > DataSanitizer.MAX_URL_LENGTH) { - url = url.substring(0, DataSanitizer.MAX_URL_LENGTH); - ApplicationInsights._InternalLogging.throwInternalUserActionable(ApplicationInsights.LoggingSeverity.WARNING, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.USRACT_UrlTooLong, "url is too long, it has been trucated to " + DataSanitizer.MAX_URL_LENGTH + " characters.", { url: url })); - } - } - return url; - }; - DataSanitizer.sanitizeMessage = function (message) { - if (message) { - if (message.length > DataSanitizer.MAX_MESSAGE_LENGTH) { - message = message.substring(0, DataSanitizer.MAX_MESSAGE_LENGTH); - ApplicationInsights._InternalLogging.throwInternalUserActionable(ApplicationInsights.LoggingSeverity.WARNING, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.USRACT_MessageTruncated, "message is too long, it has been trucated to " + DataSanitizer.MAX_MESSAGE_LENGTH + " characters.", { message: message })); - } - } - return message; - }; - DataSanitizer.sanitizeException = function (exception) { - if (exception) { - if (exception.length > DataSanitizer.MAX_EXCEPTION_LENGTH) { - exception = exception.substring(0, DataSanitizer.MAX_EXCEPTION_LENGTH); - ApplicationInsights._InternalLogging.throwInternalUserActionable(ApplicationInsights.LoggingSeverity.WARNING, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.USRACT_ExceptionTruncated, "exception is too long, it has been trucated to " + DataSanitizer.MAX_EXCEPTION_LENGTH + " characters.", { exception: exception })); - } - } - return exception; - }; - DataSanitizer.sanitizeProperties = function (properties) { - if (properties) { - var tempProps = {}; - for (var prop in properties) { - var value = DataSanitizer.sanitizeString(properties[prop]); - prop = DataSanitizer.sanitizeKeyAndAddUniqueness(prop, tempProps); - tempProps[prop] = value; - } - properties = tempProps; - } - return properties; - }; - DataSanitizer.sanitizeMeasurements = function (measurements) { - if (measurements) { - var tempMeasurements = {}; - for (var measure in measurements) { - var value = measurements[measure]; - measure = DataSanitizer.sanitizeKeyAndAddUniqueness(measure, tempMeasurements); - tempMeasurements[measure] = value; - } - measurements = tempMeasurements; - } - return measurements; - }; - DataSanitizer.padNumber = function (num) { - var s = "00" + num; - return s.substr(s.length - 3); - }; - DataSanitizer.MAX_NAME_LENGTH = 150; - DataSanitizer.MAX_STRING_LENGTH = 1024; - DataSanitizer.MAX_URL_LENGTH = 2048; - DataSanitizer.MAX_MESSAGE_LENGTH = 32768; - DataSanitizer.MAX_EXCEPTION_LENGTH = 32768; - return DataSanitizer; - })(); - Common.DataSanitizer = DataSanitizer; - })(Common = Telemetry.Common || (Telemetry.Common = {})); - })(Telemetry = ApplicationInsights.Telemetry || (ApplicationInsights.Telemetry = {})); - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -/// -/// -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - var Telemetry; - (function (Telemetry) { - "use strict"; - var Trace = (function (_super) { - __extends(Trace, _super); - function Trace(message, properties) { - _super.call(this); - this.aiDataContract = { - ver: ApplicationInsights.FieldType.Required, - message: ApplicationInsights.FieldType.Required, - severityLevel: ApplicationInsights.FieldType.Default, - measurements: ApplicationInsights.FieldType.Default, - properties: ApplicationInsights.FieldType.Default - }; - message = message || ApplicationInsights.Util.NotSpecified; - this.message = Telemetry.Common.DataSanitizer.sanitizeMessage(message); - this.properties = Telemetry.Common.DataSanitizer.sanitizeProperties(properties); - } - Trace.envelopeType = "Microsoft.ApplicationInsights.{0}.Message"; - Trace.dataType = "MessageData"; - return Trace; - })(AI.MessageData); - Telemetry.Trace = Trace; - })(Telemetry = ApplicationInsights.Telemetry || (ApplicationInsights.Telemetry = {})); - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -/// -var AI; -(function (AI) { - "use strict"; - var EventData = (function (_super) { - __extends(EventData, _super); - function EventData() { - this.ver = 2; - this.properties = {}; - this.measurements = {}; - _super.call(this); - } - return EventData; - })(Microsoft.Telemetry.Domain); - AI.EventData = EventData; -})(AI || (AI = {})); -/// -/// -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - var Telemetry; - (function (Telemetry) { - "use strict"; - var Event = (function (_super) { - __extends(Event, _super); - function Event(name, properties, measurements) { - _super.call(this); - this.aiDataContract = { - ver: ApplicationInsights.FieldType.Required, - name: ApplicationInsights.FieldType.Required, - properties: ApplicationInsights.FieldType.Default, - measurements: ApplicationInsights.FieldType.Default - }; - this.name = ApplicationInsights.Telemetry.Common.DataSanitizer.sanitizeString(name); - this.properties = ApplicationInsights.Telemetry.Common.DataSanitizer.sanitizeProperties(properties); - this.measurements = ApplicationInsights.Telemetry.Common.DataSanitizer.sanitizeMeasurements(measurements); - } - Event.envelopeType = "Microsoft.ApplicationInsights.{0}.Event"; - Event.dataType = "EventData"; - return Event; - })(AI.EventData); - Telemetry.Event = Event; - })(Telemetry = ApplicationInsights.Telemetry || (ApplicationInsights.Telemetry = {})); - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -var AI; -(function (AI) { - "use strict"; - var ExceptionDetails = (function () { - function ExceptionDetails() { - this.hasFullStack = true; - this.parsedStack = []; - } - return ExceptionDetails; - })(); - AI.ExceptionDetails = ExceptionDetails; -})(AI || (AI = {})); -/// -/// -/// -var AI; -(function (AI) { - "use strict"; - var ExceptionData = (function (_super) { - __extends(ExceptionData, _super); - function ExceptionData() { - this.ver = 2; - this.exceptions = []; - this.properties = {}; - this.measurements = {}; - _super.call(this); - } - return ExceptionData; - })(Microsoft.Telemetry.Domain); - AI.ExceptionData = ExceptionData; -})(AI || (AI = {})); -var AI; -(function (AI) { - "use strict"; - var StackFrame = (function () { - function StackFrame() { - } - return StackFrame; - })(); - AI.StackFrame = StackFrame; -})(AI || (AI = {})); -/// -/// -/// -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - var Telemetry; - (function (Telemetry) { - "use strict"; - var Exception = (function (_super) { - __extends(Exception, _super); - function Exception(exception, handledAt, properties, measurements) { - _super.call(this); - this.aiDataContract = { - ver: ApplicationInsights.FieldType.Required, - handledAt: ApplicationInsights.FieldType.Required, - exceptions: ApplicationInsights.FieldType.Required, - severityLevel: ApplicationInsights.FieldType.Default, - properties: ApplicationInsights.FieldType.Default, - measurements: ApplicationInsights.FieldType.Default - }; - this.properties = ApplicationInsights.Telemetry.Common.DataSanitizer.sanitizeProperties(properties); - this.measurements = ApplicationInsights.Telemetry.Common.DataSanitizer.sanitizeMeasurements(measurements); - this.handledAt = handledAt || "unhandled"; - this.exceptions = [new _ExceptionDetails(exception)]; - } - Exception.CreateSimpleException = function (message, typeName, assembly, fileName, details, line, handledAt) { - return { - handledAt: handledAt || "unhandled", - exceptions: [ - { - hasFullStack: true, - message: message, - stack: details, - typeName: typeName, - parsedStack: [ - { - level: 0, - assembly: assembly, - fileName: fileName, - line: line, - method: "unknown" - } - ] - } - ] - }; - }; - Exception.envelopeType = "Microsoft.ApplicationInsights.{0}.Exception"; - Exception.dataType = "ExceptionData"; - return Exception; - })(AI.ExceptionData); - Telemetry.Exception = Exception; - var _ExceptionDetails = (function (_super) { - __extends(_ExceptionDetails, _super); - function _ExceptionDetails(exception) { - _super.call(this); - this.aiDataContract = { - id: ApplicationInsights.FieldType.Default, - outerId: ApplicationInsights.FieldType.Default, - typeName: ApplicationInsights.FieldType.Required, - message: ApplicationInsights.FieldType.Required, - hasFullStack: ApplicationInsights.FieldType.Default, - stack: ApplicationInsights.FieldType.Default, - parsedStack: ApplicationInsights.FieldType.Array - }; - this.typeName = Telemetry.Common.DataSanitizer.sanitizeString(exception.name || ApplicationInsights.Util.NotSpecified); - this.message = Telemetry.Common.DataSanitizer.sanitizeMessage(exception.message || ApplicationInsights.Util.NotSpecified); - var stack = exception["stack"]; - this.parsedStack = this.parseStack(stack); - this.stack = Telemetry.Common.DataSanitizer.sanitizeException(stack); - this.hasFullStack = ApplicationInsights.Util.isArray(this.parsedStack) && this.parsedStack.length > 0; - } - _ExceptionDetails.prototype.parseStack = function (stack) { - var parsedStack = undefined; - if (typeof stack === "string") { - var frames = stack.split('\n'); - parsedStack = []; - var level = 0; - var totalSizeInBytes = 0; - for (var i = 0; i <= frames.length; i++) { - var frame = frames[i]; - if (_StackFrame.regex.test(frame)) { - var parsedFrame = new _StackFrame(frames[i], level++); - totalSizeInBytes += parsedFrame.sizeInBytes; - parsedStack.push(parsedFrame); - } - } - var exceptionParsedStackThreshold = 32 * 1024; - if (totalSizeInBytes > exceptionParsedStackThreshold) { - var left = 0; - var right = parsedStack.length - 1; - var size = 0; - var acceptedLeft = left; - var acceptedRight = right; - while (left < right) { - var lSize = parsedStack[left].sizeInBytes; - var rSize = parsedStack[right].sizeInBytes; - size += lSize + rSize; - if (size > exceptionParsedStackThreshold) { - var howMany = acceptedRight - acceptedLeft + 1; - parsedStack.splice(acceptedLeft, howMany); - break; - } - acceptedLeft = left; - acceptedRight = right; - left++; - right--; - } - } - } - return parsedStack; - }; - return _ExceptionDetails; - })(AI.ExceptionDetails); - var _StackFrame = (function (_super) { - __extends(_StackFrame, _super); - function _StackFrame(frame, level) { - _super.call(this); - this.sizeInBytes = 0; - this.aiDataContract = { - level: ApplicationInsights.FieldType.Required, - method: ApplicationInsights.FieldType.Required, - assembly: ApplicationInsights.FieldType.Default, - fileName: ApplicationInsights.FieldType.Default, - line: ApplicationInsights.FieldType.Default - }; - this.level = level; - this.method = ""; - this.assembly = ApplicationInsights.Util.trim(frame); - var matches = frame.match(_StackFrame.regex); - if (matches && matches.length >= 5) { - this.method = ApplicationInsights.Util.trim(matches[2]) || this.method; - this.fileName = ApplicationInsights.Util.trim(matches[4]); - this.line = parseInt(matches[5]) || 0; - } - this.sizeInBytes += this.method.length; - this.sizeInBytes += this.fileName.length; - this.sizeInBytes += this.assembly.length; - this.sizeInBytes += _StackFrame.baseSize; - this.sizeInBytes += this.level.toString().length; - this.sizeInBytes += this.line.toString().length; - } - _StackFrame.regex = /^([\s]+at)?(.*?)(\@|\s\(|\s)([^\(\@\n]+):([0-9]+):([0-9]+)(\)?)$/; - _StackFrame.baseSize = 58; - return _StackFrame; - })(AI.StackFrame); - Telemetry._StackFrame = _StackFrame; - })(Telemetry = ApplicationInsights.Telemetry || (ApplicationInsights.Telemetry = {})); - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -/// -var AI; -(function (AI) { - "use strict"; - var MetricData = (function (_super) { - __extends(MetricData, _super); - function MetricData() { - this.ver = 2; - this.metrics = []; - this.properties = {}; - _super.call(this); - } - return MetricData; - })(Microsoft.Telemetry.Domain); - AI.MetricData = MetricData; -})(AI || (AI = {})); -var AI; -(function (AI) { - "use strict"; - (function (DataPointType) { - DataPointType[DataPointType["Measurement"] = 0] = "Measurement"; - DataPointType[DataPointType["Aggregation"] = 1] = "Aggregation"; - })(AI.DataPointType || (AI.DataPointType = {})); - var DataPointType = AI.DataPointType; -})(AI || (AI = {})); -/// -var AI; -(function (AI) { - "use strict"; - var DataPoint = (function () { - function DataPoint() { - this.kind = AI.DataPointType.Measurement; - } - return DataPoint; - })(); - AI.DataPoint = DataPoint; -})(AI || (AI = {})); -/// -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - var Telemetry; - (function (Telemetry) { - var Common; - (function (Common) { - "use strict"; - var DataPoint = (function (_super) { - __extends(DataPoint, _super); - function DataPoint() { - _super.apply(this, arguments); - this.aiDataContract = { - name: ApplicationInsights.FieldType.Required, - kind: ApplicationInsights.FieldType.Default, - value: ApplicationInsights.FieldType.Required, - count: ApplicationInsights.FieldType.Default, - min: ApplicationInsights.FieldType.Default, - max: ApplicationInsights.FieldType.Default, - stdDev: ApplicationInsights.FieldType.Default - }; - } - return DataPoint; - })(AI.DataPoint); - Common.DataPoint = DataPoint; - })(Common = Telemetry.Common || (Telemetry.Common = {})); - })(Telemetry = ApplicationInsights.Telemetry || (ApplicationInsights.Telemetry = {})); - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -/// -/// -/// -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - var Telemetry; - (function (Telemetry) { - "use strict"; - var Metric = (function (_super) { - __extends(Metric, _super); - function Metric(name, value, count, min, max, properties) { - _super.call(this); - this.aiDataContract = { - ver: ApplicationInsights.FieldType.Required, - metrics: ApplicationInsights.FieldType.Required, - properties: ApplicationInsights.FieldType.Default - }; - var dataPoint = new Microsoft.ApplicationInsights.Telemetry.Common.DataPoint(); - dataPoint.count = count > 0 ? count : undefined; - dataPoint.max = isNaN(max) || max === null ? undefined : max; - dataPoint.min = isNaN(min) || min === null ? undefined : min; - dataPoint.name = Telemetry.Common.DataSanitizer.sanitizeString(name); - dataPoint.value = value; - this.metrics = [dataPoint]; - this.properties = ApplicationInsights.Telemetry.Common.DataSanitizer.sanitizeProperties(properties); - } - Metric.envelopeType = "Microsoft.ApplicationInsights.{0}.Metric"; - Metric.dataType = "MetricData"; - return Metric; - })(AI.MetricData); - Telemetry.Metric = Metric; - })(Telemetry = ApplicationInsights.Telemetry || (ApplicationInsights.Telemetry = {})); - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -/// -var AI; -(function (AI) { - "use strict"; - var PageViewData = (function (_super) { - __extends(PageViewData, _super); - function PageViewData() { - this.ver = 2; - this.properties = {}; - this.measurements = {}; - _super.call(this); - } - return PageViewData; - })(AI.EventData); - AI.PageViewData = PageViewData; -})(AI || (AI = {})); -/// -/// -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - var Telemetry; - (function (Telemetry) { - "use strict"; - var PageView = (function (_super) { - __extends(PageView, _super); - function PageView(name, url, durationMs, properties, measurements) { - _super.call(this); - this.aiDataContract = { - ver: ApplicationInsights.FieldType.Required, - name: ApplicationInsights.FieldType.Default, - url: ApplicationInsights.FieldType.Default, - duration: ApplicationInsights.FieldType.Default, - properties: ApplicationInsights.FieldType.Default, - measurements: ApplicationInsights.FieldType.Default - }; - this.url = Telemetry.Common.DataSanitizer.sanitizeUrl(url); - this.name = Telemetry.Common.DataSanitizer.sanitizeString(name || ApplicationInsights.Util.NotSpecified); - if (!isNaN(durationMs)) { - this.duration = ApplicationInsights.Util.msToTimeSpan(durationMs); - } - this.properties = ApplicationInsights.Telemetry.Common.DataSanitizer.sanitizeProperties(properties); - this.measurements = ApplicationInsights.Telemetry.Common.DataSanitizer.sanitizeMeasurements(measurements); - } - PageView.envelopeType = "Microsoft.ApplicationInsights.{0}.Pageview"; - PageView.dataType = "PageviewData"; - return PageView; - })(AI.PageViewData); - Telemetry.PageView = PageView; - })(Telemetry = ApplicationInsights.Telemetry || (ApplicationInsights.Telemetry = {})); - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -/// -var AI; -(function (AI) { - "use strict"; - var PageViewPerfData = (function (_super) { - __extends(PageViewPerfData, _super); - function PageViewPerfData() { - this.ver = 2; - this.properties = {}; - this.measurements = {}; - _super.call(this); - } - return PageViewPerfData; - })(AI.PageViewData); - AI.PageViewPerfData = PageViewPerfData; -})(AI || (AI = {})); -/// -/// -/// -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - var Telemetry; - (function (Telemetry) { - "use strict"; - var PageViewPerformance = (function (_super) { - __extends(PageViewPerformance, _super); - function PageViewPerformance(name, url, unused, properties, measurements) { - _super.call(this); - this.aiDataContract = { - ver: ApplicationInsights.FieldType.Required, - name: ApplicationInsights.FieldType.Default, - url: ApplicationInsights.FieldType.Default, - duration: ApplicationInsights.FieldType.Default, - perfTotal: ApplicationInsights.FieldType.Default, - networkConnect: ApplicationInsights.FieldType.Default, - sentRequest: ApplicationInsights.FieldType.Default, - receivedResponse: ApplicationInsights.FieldType.Default, - domProcessing: ApplicationInsights.FieldType.Default, - properties: ApplicationInsights.FieldType.Default, - measurements: ApplicationInsights.FieldType.Default - }; - this.isValid = false; - var timing = PageViewPerformance.getPerformanceTiming(); - if (timing) { - var total = PageViewPerformance.getDuration(timing.navigationStart, timing.loadEventEnd); - var network = PageViewPerformance.getDuration(timing.navigationStart, timing.connectEnd); - var request = PageViewPerformance.getDuration(timing.requestStart, timing.responseStart); - var response = PageViewPerformance.getDuration(timing.responseStart, timing.responseEnd); - var dom = PageViewPerformance.getDuration(timing.responseEnd, timing.loadEventEnd); - if (total == 0) { - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.WARNING, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_ErrorPVCalc, "error calculating page view performance.", { total: total, network: network, request: request, response: response, dom: dom })); - } - else if (total < Math.floor(network) + Math.floor(request) + Math.floor(response) + Math.floor(dom)) { - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.WARNING, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_ClientPerformanceMathError, "client performance math error.", { total: total, network: network, request: request, response: response, dom: dom })); - } - else { - this.durationMs = total; - this.perfTotal = this.duration = ApplicationInsights.Util.msToTimeSpan(total); - this.networkConnect = ApplicationInsights.Util.msToTimeSpan(network); - this.sentRequest = ApplicationInsights.Util.msToTimeSpan(request); - this.receivedResponse = ApplicationInsights.Util.msToTimeSpan(response); - this.domProcessing = ApplicationInsights.Util.msToTimeSpan(dom); - this.isValid = true; - } - } - this.url = Telemetry.Common.DataSanitizer.sanitizeUrl(url); - this.name = Telemetry.Common.DataSanitizer.sanitizeString(name || ApplicationInsights.Util.NotSpecified); - this.properties = ApplicationInsights.Telemetry.Common.DataSanitizer.sanitizeProperties(properties); - this.measurements = ApplicationInsights.Telemetry.Common.DataSanitizer.sanitizeMeasurements(measurements); - } - PageViewPerformance.prototype.getIsValid = function () { - return this.isValid; - }; - PageViewPerformance.prototype.getDurationMs = function () { - return this.durationMs; - }; - PageViewPerformance.getPerformanceTiming = function () { - if (typeof window != "undefined" && window.performance && window.performance.timing) { - return window.performance.timing; - } - return null; - }; - PageViewPerformance.isPerformanceTimingSupported = function () { - return typeof window != "undefined" && window.performance && window.performance.timing; - }; - PageViewPerformance.isPerformanceTimingDataReady = function () { - var timing = window.performance.timing; - return timing.domainLookupStart > 0 - && timing.navigationStart > 0 - && timing.responseStart > 0 - && timing.requestStart > 0 - && timing.loadEventEnd > 0 - && timing.responseEnd > 0 - && timing.connectEnd > 0 - && timing.domLoading > 0; - }; - PageViewPerformance.getDuration = function (start, end) { - var duration = 0; - if (!(isNaN(start) || isNaN(end))) { - duration = Math.max(end - start, 0); - } - return duration; - }; - PageViewPerformance.envelopeType = "Microsoft.ApplicationInsights.{0}.PageviewPerformance"; - PageViewPerformance.dataType = "PageviewPerformanceData"; - return PageViewPerformance; - })(AI.PageViewPerfData); - Telemetry.PageViewPerformance = PageViewPerformance; - })(Telemetry = ApplicationInsights.Telemetry || (ApplicationInsights.Telemetry = {})); - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -/// -/// -/// -/// -/// -/// -/// -/// -/// -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - "use strict"; - var TelemetryContext = (function () { - function TelemetryContext(config) { - this._config = config; - this._sender = new ApplicationInsights.Sender(config); - if (typeof window !== 'undefined') { - this._sessionManager = new ApplicationInsights.Context._SessionManager(config); - this.application = new ApplicationInsights.Context.Application(); - this.device = new ApplicationInsights.Context.Device(); - this.internal = new ApplicationInsights.Context.Internal(); - this.location = new ApplicationInsights.Context.Location(); - this.user = new ApplicationInsights.Context.User(config); - this.operation = new ApplicationInsights.Context.Operation(); - this.session = new ApplicationInsights.Context.Session(); - this.sample = new ApplicationInsights.Context.Sample(config.sampleRate()); - } - } - TelemetryContext.prototype.addTelemetryInitializer = function (telemetryInitializer) { - this.telemetryInitializers = this.telemetryInitializers || []; - this.telemetryInitializers.push(telemetryInitializer); - }; - TelemetryContext.prototype.track = function (envelope) { - if (!envelope) { - ApplicationInsights._InternalLogging.throwInternalUserActionable(ApplicationInsights.LoggingSeverity.CRITICAL, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.USRACT_TrackArgumentsNotSpecified, "cannot call .track() with a null or undefined argument")); - } - else { - if (envelope.name === ApplicationInsights.Telemetry.PageView.envelopeType) { - ApplicationInsights._InternalLogging.resetInternalMessageCount(); - } - if (this.session) { - if (typeof this.session.id !== "string") { - this._sessionManager.update(); - } - } - this._track(envelope); - } - return envelope; - }; - TelemetryContext.prototype._track = function (envelope) { - if (this.session) { - if (typeof this.session.id === "string") { - this._applySessionContext(envelope, this.session); - } - else { - this._applySessionContext(envelope, this._sessionManager.automaticSession); - } - } - this._applyApplicationContext(envelope, this.application); - this._applyDeviceContext(envelope, this.device); - this._applyInternalContext(envelope, this.internal); - this._applyLocationContext(envelope, this.location); - this._applySampleContext(envelope, this.sample); - this._applyUserContext(envelope, this.user); - this._applyOperationContext(envelope, this.operation); - envelope.iKey = this._config.instrumentationKey(); - var doNotSendItem = false; - try { - this.telemetryInitializers = this.telemetryInitializers || []; - var telemetryInitializersCount = this.telemetryInitializers.length; - for (var i = 0; i < telemetryInitializersCount; ++i) { - var telemetryInitializer = this.telemetryInitializers[i]; - if (telemetryInitializer) { - if (telemetryInitializer.apply(null, [envelope]) === false) { - doNotSendItem = true; - break; - } - } - } - } - catch (e) { - doNotSendItem = true; - ApplicationInsights._InternalLogging.throwInternalUserActionable(ApplicationInsights.LoggingSeverity.CRITICAL, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.USRACT_TelemetryInitializerFailed, "One of telemetry initializers failed, telemetry item will not be sent: " + ApplicationInsights.Util.getExceptionName(e), { exception: ApplicationInsights.Util.dump(e) })); - } - if (!doNotSendItem) { - if (envelope.name === ApplicationInsights.Telemetry.Metric.envelopeType || - this.sample.isSampledIn(envelope)) { - var iKeyNoDashes = this._config.instrumentationKey().replace(/-/g, ""); - envelope.name = envelope.name.replace("{0}", iKeyNoDashes); - this._sender.send(envelope); - } - else { - ApplicationInsights._InternalLogging.throwInternalUserActionable(ApplicationInsights.LoggingSeverity.WARNING, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_TelemetrySampledAndNotSent, "Telemetry is sampled and not sent to the AI service.", { SampleRate: this.sample.sampleRate })); - } - } - return envelope; - }; - TelemetryContext.prototype._applyApplicationContext = function (envelope, appContext) { - if (appContext) { - var tagKeys = new AI.ContextTagKeys(); - if (typeof appContext.ver === "string") { - envelope.tags[tagKeys.applicationVersion] = appContext.ver; - } - if (typeof appContext.build === "string") { - envelope.tags[tagKeys.applicationBuild] = appContext.build; - } - } - }; - TelemetryContext.prototype._applyDeviceContext = function (envelope, deviceContext) { - var tagKeys = new AI.ContextTagKeys(); - if (deviceContext) { - if (typeof deviceContext.id === "string") { - envelope.tags[tagKeys.deviceId] = deviceContext.id; - } - if (typeof deviceContext.ip === "string") { - envelope.tags[tagKeys.deviceIp] = deviceContext.ip; - } - if (typeof deviceContext.language === "string") { - envelope.tags[tagKeys.deviceLanguage] = deviceContext.language; - } - if (typeof deviceContext.locale === "string") { - envelope.tags[tagKeys.deviceLocale] = deviceContext.locale; - } - if (typeof deviceContext.model === "string") { - envelope.tags[tagKeys.deviceModel] = deviceContext.model; - } - if (typeof deviceContext.network !== "undefined") { - envelope.tags[tagKeys.deviceNetwork] = deviceContext.network; - } - if (typeof deviceContext.oemName === "string") { - envelope.tags[tagKeys.deviceOEMName] = deviceContext.oemName; - } - if (typeof deviceContext.os === "string") { - envelope.tags[tagKeys.deviceOS] = deviceContext.os; - } - if (typeof deviceContext.osversion === "string") { - envelope.tags[tagKeys.deviceOSVersion] = deviceContext.osversion; - } - if (typeof deviceContext.resolution === "string") { - envelope.tags[tagKeys.deviceScreenResolution] = deviceContext.resolution; - } - if (typeof deviceContext.type === "string") { - envelope.tags[tagKeys.deviceType] = deviceContext.type; - } - } - }; - TelemetryContext.prototype._applyInternalContext = function (envelope, internalContext) { - if (internalContext) { - var tagKeys = new AI.ContextTagKeys(); - if (typeof internalContext.agentVersion === "string") { - envelope.tags[tagKeys.internalAgentVersion] = internalContext.agentVersion; - } - if (typeof internalContext.sdkVersion === "string") { - envelope.tags[tagKeys.internalSdkVersion] = internalContext.sdkVersion; - } - } - }; - TelemetryContext.prototype._applyLocationContext = function (envelope, locationContext) { - if (locationContext) { - var tagKeys = new AI.ContextTagKeys(); - if (typeof locationContext.ip === "string") { - envelope.tags[tagKeys.locationIp] = locationContext.ip; - } - } - }; - TelemetryContext.prototype._applyOperationContext = function (envelope, operationContext) { - if (operationContext) { - var tagKeys = new AI.ContextTagKeys(); - if (typeof operationContext.id === "string") { - envelope.tags[tagKeys.operationId] = operationContext.id; - } - if (typeof operationContext.name === "string") { - envelope.tags[tagKeys.operationName] = operationContext.name; - } - if (typeof operationContext.parentId === "string") { - envelope.tags[tagKeys.operationParentId] = operationContext.parentId; - } - if (typeof operationContext.rootId === "string") { - envelope.tags[tagKeys.operationRootId] = operationContext.rootId; - } - if (typeof operationContext.syntheticSource === "string") { - envelope.tags[tagKeys.operationSyntheticSource] = operationContext.syntheticSource; - } - } - }; - TelemetryContext.prototype._applySampleContext = function (envelope, sampleContext) { - if (sampleContext) { - envelope.sampleRate = sampleContext.sampleRate; - } - }; - TelemetryContext.prototype._applySessionContext = function (envelope, sessionContext) { - if (sessionContext) { - var tagKeys = new AI.ContextTagKeys(); - if (typeof sessionContext.id === "string") { - envelope.tags[tagKeys.sessionId] = sessionContext.id; - } - if (typeof sessionContext.isFirst !== "undefined") { - envelope.tags[tagKeys.sessionIsFirst] = sessionContext.isFirst; - } - } - }; - TelemetryContext.prototype._applyUserContext = function (envelope, userContext) { - if (userContext) { - var tagKeys = new AI.ContextTagKeys(); - if (typeof userContext.accountId === "string") { - envelope.tags[tagKeys.userAccountId] = userContext.accountId; - } - if (typeof userContext.agent === "string") { - envelope.tags[tagKeys.userAgent] = userContext.agent; - } - if (typeof userContext.id === "string") { - envelope.tags[tagKeys.userId] = userContext.id; - } - if (typeof userContext.authenticatedId === "string") { - envelope.tags[tagKeys.userAuthUserId] = userContext.authenticatedId; - } - if (typeof userContext.storeRegion === "string") { - envelope.tags[tagKeys.userStoreRegion] = userContext.storeRegion; - } - } - }; - return TelemetryContext; - })(); - ApplicationInsights.TelemetryContext = TelemetryContext; - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -/// -var Microsoft; -(function (Microsoft) { - var Telemetry; - (function (Telemetry) { - "use strict"; - var Data = (function (_super) { - __extends(Data, _super); - function Data() { - _super.call(this); - } - return Data; - })(Microsoft.Telemetry.Base); - Telemetry.Data = Data; - })(Telemetry = Microsoft.Telemetry || (Microsoft.Telemetry = {})); -})(Microsoft || (Microsoft = {})); -/// -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - var Telemetry; - (function (Telemetry) { - var Common; - (function (Common) { - "use strict"; - var Data = (function (_super) { - __extends(Data, _super); - function Data(type, data) { - _super.call(this); - this.aiDataContract = { - baseType: ApplicationInsights.FieldType.Required, - baseData: ApplicationInsights.FieldType.Required - }; - this.baseType = type; - this.baseData = data; - } - return Data; - })(Microsoft.Telemetry.Data); - Common.Data = Data; - })(Common = Telemetry.Common || (Telemetry.Common = {})); - })(Telemetry = ApplicationInsights.Telemetry || (ApplicationInsights.Telemetry = {})); - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -/// -/// -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - var Telemetry; - (function (Telemetry) { - "use strict"; - var PageViewManager = (function () { - function PageViewManager(appInsights, overridePageViewDuration) { - this.pageViewPerformanceSent = false; - this.overridePageViewDuration = false; - this.overridePageViewDuration = overridePageViewDuration; - this.appInsights = appInsights; - } - PageViewManager.prototype.trackPageView = function (name, url, properties, measurements, duration) { - var _this = this; - if (typeof name !== "string") { - name = window.document && window.document.title || ""; - } - if (typeof url !== "string") { - url = window.location && window.location.href || ""; - } - var pageViewSent = false; - var customDuration = 0; - if (Telemetry.PageViewPerformance.isPerformanceTimingSupported()) { - var start = Telemetry.PageViewPerformance.getPerformanceTiming().navigationStart; - customDuration = Telemetry.PageViewPerformance.getDuration(start, +new Date); - } - else { - this.appInsights.sendPageViewInternal(name, url, !isNaN(duration) ? duration : 0, properties, measurements); - this.appInsights.flush(); - pageViewSent = true; - } - if (this.overridePageViewDuration || !isNaN(duration)) { - this.appInsights.sendPageViewInternal(name, url, !isNaN(duration) ? duration : customDuration, properties, measurements); - this.appInsights.flush(); - pageViewSent = true; - } - var maxDurationLimit = 60000; - if (!Telemetry.PageViewPerformance.isPerformanceTimingSupported()) { - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.WARNING, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_NavigationTimingNotSupported, "trackPageView: navigation timing API used for calculation of page duration is not supported in this browser. This page view will be collected without duration and timing info.")); - return; - } - var handle = setInterval(function () { - try { - if (Telemetry.PageViewPerformance.isPerformanceTimingDataReady()) { - clearInterval(handle); - var pageViewPerformance = new Telemetry.PageViewPerformance(name, url, null, properties, measurements); - if (!pageViewPerformance.getIsValid() && !pageViewSent) { - _this.appInsights.sendPageViewInternal(name, url, customDuration, properties, measurements); - _this.appInsights.flush(); - } - else { - if (!pageViewSent) { - _this.appInsights.sendPageViewInternal(name, url, pageViewPerformance.getDurationMs(), properties, measurements); - } - if (!_this.pageViewPerformanceSent) { - _this.appInsights.sendPageViewPerformanceInternal(pageViewPerformance); - _this.pageViewPerformanceSent = true; - } - _this.appInsights.flush(); - } - } - else if (Telemetry.PageViewPerformance.getDuration(start, +new Date) > maxDurationLimit) { - clearInterval(handle); - if (!pageViewSent) { - _this.appInsights.sendPageViewInternal(name, url, maxDurationLimit, properties, measurements); - _this.appInsights.flush(); - } - } - } - catch (e) { - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.CRITICAL, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_TrackPVFailedCalc, "trackPageView failed on page load calculation: " + ApplicationInsights.Util.getExceptionName(e), { exception: ApplicationInsights.Util.dump(e) })); - } - }, 100); - }; - return PageViewManager; - })(); - Telemetry.PageViewManager = PageViewManager; - })(Telemetry = ApplicationInsights.Telemetry || (ApplicationInsights.Telemetry = {})); - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -/// -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - var Telemetry; - (function (Telemetry) { - "use strict"; - var PageVisitTimeManager = (function () { - function PageVisitTimeManager(pageVisitTimeTrackingHandler) { - this.prevPageVisitDataKeyName = "prevPageVisitData"; - this.pageVisitTimeTrackingHandler = pageVisitTimeTrackingHandler; - } - PageVisitTimeManager.prototype.trackPreviousPageVisit = function (currentPageName, currentPageUrl) { - try { - var prevPageVisitTimeData = this.restartPageVisitTimer(currentPageName, currentPageUrl); - if (prevPageVisitTimeData) { - this.pageVisitTimeTrackingHandler(prevPageVisitTimeData.pageName, prevPageVisitTimeData.pageUrl, prevPageVisitTimeData.pageVisitTime); - } - } - catch (e) { - ApplicationInsights._InternalLogging.warnToConsole("Auto track page visit time failed, metric will not be collected: " + ApplicationInsights.Util.dump(e)); - } - }; - PageVisitTimeManager.prototype.restartPageVisitTimer = function (pageName, pageUrl) { - try { - var prevPageVisitData = this.stopPageVisitTimer(); - this.startPageVisitTimer(pageName, pageUrl); - return prevPageVisitData; - } - catch (e) { - ApplicationInsights._InternalLogging.warnToConsole("Call to restart failed: " + ApplicationInsights.Util.dump(e)); - return null; - } - }; - PageVisitTimeManager.prototype.startPageVisitTimer = function (pageName, pageUrl) { - try { - if (ApplicationInsights.Util.canUseSessionStorage()) { - if (ApplicationInsights.Util.getSessionStorage(this.prevPageVisitDataKeyName) != null) { - throw new Error("Cannot call startPageVisit consecutively without first calling stopPageVisit"); - } - var currPageVisitData = new PageVisitData(pageName, pageUrl); - var currPageVisitDataStr = JSON.stringify(currPageVisitData); - ApplicationInsights.Util.setSessionStorage(this.prevPageVisitDataKeyName, currPageVisitDataStr); - } - } - catch (e) { - ApplicationInsights._InternalLogging.warnToConsole("Call to start failed: " + ApplicationInsights.Util.dump(e)); - } - }; - PageVisitTimeManager.prototype.stopPageVisitTimer = function () { - try { - if (ApplicationInsights.Util.canUseSessionStorage()) { - var pageVisitEndTime = Date.now(); - var pageVisitDataJsonStr = ApplicationInsights.Util.getSessionStorage(this.prevPageVisitDataKeyName); - if (pageVisitDataJsonStr) { - var prevPageVisitData = JSON.parse(pageVisitDataJsonStr); - prevPageVisitData.pageVisitTime = pageVisitEndTime - prevPageVisitData.pageVisitStartTime; - ApplicationInsights.Util.removeSessionStorage(this.prevPageVisitDataKeyName); - return prevPageVisitData; - } - else { - return null; - } - } - return null; - } - catch (e) { - ApplicationInsights._InternalLogging.warnToConsole("Stop page visit timer failed: " + ApplicationInsights.Util.dump(e)); - return null; - } - }; - return PageVisitTimeManager; - })(); - Telemetry.PageVisitTimeManager = PageVisitTimeManager; - var PageVisitData = (function () { - function PageVisitData(pageName, pageUrl) { - this.pageVisitStartTime = Date.now(); - this.pageName = pageName; - this.pageUrl = pageUrl; - } - return PageVisitData; - })(); - Telemetry.PageVisitData = PageVisitData; - })(Telemetry = ApplicationInsights.Telemetry || (ApplicationInsights.Telemetry = {})); - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -var AI; -(function (AI) { - "use strict"; - (function (DependencyKind) { - DependencyKind[DependencyKind["SQL"] = 0] = "SQL"; - DependencyKind[DependencyKind["Http"] = 1] = "Http"; - DependencyKind[DependencyKind["Other"] = 2] = "Other"; - })(AI.DependencyKind || (AI.DependencyKind = {})); - var DependencyKind = AI.DependencyKind; -})(AI || (AI = {})); -var AI; -(function (AI) { - "use strict"; - (function (DependencySourceType) { - DependencySourceType[DependencySourceType["Undefined"] = 0] = "Undefined"; - DependencySourceType[DependencySourceType["Aic"] = 1] = "Aic"; - DependencySourceType[DependencySourceType["Apmc"] = 2] = "Apmc"; - })(AI.DependencySourceType || (AI.DependencySourceType = {})); - var DependencySourceType = AI.DependencySourceType; -})(AI || (AI = {})); -/// -/// -/// -/// -var AI; -(function (AI) { - "use strict"; - var RemoteDependencyData = (function (_super) { - __extends(RemoteDependencyData, _super); - function RemoteDependencyData() { - this.ver = 2; - this.kind = AI.DataPointType.Aggregation; - this.dependencyKind = AI.DependencyKind.Other; - this.success = true; - this.dependencySource = AI.DependencySourceType.Apmc; - this.properties = {}; - _super.call(this); - } - return RemoteDependencyData; - })(Microsoft.Telemetry.Domain); - AI.RemoteDependencyData = RemoteDependencyData; -})(AI || (AI = {})); -/// -/// -/// -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - var Telemetry; - (function (Telemetry) { - "use strict"; - var RemoteDependencyData = (function (_super) { - __extends(RemoteDependencyData, _super); - function RemoteDependencyData(id, name, commandName, value, success, resultCode) { - _super.call(this); - this.aiDataContract = { - id: ApplicationInsights.FieldType.Required, - ver: ApplicationInsights.FieldType.Required, - name: ApplicationInsights.FieldType.Default, - kind: ApplicationInsights.FieldType.Required, - value: ApplicationInsights.FieldType.Default, - count: ApplicationInsights.FieldType.Default, - min: ApplicationInsights.FieldType.Default, - max: ApplicationInsights.FieldType.Default, - stdDev: ApplicationInsights.FieldType.Default, - dependencyKind: ApplicationInsights.FieldType.Default, - success: ApplicationInsights.FieldType.Default, - async: ApplicationInsights.FieldType.Default, - dependencySource: ApplicationInsights.FieldType.Default, - commandName: ApplicationInsights.FieldType.Default, - dependencyTypeName: ApplicationInsights.FieldType.Default, - properties: ApplicationInsights.FieldType.Default, - resultCode: ApplicationInsights.FieldType.Default - }; - this.id = id; - this.name = name; - this.commandName = commandName; - this.value = value; - this.success = success; - this.resultCode = resultCode + ""; - this.dependencyKind = AI.DependencyKind.Http; - this.dependencyTypeName = "Ajax"; - } - RemoteDependencyData.envelopeType = "Microsoft.ApplicationInsights.{0}.RemoteDependency"; - RemoteDependencyData.dataType = "RemoteDependencyData"; - return RemoteDependencyData; - })(AI.RemoteDependencyData); - Telemetry.RemoteDependencyData = RemoteDependencyData; - })(Telemetry = ApplicationInsights.Telemetry || (ApplicationInsights.Telemetry = {})); - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -/// -/// -/// -/// -/// -/// -/// -/// -/// -/// -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - "use strict"; - ApplicationInsights.Version = "0.22.9"; - var AppInsights = (function () { - function AppInsights(config) { - var _this = this; - this._trackAjaxAttempts = 0; - this.config = config || {}; - var defaults = AppInsights.defaultConfig; - if (defaults !== undefined) { - for (var field in defaults) { - if (this.config[field] === undefined) { - this.config[field] = defaults[field]; - } - } - } - ApplicationInsights._InternalLogging.verboseLogging = function () { return _this.config.verboseLogging; }; - ApplicationInsights._InternalLogging.enableDebugExceptions = function () { return _this.config.enableDebug; }; - var configGetters = { - instrumentationKey: function () { return _this.config.instrumentationKey; }, - accountId: function () { return _this.config.accountId; }, - sessionRenewalMs: function () { return _this.config.sessionRenewalMs; }, - sessionExpirationMs: function () { return _this.config.sessionExpirationMs; }, - endpointUrl: function () { return _this.config.endpointUrl; }, - emitLineDelimitedJson: function () { return _this.config.emitLineDelimitedJson; }, - maxBatchSizeInBytes: function () { return _this.config.maxBatchSizeInBytes; }, - maxBatchInterval: function () { return _this.config.maxBatchInterval; }, - disableTelemetry: function () { return _this.config.disableTelemetry; }, - sampleRate: function () { return _this.config.samplingPercentage; }, - cookieDomain: function () { return _this.config.cookieDomain; } - }; - this.context = new ApplicationInsights.TelemetryContext(configGetters); - this._pageViewManager = new Microsoft.ApplicationInsights.Telemetry.PageViewManager(this, this.config.overridePageViewDuration); - this._eventTracking = new Timing("trackEvent"); - this._eventTracking.action = function (name, url, duration, properties, measurements) { - if (!measurements) { - measurements = { duration: duration }; - } - else { - if (isNaN(measurements["duration"])) { - measurements["duration"] = duration; - } - } - var event = new ApplicationInsights.Telemetry.Event(name, properties, measurements); - var data = new ApplicationInsights.Telemetry.Common.Data(ApplicationInsights.Telemetry.Event.dataType, event); - var envelope = new ApplicationInsights.Telemetry.Common.Envelope(data, ApplicationInsights.Telemetry.Event.envelopeType); - _this.context.track(envelope); - }; - this._pageTracking = new Timing("trackPageView"); - this._pageTracking.action = function (name, url, duration, properties, measurements) { - _this.sendPageViewInternal(name, url, duration, properties, measurements); - }; - this._pageVisitTimeManager = new ApplicationInsights.Telemetry.PageVisitTimeManager(function (pageName, pageUrl, pageVisitTime) { return _this.trackPageVisitTime(pageName, pageUrl, pageVisitTime); }); - if (!this.config.disableAjaxTracking) { - new Microsoft.ApplicationInsights.AjaxMonitor(this); - } - } - AppInsights.prototype.sendPageViewInternal = function (name, url, duration, properties, measurements) { - var pageView = new ApplicationInsights.Telemetry.PageView(name, url, duration, properties, measurements); - var data = new ApplicationInsights.Telemetry.Common.Data(ApplicationInsights.Telemetry.PageView.dataType, pageView); - var envelope = new ApplicationInsights.Telemetry.Common.Envelope(data, ApplicationInsights.Telemetry.PageView.envelopeType); - this.context.track(envelope); - this._trackAjaxAttempts = 0; - }; - AppInsights.prototype.sendPageViewPerformanceInternal = function (pageViewPerformance) { - var pageViewPerformanceData = new ApplicationInsights.Telemetry.Common.Data(ApplicationInsights.Telemetry.PageViewPerformance.dataType, pageViewPerformance); - var pageViewPerformanceEnvelope = new ApplicationInsights.Telemetry.Common.Envelope(pageViewPerformanceData, ApplicationInsights.Telemetry.PageViewPerformance.envelopeType); - this.context.track(pageViewPerformanceEnvelope); - }; - AppInsights.prototype.startTrackPage = function (name) { - try { - if (typeof name !== "string") { - name = window.document && window.document.title || ""; - } - this._pageTracking.start(name); - } - catch (e) { - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.CRITICAL, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_StartTrackFailed, "startTrackPage failed, page view may not be collected: " + ApplicationInsights.Util.getExceptionName(e), { exception: ApplicationInsights.Util.dump(e) })); - } - }; - AppInsights.prototype.stopTrackPage = function (name, url, properties, measurements) { - try { - if (typeof name !== "string") { - name = window.document && window.document.title || ""; - } - if (typeof url !== "string") { - url = window.location && window.location.href || ""; - } - this._pageTracking.stop(name, url, properties, measurements); - if (this.config.autoTrackPageVisitTime) { - this._pageVisitTimeManager.trackPreviousPageVisit(name, url); - } - } - catch (e) { - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.CRITICAL, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_StopTrackFailed, "stopTrackPage failed, page view will not be collected: " + ApplicationInsights.Util.getExceptionName(e), { exception: ApplicationInsights.Util.dump(e) })); - } - }; - AppInsights.prototype.trackPageView = function (name, url, properties, measurements, duration) { - try { - this._pageViewManager.trackPageView(name, url, properties, measurements, duration); - if (this.config.autoTrackPageVisitTime) { - this._pageVisitTimeManager.trackPreviousPageVisit(name, url); - } - } - catch (e) { - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.CRITICAL, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_TrackPVFailed, "trackPageView failed, page view will not be collected: " + ApplicationInsights.Util.getExceptionName(e), { exception: ApplicationInsights.Util.dump(e) })); - } - }; - AppInsights.prototype.startTrackEvent = function (name) { - try { - this._eventTracking.start(name); - } - catch (e) { - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.CRITICAL, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_StartTrackEventFailed, "startTrackEvent failed, event will not be collected: " + ApplicationInsights.Util.getExceptionName(e), { exception: ApplicationInsights.Util.dump(e) })); - } - }; - AppInsights.prototype.stopTrackEvent = function (name, properties, measurements) { - try { - this._eventTracking.stop(name, undefined, properties, measurements); - } - catch (e) { - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.CRITICAL, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_StopTrackEventFailed, "stopTrackEvent failed, event will not be collected: " + ApplicationInsights.Util.getExceptionName(e), { exception: ApplicationInsights.Util.dump(e) })); - } - }; - AppInsights.prototype.trackEvent = function (name, properties, measurements) { - try { - var eventTelemetry = new ApplicationInsights.Telemetry.Event(name, properties, measurements); - var data = new ApplicationInsights.Telemetry.Common.Data(ApplicationInsights.Telemetry.Event.dataType, eventTelemetry); - var envelope = new ApplicationInsights.Telemetry.Common.Envelope(data, ApplicationInsights.Telemetry.Event.envelopeType); - this.context.track(envelope); - } - catch (e) { - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.CRITICAL, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_TrackEventFailed, "trackEvent failed, event will not be collected: " + ApplicationInsights.Util.getExceptionName(e), { exception: ApplicationInsights.Util.dump(e) })); - } - }; - AppInsights.prototype.trackAjax = function (id, absoluteUrl, pathName, totalTime, success, resultCode) { - if (this.config.maxAjaxCallsPerView === -1 || - this._trackAjaxAttempts < this.config.maxAjaxCallsPerView) { - var dependency = new ApplicationInsights.Telemetry.RemoteDependencyData(id, absoluteUrl, pathName, totalTime, success, resultCode); - var dependencyData = new ApplicationInsights.Telemetry.Common.Data(ApplicationInsights.Telemetry.RemoteDependencyData.dataType, dependency); - var envelope = new ApplicationInsights.Telemetry.Common.Envelope(dependencyData, ApplicationInsights.Telemetry.RemoteDependencyData.envelopeType); - this.context.track(envelope); - } - else if (this._trackAjaxAttempts === this.config.maxAjaxCallsPerView) { - ApplicationInsights._InternalLogging.throwInternalUserActionable(ApplicationInsights.LoggingSeverity.CRITICAL, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.USRACT_MaxAjaxPerPVExceeded, "Maximum ajax per page view limit reached, ajax monitoring is paused until the next trackPageView(). In order to increase the limit set the maxAjaxCallsPerView configuration parameter.")); - } - ++this._trackAjaxAttempts; - }; - AppInsights.prototype.trackException = function (exception, handledAt, properties, measurements) { - try { - if (!ApplicationInsights.Util.isError(exception)) { - try { - throw new Error(exception); - } - catch (error) { - exception = error; - } - } - var exceptionTelemetry = new ApplicationInsights.Telemetry.Exception(exception, handledAt, properties, measurements); - var data = new ApplicationInsights.Telemetry.Common.Data(ApplicationInsights.Telemetry.Exception.dataType, exceptionTelemetry); - var envelope = new ApplicationInsights.Telemetry.Common.Envelope(data, ApplicationInsights.Telemetry.Exception.envelopeType); - this.context.track(envelope); - } - catch (e) { - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.CRITICAL, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_TrackExceptionFailed, "trackException failed, exception will not be collected: " + ApplicationInsights.Util.getExceptionName(e), { exception: ApplicationInsights.Util.dump(e) })); - } - }; - AppInsights.prototype.trackMetric = function (name, average, sampleCount, min, max, properties) { - try { - var telemetry = new ApplicationInsights.Telemetry.Metric(name, average, sampleCount, min, max, properties); - var data = new ApplicationInsights.Telemetry.Common.Data(ApplicationInsights.Telemetry.Metric.dataType, telemetry); - var envelope = new ApplicationInsights.Telemetry.Common.Envelope(data, ApplicationInsights.Telemetry.Metric.envelopeType); - this.context.track(envelope); - } - catch (e) { - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.CRITICAL, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_TrackMetricFailed, "trackMetric failed, metric will not be collected: " + ApplicationInsights.Util.getExceptionName(e), { exception: ApplicationInsights.Util.dump(e) })); - } - }; - AppInsights.prototype.trackTrace = function (message, properties) { - try { - var telemetry = new ApplicationInsights.Telemetry.Trace(message, properties); - var data = new ApplicationInsights.Telemetry.Common.Data(ApplicationInsights.Telemetry.Trace.dataType, telemetry); - var envelope = new ApplicationInsights.Telemetry.Common.Envelope(data, ApplicationInsights.Telemetry.Trace.envelopeType); - this.context.track(envelope); - } - catch (e) { - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.WARNING, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_TrackTraceFailed, "trackTrace failed, trace will not be collected: " + ApplicationInsights.Util.getExceptionName(e), { exception: ApplicationInsights.Util.dump(e) })); - } - }; - AppInsights.prototype.trackPageVisitTime = function (pageName, pageUrl, pageVisitTime) { - var properties = { PageName: pageName, PageUrl: pageUrl }; - this.trackMetric("PageVisitTime", pageVisitTime, 1, pageVisitTime, pageVisitTime, properties); - }; - AppInsights.prototype.flush = function () { - try { - this.context._sender.triggerSend(); - } - catch (e) { - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.CRITICAL, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_FlushFailed, "flush failed, telemetry will not be collected: " + ApplicationInsights.Util.getExceptionName(e), { exception: ApplicationInsights.Util.dump(e) })); - } - }; - AppInsights.prototype.setAuthenticatedUserContext = function (authenticatedUserId, accountId) { - try { - this.context.user.setAuthenticatedUserContext(authenticatedUserId, accountId); - } - catch (e) { - ApplicationInsights._InternalLogging.throwInternalUserActionable(ApplicationInsights.LoggingSeverity.WARNING, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.USRACT_SetAuthContextFailed, "Setting auth user context failed. " + ApplicationInsights.Util.getExceptionName(e), { exception: ApplicationInsights.Util.dump(e) })); - } - }; - AppInsights.prototype.clearAuthenticatedUserContext = function () { - try { - this.context.user.clearAuthenticatedUserContext(); - } - catch (e) { - ApplicationInsights._InternalLogging.throwInternalUserActionable(ApplicationInsights.LoggingSeverity.WARNING, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.USRACT_SetAuthContextFailed, "Clearing auth user context failed. " + ApplicationInsights.Util.getExceptionName(e), { exception: ApplicationInsights.Util.dump(e) })); - } - }; - AppInsights.prototype.SendCORSException = function (properties) { - var exceptionData = Microsoft.ApplicationInsights.Telemetry.Exception.CreateSimpleException("Script error.", "Error", "unknown", "unknown", "The browser’s same-origin policy prevents us from getting the details of this exception.The exception occurred in a script loaded from an origin different than the web page.For cross- domain error reporting you can use crossorigin attribute together with appropriate CORS HTTP headers.For more information please see http://www.w3.org/TR/cors/.", 0, null); - exceptionData.properties = properties; - var data = new ApplicationInsights.Telemetry.Common.Data(ApplicationInsights.Telemetry.Exception.dataType, exceptionData); - var envelope = new ApplicationInsights.Telemetry.Common.Envelope(data, ApplicationInsights.Telemetry.Exception.envelopeType); - this.context.track(envelope); - }; - AppInsights.prototype._onerror = function (message, url, lineNumber, columnNumber, error) { - try { - var properties = { url: url ? url : document.URL }; - if (ApplicationInsights.Util.isCrossOriginError(message, url, lineNumber, columnNumber, error)) { - this.SendCORSException(properties); - } - else { - if (!ApplicationInsights.Util.isError(error)) { - var stack = "window.onerror@" + properties.url + ":" + lineNumber + ":" + (columnNumber || 0); - error = new Error(message); - error["stack"] = stack; - } - this.trackException(error, null, properties); - } - } - catch (exception) { - var errorString = error ? (error.name + ", " + error.message) : "null"; - var exceptionDump = ApplicationInsights.Util.dump(exception); - ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.CRITICAL, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_ExceptionWhileLoggingError, "_onerror threw exception while logging error, error will not be collected: " + ApplicationInsights.Util.getExceptionName(exception), { exception: exceptionDump, errorString: errorString })); - } - }; - return AppInsights; - })(); - ApplicationInsights.AppInsights = AppInsights; - var Timing = (function () { - function Timing(name) { - this._name = name; - this._events = {}; - } - Timing.prototype.start = function (name) { - if (typeof this._events[name] !== "undefined") { - ApplicationInsights._InternalLogging.throwInternalUserActionable(ApplicationInsights.LoggingSeverity.WARNING, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.USRACT_StartCalledMoreThanOnce, "start was called more than once for this event without calling stop.", { name: this._name, key: name })); - } - this._events[name] = +new Date; - }; - Timing.prototype.stop = function (name, url, properties, measurements) { - var start = this._events[name]; - if (isNaN(start)) { - ApplicationInsights._InternalLogging.throwInternalUserActionable(ApplicationInsights.LoggingSeverity.WARNING, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.USRACT_StopCalledWithoutStart, "stop was called without a corresponding start.", { name: this._name, key: name })); - } - else { - var end = +new Date; - var duration = ApplicationInsights.Telemetry.PageViewPerformance.getDuration(start, end); - this.action(name, url, duration, properties, measurements); - } - delete this._events[name]; - this._events[name] = undefined; - }; - return Timing; - })(); - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -/// -var AI; -(function (AI) { - "use strict"; - var AjaxCallData = (function (_super) { - __extends(AjaxCallData, _super); - function AjaxCallData() { - this.ver = 2; - this.properties = {}; - this.measurements = {}; - _super.call(this); - } - return AjaxCallData; - })(AI.PageViewData); - AI.AjaxCallData = AjaxCallData; -})(AI || (AI = {})); -/// -var AI; -(function (AI) { - "use strict"; - var RequestData = (function (_super) { - __extends(RequestData, _super); - function RequestData() { - this.ver = 2; - this.properties = {}; - this.measurements = {}; - _super.call(this); - } - return RequestData; - })(Microsoft.Telemetry.Domain); - AI.RequestData = RequestData; -})(AI || (AI = {})); -/// -/// -var AI; -(function (AI) { - "use strict"; - var SessionStateData = (function (_super) { - __extends(SessionStateData, _super); - function SessionStateData() { - this.ver = 2; - this.state = AI.SessionState.Start; - _super.call(this); - } - return SessionStateData; - })(Microsoft.Telemetry.Domain); - AI.SessionStateData = SessionStateData; -})(AI || (AI = {})); -/// -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - "use strict"; - var Initialization = (function () { - function Initialization(snippet) { - snippet.queue = snippet.queue || []; - var config = snippet.config || {}; - if (config && !config.instrumentationKey) { - config = snippet; - if (config["iKey"]) { - Microsoft.ApplicationInsights.Version = "0.10.0.0"; - config.instrumentationKey = config["iKey"]; - } - else if (config["applicationInsightsId"]) { - Microsoft.ApplicationInsights.Version = "0.7.2.0"; - config.instrumentationKey = config["applicationInsightsId"]; - } - else { - throw new Error("Cannot load Application Insights SDK, no instrumentationKey was provided."); - } - } - config = Initialization.getDefaultConfig(config); - this.snippet = snippet; - this.config = config; - } - Initialization.prototype.loadAppInsights = function () { - var appInsights = new Microsoft.ApplicationInsights.AppInsights(this.config); - if (this.config["iKey"]) { - var originalTrackPageView = appInsights.trackPageView; - appInsights.trackPageView = function (pagePath, properties, measurements) { - originalTrackPageView.apply(appInsights, [null, pagePath, properties, measurements]); - }; - } - var legacyPageView = "logPageView"; - if (typeof this.snippet[legacyPageView] === "function") { - appInsights[legacyPageView] = function (pagePath, properties, measurements) { - appInsights.trackPageView(null, pagePath, properties, measurements); - }; - } - var legacyEvent = "logEvent"; - if (typeof this.snippet[legacyEvent] === "function") { - appInsights[legacyEvent] = function (name, properties, measurements) { - appInsights.trackEvent(name, properties, measurements); - }; - } - return appInsights; - }; - Initialization.prototype.emptyQueue = function () { - try { - if (Microsoft.ApplicationInsights.Util.isArray(this.snippet.queue)) { - var length = this.snippet.queue.length; - for (var i = 0; i < length; i++) { - var call = this.snippet.queue[i]; - call(); - } - this.snippet.queue = undefined; - delete this.snippet.queue; - } - } - catch (exception) { - var properties = {}; - if (exception && typeof exception.toString === "function") { - properties.exception = exception.toString(); - } - var message = new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_FailedToSendQueuedTelemetry, "Failed to send queued telemetry", properties); - Microsoft.ApplicationInsights._InternalLogging.throwInternalNonUserActionable(ApplicationInsights.LoggingSeverity.WARNING, message); - } - }; - Initialization.prototype.pollInteralLogs = function (appInsightsInstance) { - return setInterval(function () { - var queue = Microsoft.ApplicationInsights._InternalLogging.queue; - var length = queue.length; - for (var i = 0; i < length; i++) { - appInsightsInstance.trackTrace(queue[i].message); - } - queue.length = 0; - }, this.config.diagnosticLogInterval); - }; - Initialization.prototype.addHousekeepingBeforeUnload = function (appInsightsInstance) { - // Add callback to push events when the user navigates away - if (!appInsightsInstance.config.disableFlushOnBeforeUnload && ('onbeforeunload' in window)) { - var performHousekeeping = function () { - appInsightsInstance.context._sender.triggerSend(); - appInsightsInstance.context._sessionManager.backup(); - }; - if (!Microsoft.ApplicationInsights.Util.addEventHandler('beforeunload', performHousekeeping)) { - Microsoft.ApplicationInsights._InternalLogging.throwInternalNonUserActionable(Microsoft.ApplicationInsights.LoggingSeverity.CRITICAL, new ApplicationInsights._InternalLogMessage(ApplicationInsights._InternalMessageId.NONUSRACT_FailedToAddHandlerForOnBeforeUnload, 'Could not add handler for beforeunload')); - } - } - }; - Initialization.getDefaultConfig = function (config) { - if (!config) { - config = {}; - } - config.endpointUrl = config.endpointUrl || "//dc.services.visualstudio.com/v2/track"; - config.sessionRenewalMs = 30 * 60 * 1000; - config.sessionExpirationMs = 24 * 60 * 60 * 1000; - config.maxBatchSizeInBytes = config.maxBatchSizeInBytes > 0 ? config.maxBatchSizeInBytes : 1000000; - config.maxBatchInterval = !isNaN(config.maxBatchInterval) ? config.maxBatchInterval : 15000; - config.enableDebug = ApplicationInsights.Util.stringToBoolOrDefault(config.enableDebug); - config.disableExceptionTracking = (config.disableExceptionTracking !== undefined && config.disableExceptionTracking !== null) ? - ApplicationInsights.Util.stringToBoolOrDefault(config.disableExceptionTracking) : - false; - config.disableTelemetry = ApplicationInsights.Util.stringToBoolOrDefault(config.disableTelemetry); - config.verboseLogging = ApplicationInsights.Util.stringToBoolOrDefault(config.verboseLogging); - config.emitLineDelimitedJson = ApplicationInsights.Util.stringToBoolOrDefault(config.emitLineDelimitedJson); - config.diagnosticLogInterval = config.diagnosticLogInterval || 10000; - config.autoTrackPageVisitTime = ApplicationInsights.Util.stringToBoolOrDefault(config.autoTrackPageVisitTime); - if (isNaN(config.samplingPercentage) || config.samplingPercentage <= 0 || config.samplingPercentage >= 100) { - config.samplingPercentage = 100; - } - config.disableAjaxTracking = (config.disableAjaxTracking !== undefined && config.disableAjaxTracking !== null) ? - ApplicationInsights.Util.stringToBoolOrDefault(config.disableAjaxTracking) : - false; - config.maxAjaxCallsPerView = !isNaN(config.maxAjaxCallsPerView) ? config.maxAjaxCallsPerView : 500; - config.disableCorrelationHeaders = (config.disableCorrelationHeaders !== undefined && config.disableCorrelationHeaders !== null) ? - ApplicationInsights.Util.stringToBoolOrDefault(config.disableCorrelationHeaders) : - true; - config.disableFlushOnBeforeUnload = (config.disableFlushOnBeforeUnload !== undefined && config.disableFlushOnBeforeUnload !== null) ? - ApplicationInsights.Util.stringToBoolOrDefault(config.disableFlushOnBeforeUnload) : - false; - return config; - }; - return Initialization; - })(); - ApplicationInsights.Initialization = Initialization; - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); -/// -var Microsoft; -(function (Microsoft) { - var ApplicationInsights; - (function (ApplicationInsights) { - "use strict"; - try { - if (typeof window !== "undefined" && typeof JSON !== "undefined") { - var aiName = "appInsights"; - if (window[aiName] === undefined) { - Microsoft.ApplicationInsights.AppInsights.defaultConfig = Microsoft.ApplicationInsights.Initialization.getDefaultConfig(); - } - else { - var snippet = window[aiName] || {}; - var init = new Microsoft.ApplicationInsights.Initialization(snippet); - var appInsightsLocal = init.loadAppInsights(); - for (var field in appInsightsLocal) { - snippet[field] = appInsightsLocal[field]; - } - init.emptyQueue(); - init.pollInteralLogs(appInsightsLocal); - init.addHousekeepingBeforeUnload(appInsightsLocal); - } - } - } - catch (e) { - Microsoft.ApplicationInsights._InternalLogging.warnToConsole('Failed to initialize AppInsights JS SDK: ' + e.message); - } - })(ApplicationInsights = Microsoft.ApplicationInsights || (Microsoft.ApplicationInsights = {})); -})(Microsoft || (Microsoft = {})); diff --git a/src/NuGet.Services.BasicSearch/scripts/ai.0.22.9-build00167.min.js b/src/NuGet.Services.BasicSearch/scripts/ai.0.22.9-build00167.min.js deleted file mode 100644 index 4283fa31e..000000000 --- a/src/NuGet.Services.BasicSearch/scripts/ai.0.22.9-build00167.min.js +++ /dev/null @@ -1 +0,0 @@ -var __extends,AI,Microsoft;(function(n){var t;(function(n){var r,t,i,u;(function(n){n[n.CRITICAL=0]="CRITICAL";n[n.WARNING=1]="WARNING"})(n.LoggingSeverity||(n.LoggingSeverity={}));r=n.LoggingSeverity,function(n){n[n.NONUSRACT_BrowserDoesNotSupportLocalStorage=0]="NONUSRACT_BrowserDoesNotSupportLocalStorage";n[n.NONUSRACT_BrowserCannotReadLocalStorage=1]="NONUSRACT_BrowserCannotReadLocalStorage";n[n.NONUSRACT_BrowserCannotReadSessionStorage=2]="NONUSRACT_BrowserCannotReadSessionStorage";n[n.NONUSRACT_BrowserCannotWriteLocalStorage=3]="NONUSRACT_BrowserCannotWriteLocalStorage";n[n.NONUSRACT_BrowserCannotWriteSessionStorage=4]="NONUSRACT_BrowserCannotWriteSessionStorage";n[n.NONUSRACT_BrowserFailedRemovalFromLocalStorage=5]="NONUSRACT_BrowserFailedRemovalFromLocalStorage";n[n.NONUSRACT_BrowserFailedRemovalFromSessionStorage=6]="NONUSRACT_BrowserFailedRemovalFromSessionStorage";n[n.NONUSRACT_CannotSendEmptyTelemetry=7]="NONUSRACT_CannotSendEmptyTelemetry";n[n.NONUSRACT_ClientPerformanceMathError=8]="NONUSRACT_ClientPerformanceMathError";n[n.NONUSRACT_ErrorParsingAISessionCookie=9]="NONUSRACT_ErrorParsingAISessionCookie";n[n.NONUSRACT_ErrorPVCalc=10]="NONUSRACT_ErrorPVCalc";n[n.NONUSRACT_ExceptionWhileLoggingError=11]="NONUSRACT_ExceptionWhileLoggingError";n[n.NONUSRACT_FailedAddingTelemetryToBuffer=12]="NONUSRACT_FailedAddingTelemetryToBuffer";n[n.NONUSRACT_FailedMonitorAjaxAbort=13]="NONUSRACT_FailedMonitorAjaxAbort";n[n.NONUSRACT_FailedMonitorAjaxDur=14]="NONUSRACT_FailedMonitorAjaxDur";n[n.NONUSRACT_FailedMonitorAjaxOpen=15]="NONUSRACT_FailedMonitorAjaxOpen";n[n.NONUSRACT_FailedMonitorAjaxRSC=16]="NONUSRACT_FailedMonitorAjaxRSC";n[n.NONUSRACT_FailedMonitorAjaxSend=17]="NONUSRACT_FailedMonitorAjaxSend";n[n.NONUSRACT_FailedToAddHandlerForOnBeforeUnload=18]="NONUSRACT_FailedToAddHandlerForOnBeforeUnload";n[n.NONUSRACT_FailedToSendQueuedTelemetry=19]="NONUSRACT_FailedToSendQueuedTelemetry";n[n.NONUSRACT_FailedToReportDataLoss=20]="NONUSRACT_FailedToReportDataLoss";n[n.NONUSRACT_FlushFailed=21]="NONUSRACT_FlushFailed";n[n.NONUSRACT_MessageLimitPerPVExceeded=22]="NONUSRACT_MessageLimitPerPVExceeded";n[n.NONUSRACT_MissingRequiredFieldSpecification=23]="NONUSRACT_MissingRequiredFieldSpecification";n[n.NONUSRACT_NavigationTimingNotSupported=24]="NONUSRACT_NavigationTimingNotSupported";n[n.NONUSRACT_OnError=25]="NONUSRACT_OnError";n[n.NONUSRACT_SessionRenewalDateIsZero=26]="NONUSRACT_SessionRenewalDateIsZero";n[n.NONUSRACT_SenderNotInitialized=27]="NONUSRACT_SenderNotInitialized";n[n.NONUSRACT_StartTrackEventFailed=28]="NONUSRACT_StartTrackEventFailed";n[n.NONUSRACT_StopTrackEventFailed=29]="NONUSRACT_StopTrackEventFailed";n[n.NONUSRACT_StartTrackFailed=30]="NONUSRACT_StartTrackFailed";n[n.NONUSRACT_StopTrackFailed=31]="NONUSRACT_StopTrackFailed";n[n.NONUSRACT_TelemetrySampledAndNotSent=32]="NONUSRACT_TelemetrySampledAndNotSent";n[n.NONUSRACT_TrackEventFailed=33]="NONUSRACT_TrackEventFailed";n[n.NONUSRACT_TrackExceptionFailed=34]="NONUSRACT_TrackExceptionFailed";n[n.NONUSRACT_TrackMetricFailed=35]="NONUSRACT_TrackMetricFailed";n[n.NONUSRACT_TrackPVFailed=36]="NONUSRACT_TrackPVFailed";n[n.NONUSRACT_TrackPVFailedCalc=37]="NONUSRACT_TrackPVFailedCalc";n[n.NONUSRACT_TrackTraceFailed=38]="NONUSRACT_TrackTraceFailed";n[n.NONUSRACT_TransmissionFailed=39]="NONUSRACT_TransmissionFailed";n[n.USRACT_CannotSerializeObject=40]="USRACT_CannotSerializeObject";n[n.USRACT_CannotSerializeObjectNonSerializable=41]="USRACT_CannotSerializeObjectNonSerializable";n[n.USRACT_CircularReferenceDetected=42]="USRACT_CircularReferenceDetected";n[n.USRACT_ClearAuthContextFailed=43]="USRACT_ClearAuthContextFailed";n[n.USRACT_ExceptionTruncated=44]="USRACT_ExceptionTruncated";n[n.USRACT_IllegalCharsInName=45]="USRACT_IllegalCharsInName";n[n.USRACT_ItemNotInArray=46]="USRACT_ItemNotInArray";n[n.USRACT_MaxAjaxPerPVExceeded=47]="USRACT_MaxAjaxPerPVExceeded";n[n.USRACT_MessageTruncated=48]="USRACT_MessageTruncated";n[n.USRACT_NameTooLong=49]="USRACT_NameTooLong";n[n.USRACT_SampleRateOutOfRange=50]="USRACT_SampleRateOutOfRange";n[n.USRACT_SetAuthContextFailed=51]="USRACT_SetAuthContextFailed";n[n.USRACT_SetAuthContextFailedAccountName=52]="USRACT_SetAuthContextFailedAccountName";n[n.USRACT_StringValueTooLong=53]="USRACT_StringValueTooLong";n[n.USRACT_StartCalledMoreThanOnce=54]="USRACT_StartCalledMoreThanOnce";n[n.USRACT_StopCalledWithoutStart=55]="USRACT_StopCalledWithoutStart";n[n.USRACT_TelemetryInitializerFailed=56]="USRACT_TelemetryInitializerFailed";n[n.USRACT_TrackArgumentsNotSpecified=57]="USRACT_TrackArgumentsNotSpecified";n[n.USRACT_UrlTooLong=58]="USRACT_UrlTooLong"}(n._InternalMessageId||(n._InternalMessageId={}));t=n._InternalMessageId;i=function(){function n(i,r,u){this.message=t[i].toString();this.messageId=i;var f=(r?" message:"+n.sanitizeDiagnosticText(r):"")+(u?" props:"+n.sanitizeDiagnosticText(JSON.stringify(u)):"");this.message+=f}return n.sanitizeDiagnosticText=function(n){return'"'+n.replace(/\"/g,"")+'"'},n}();n._InternalLogMessage=i;u=function(){function u(){}return u.throwInternalNonUserActionable=function(n,t){if(this.enableDebugExceptions())throw t;else typeof t=="undefined"||!t||typeof t.message!="undefined"&&(t.message=this.AiNonUserActionablePrefix+t.message,this.warnToConsole(t.message),this.logInternalMessage(n,t))},u.throwInternalUserActionable=function(n,t){if(this.enableDebugExceptions())throw t;else typeof t=="undefined"||!t||typeof t.message!="undefined"&&(t.message=this.AiUserActionablePrefix+t.message,this.warnToConsole(t.message),this.logInternalMessage(n,t))},u.warnToConsole=function(n){typeof console=="undefined"||!console||(typeof console.warn=="function"?console.warn(n):typeof console.log=="function"&&console.log(n))},u.resetInternalMessageCount=function(){this._messageCount=0},u.clearInternalMessageLoggedTypes=function(){var i,t;if(n.Util.canUseSessionStorage())for(i=n.Util.getSessionStorageKeys(),t=0;t=this.MAX_INTERNAL_MESSAGE_LIMIT},u.AiUserActionablePrefix="AI: ",u.AIInternalMessagePrefix="AITR_",u.AiNonUserActionablePrefix="AI (Internal): ",u.enableDebugExceptions=function(){return!1},u.verboseLogging=function(){return!1},u.queue=[],u.MAX_INTERNAL_MESSAGE_LIMIT=25,u._messageCount=0,u}();n._InternalLogging=u})(t=n.ApplicationInsights||(n.ApplicationInsights={}))})(Microsoft||(Microsoft={})),function(n){var t;(function(n){var t,i,r;(function(n){n[n.LocalStorage=0]="LocalStorage";n[n.SessionStorage=1]="SessionStorage"})(t||(t={}));i=function(){function i(){}return i._getLocalStorageObject=function(){return i._getVerifiedStorageObject(t.LocalStorage)},i._getVerifiedStorageObject=function(n){var i=null,u,r;try{r=new Date;i=n===t.LocalStorage?window.localStorage:window.sessionStorage;i.setItem(r,r);u=i.getItem(r)!=r;i.removeItem(r);u&&(i=null)}catch(f){i=null}return i},i.canUseLocalStorage=function(){return!!i._getLocalStorageObject()},i.getStorage=function(t){var r=i._getLocalStorageObject(),f;if(r!==null)try{return r.getItem(t)}catch(u){f=new n._InternalLogMessage(n._InternalMessageId.NONUSRACT_BrowserCannotReadLocalStorage,"Browser failed read of local storage. "+i.getExceptionName(u),{exception:i.dump(u)});n._InternalLogging.throwInternalNonUserActionable(n.LoggingSeverity.WARNING,f)}return null},i.setStorage=function(t,r){var u=i._getLocalStorageObject(),e;if(u!==null)try{return u.setItem(t,r),!0}catch(f){e=new n._InternalLogMessage(n._InternalMessageId.NONUSRACT_BrowserCannotWriteLocalStorage,"Browser failed write to local storage. "+i.getExceptionName(f),{exception:i.dump(f)});n._InternalLogging.throwInternalNonUserActionable(n.LoggingSeverity.WARNING,e)}return!1},i.removeStorage=function(t){var r=i._getLocalStorageObject(),f;if(r!==null)try{return r.removeItem(t),!0}catch(u){f=new n._InternalLogMessage(n._InternalMessageId.NONUSRACT_BrowserFailedRemovalFromLocalStorage,"Browser failed removal of local storage item. "+i.getExceptionName(u),{exception:i.dump(u)});n._InternalLogging.throwInternalNonUserActionable(n.LoggingSeverity.WARNING,f)}return!1},i._getSessionStorageObject=function(){return i._getVerifiedStorageObject(t.SessionStorage)},i.canUseSessionStorage=function(){return!!i._getSessionStorageObject()},i.getSessionStorageKeys=function(){var n=[],t;if(i.canUseSessionStorage())for(t in window.sessionStorage)n.push(t);return n},i.getSessionStorage=function(t){var r=i._getSessionStorageObject(),f;if(r!==null)try{return r.getItem(t)}catch(u){f=new n._InternalLogMessage(n._InternalMessageId.NONUSRACT_BrowserCannotReadSessionStorage,"Browser failed read of session storage. "+i.getExceptionName(u),{exception:i.dump(u)});n._InternalLogging.throwInternalNonUserActionable(n.LoggingSeverity.CRITICAL,f)}return null},i.setSessionStorage=function(t,r){var u=i._getSessionStorageObject(),e;if(u!==null)try{return u.setItem(t,r),!0}catch(f){e=new n._InternalLogMessage(n._InternalMessageId.NONUSRACT_BrowserCannotWriteSessionStorage,"Browser failed write to session storage. "+i.getExceptionName(f),{exception:i.dump(f)});n._InternalLogging.throwInternalNonUserActionable(n.LoggingSeverity.CRITICAL,e)}return!1},i.removeSessionStorage=function(t){var r=i._getSessionStorageObject(),f;if(r!==null)try{return r.removeItem(t),!0}catch(u){f=new n._InternalLogMessage(n._InternalMessageId.NONUSRACT_BrowserFailedRemovalFromSessionStorage,"Browser failed removal of session storage item. "+i.getExceptionName(u),{exception:i.dump(u)});n._InternalLogging.throwInternalNonUserActionable(n.LoggingSeverity.CRITICAL,f)}return!1},i.setCookie=function(n,t,r){var u="";r&&(u=";domain="+r);i.document.cookie=n+"="+t+u+";path=/"},i.stringToBoolOrDefault=function(n){return n?n.toString().toLowerCase()==="true":!1},i.getCookie=function(n){var e="",f,u,r,t;if(n&&n.length)for(f=n+"=",u=i.document.cookie.split(";"),r=0;r0;)i="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/".charAt(n%64),t+=i,n=Math.floor(n/64);return t},i.isArray=function(n){return Object.prototype.toString.call(n)==="[object Array]"},i.isError=function(n){return Object.prototype.toString.call(n)==="[object Error]"},i.isDate=function(n){return Object.prototype.toString.call(n)==="[object Date]"},i.toISOStringForIE8=function(n){if(i.isDate(n)){function t(n){var t=String(n);return t.length===1&&(t="0"+t),t}return Date.prototype.toISOString?n.toISOString():n.getUTCFullYear()+"-"+t(n.getUTCMonth()+1)+"-"+t(n.getUTCDate())+"T"+t(n.getUTCHours())+":"+t(n.getUTCMinutes())+":"+t(n.getUTCSeconds())+"."+String((n.getUTCMilliseconds()/1e3).toFixed(3)).slice(2,5)+"Z"}},i.getIEVersion=function(n){n===void 0&&(n=null);var t=n?n.toLowerCase():navigator.userAgent.toLowerCase();return t.indexOf("msie")!=-1?parseInt(t.split("msie")[1]):null},i.msToTimeSpan=function(n){(isNaN(n)||n<0)&&(n=0);var t=""+n%1e3,i=""+Math.floor(n/1e3)%60,r=""+Math.floor(n/6e4)%60,u=""+Math.floor(n/36e5)%24;return t=t.length===1?"00"+t:t.length===2?"0"+t:t,i=i.length<2?"0"+i:i,r=r.length<2?"0"+r:r,u=u.length<2?"0"+u:u,u+":"+r+":"+i+"."+t},i.isCrossOriginError=function(n,t,i,r,u){return(n==="Script error."||n==="Script error")&&u===null},i.dump=function(n){var t=Object.prototype.toString.call(n),i=JSON.stringify(n);return t==="[object Error]"&&(i="{ stack: '"+n.stack+"', message: '"+n.message+"', name: '"+n.name+"'"),t+i},i.getExceptionName=function(n){var t=Object.prototype.toString.call(n);return t==="[object Error]"?n.name:""},i.addEventHandler=function(n,t){if(!window||typeof n!="string"||typeof t!="function")return!1;var i="on"+n;if(window.addEventListener)window.addEventListener(n,t,!1);else if(window.attachEvent)window.attachEvent.call(i,t);else return!1;return!0},i.document=typeof document!="undefined"?document:{},i.NotSpecified="not_specified",i}();n.Util=i;r=function(){function n(){}return n.parseUrl=function(t){return n.htmlAnchorElement||(n.htmlAnchorElement=n.document.createElement("a")),n.htmlAnchorElement.href=t,n.htmlAnchorElement},n.getAbsoluteUrl=function(t){var i,r=n.parseUrl(t);return r&&(i=r.href),i},n.getPathName=function(t){var i,r=n.parseUrl(t);return r&&(i=r.pathname),i},n.document=typeof document!="undefined"?document:{},n}();n.UrlHelper=r})(t=n.ApplicationInsights||(n.ApplicationInsights={}))}(Microsoft||(Microsoft={})),function(n){var t;(function(n){"use strict";var t=function(){function n(){}return n.IsNullOrUndefined=function(n){return typeof n=="undefined"||n===null},n}(),i,r,u;n.extensions=t;i=function(){function n(){}return n.GetLength=function(n){var i=0,r;if(!t.IsNullOrUndefined(n)){r="";try{r=n.toString()}catch(u){}i=r.length;i=isNaN(i)?0:i}return i},n}();n.stringUtils=i;r=function(){function n(){}return n.Now=window.performance&&window.performance.now?function(){return performance.now()}:function(){return(new Date).getTime()},n.GetDuration=function(n,i){var r=null;return n===0||i===0||t.IsNullOrUndefined(n)||t.IsNullOrUndefined(i)||(r=i-n),r},n}();n.dateTime=r;u=function(){function n(){}return n.AttachEvent=function(n,i,r){var u=!1;return t.IsNullOrUndefined(n)||(t.IsNullOrUndefined(n.attachEvent)?t.IsNullOrUndefined(n.addEventListener)||(n.addEventListener(i,r,!1),u=!0):(n.attachEvent("on"+i,r),u=!0)),u},n.DetachEvent=function(n,i,r){t.IsNullOrUndefined(n)||(t.IsNullOrUndefined(n.detachEvent)?t.IsNullOrUndefined(n.removeEventListener)||n.removeEventListener(i,r,!1):n.detachEvent("on"+i,r))},n}();n.EventHelper=u})(t=n.ApplicationInsights||(n.ApplicationInsights={}))}(Microsoft||(Microsoft={})),function(n){var t;(function(n){"use strict";var t=function(){function n(){this.openDone=!1;this.setRequestHeaderDone=!1;this.sendDone=!1;this.abortDone=!1;this.onreadystatechangeCallbackAttached=!1}return n}(),i;n.XHRMonitoringState=t;i=function(){function i(i){this.completed=!1;this.requestHeadersSize=null;this.ttfb=null;this.responseReceivingDuration=null;this.callbackDuration=null;this.ajaxTotalDuration=null;this.aborted=null;this.pageUrl=null;this.requestUrl=null;this.requestSize=0;this.method=null;this.status=null;this.requestSentTime=null;this.responseStartedTime=null;this.responseFinishedTime=null;this.callbackFinishedTime=null;this.endTime=null;this.originalOnreadystatechage=null;this.xhrMonitoringState=new t;this.clientFailure=0;this.CalculateMetrics=function(){var t=this;t.ajaxTotalDuration=n.dateTime.GetDuration(t.requestSentTime,t.responseFinishedTime)};this.id=i}return i.prototype.getAbsoluteUrl=function(){return this.requestUrl?n.UrlHelper.getAbsoluteUrl(this.requestUrl):null},i.prototype.getPathName=function(){return this.requestUrl?n.UrlHelper.getPathName(this.requestUrl):null},i}();n.ajaxRecord=i})(t=n.ApplicationInsights||(n.ApplicationInsights={}))}(Microsoft||(Microsoft={})),function(n){var t;(function(t){"use strict";var i=function(){function i(n){this.currentWindowHost=window.location.host;this.appInsights=n;this.initialized=!1;this.Init()}return i.prototype.Init=function(){this.supportsMonitoring()&&(this.instrumentOpen(),this.instrumentSend(),this.instrumentAbort(),this.initialized=!0)},i.prototype.isMonitoredInstance=function(n,r){return this.initialized&&(r===!0||!t.extensions.IsNullOrUndefined(n.ajaxData))&&n[i.DisabledPropertyName]!==!0},i.prototype.supportsMonitoring=function(){var n=!1;return t.extensions.IsNullOrUndefined(XMLHttpRequest)||(n=!0),n},i.prototype.instrumentOpen=function(){var u=XMLHttpRequest.prototype.open,r=this;XMLHttpRequest.prototype.open=function(f,e,o){try{!r.isMonitoredInstance(this,!0)||this.ajaxData&&this.ajaxData.xhrMonitoringState.openDone||r.openHandler(this,f,e,o)}catch(s){t._InternalLogging.throwInternalNonUserActionable(t.LoggingSeverity.CRITICAL,new t._InternalLogMessage(t._InternalMessageId.NONUSRACT_FailedMonitorAjaxOpen,"Failed to monitor XMLHttpRequest.open, monitoring data for this ajax call may be incorrect.",{ajaxDiagnosticsMessage:i.getFailedAjaxDiagnosticsMessage(this),exception:n.ApplicationInsights.Util.dump(s)}))}return u.apply(this,arguments)}},i.prototype.openHandler=function(n,i,r){var u=new t.ajaxRecord(t.Util.newId());u.method=i;u.requestUrl=r;u.xhrMonitoringState.openDone=!0;n.ajaxData=u;this.attachToOnReadyStateChange(n)},i.getFailedAjaxDiagnosticsMessage=function(n){var i="";try{t.extensions.IsNullOrUndefined(n)||t.extensions.IsNullOrUndefined(n.ajaxData)||t.extensions.IsNullOrUndefined(n.ajaxData.requestUrl)||(i+="(url: '"+n.ajaxData.requestUrl+"')")}catch(r){}return i},i.prototype.instrumentSend=function(){var u=XMLHttpRequest.prototype.send,r=this;XMLHttpRequest.prototype.send=function(f){try{r.isMonitoredInstance(this)&&!this.ajaxData.xhrMonitoringState.sendDone&&r.sendHandler(this,f)}catch(e){t._InternalLogging.throwInternalNonUserActionable(t.LoggingSeverity.CRITICAL,new t._InternalLogMessage(t._InternalMessageId.NONUSRACT_FailedMonitorAjaxSend,"Failed to monitor XMLHttpRequest, monitoring data for this ajax call may be incorrect.",{ajaxDiagnosticsMessage:i.getFailedAjaxDiagnosticsMessage(this),exception:n.ApplicationInsights.Util.dump(e)}))}return u.apply(this,arguments)}},i.prototype.sendHandler=function(n){n.ajaxData.requestSentTime=t.dateTime.Now();this.appInsights.config.disableCorrelationHeaders||t.UrlHelper.parseUrl(n.ajaxData.getAbsoluteUrl()).host!=this.currentWindowHost||n.setRequestHeader("x-ms-request-id",n.ajaxData.id);n.ajaxData.xhrMonitoringState.sendDone=!0},i.prototype.instrumentAbort=function(){var r=XMLHttpRequest.prototype.abort,u=this;XMLHttpRequest.prototype.abort=function(){try{u.isMonitoredInstance(this)&&!this.ajaxData.xhrMonitoringState.abortDone&&(this.ajaxData.aborted=1,this.ajaxData.xhrMonitoringState.abortDone=!0)}catch(f){t._InternalLogging.throwInternalNonUserActionable(t.LoggingSeverity.CRITICAL,new t._InternalLogMessage(t._InternalMessageId.NONUSRACT_FailedMonitorAjaxAbort,"Failed to monitor XMLHttpRequest.abort, monitoring data for this ajax call may be incorrect.",{ajaxDiagnosticsMessage:i.getFailedAjaxDiagnosticsMessage(this),exception:n.ApplicationInsights.Util.dump(f)}))}return r.apply(this,arguments)}},i.prototype.attachToOnReadyStateChange=function(r){var u=this;r.ajaxData.xhrMonitoringState.onreadystatechangeCallbackAttached=t.EventHelper.AttachEvent(r,"readystatechange",function(){try{if(u.isMonitoredInstance(r)&&r.readyState===4)u.onAjaxComplete(r)}catch(f){var e=n.ApplicationInsights.Util.dump(f);e&&e.toLowerCase().indexOf("c00c023f")!=-1||t._InternalLogging.throwInternalNonUserActionable(t.LoggingSeverity.CRITICAL,new t._InternalLogMessage(t._InternalMessageId.NONUSRACT_FailedMonitorAjaxRSC,"Failed to monitor XMLHttpRequest 'readystatechange' event handler, monitoring data for this ajax call may be incorrect.",{ajaxDiagnosticsMessage:i.getFailedAjaxDiagnosticsMessage(r),exception:n.ApplicationInsights.Util.dump(f)}))}})},i.prototype.onAjaxComplete=function(n){n.ajaxData.responseFinishedTime=t.dateTime.Now();n.ajaxData.status=n.status;n.ajaxData.CalculateMetrics();n.ajaxData.ajaxTotalDuration<0?t._InternalLogging.throwInternalNonUserActionable(t.LoggingSeverity.WARNING,new t._InternalLogMessage(t._InternalMessageId.NONUSRACT_FailedMonitorAjaxDur,"Failed to calculate the duration of the ajax call, monitoring data for this ajax call won't be sent.",{ajaxDiagnosticsMessage:i.getFailedAjaxDiagnosticsMessage(n),requestSentTime:n.ajaxData.requestSentTime,responseFinishedTime:n.ajaxData.responseFinishedTime})):(this.appInsights.trackAjax(n.ajaxData.id,n.ajaxData.getAbsoluteUrl(),n.ajaxData.getPathName(),n.ajaxData.ajaxTotalDuration,+n.ajaxData.status>=200&&+n.ajaxData.status<400,+n.ajaxData.status),n.ajaxData=null)},i.instrumentedByAppInsightsName="InstrumentedByAppInsights",i.DisabledPropertyName="Microsoft_ApplicationInsights_BypassAjaxInstrumentation",i}();t.AjaxMonitor=i})(t=n.ApplicationInsights||(n.ApplicationInsights={}))}(Microsoft||(Microsoft={})),function(n){var t;(function(n){var t=function(){function n(){}return n.prototype.getHashCodeScore=function(t){var i=this.getHashCode(t)/n.INT_MAX_VALUE;return i*100},n.prototype.getHashCode=function(t){var i,r;if(t=="")return 0;while(t.length100||t<0)&&(n._InternalLogging.throwInternalUserActionable(n.LoggingSeverity.WARNING,new n._InternalLogMessage(n._InternalMessageId.USRACT_SampleRateOutOfRange,"Sampling rate is out of range (0..100). Sampling will be disabled, you may be sending too much data which may affect your AI service level.",{samplingRate:t})),this.sampleRate=100);this.sampleRate=t;this.samplingScoreGenerator=new n.SamplingScoreGenerator}return t.prototype.isSampledIn=function(n){if(this.sampleRate==100)return!0;var t=this.samplingScoreGenerator.getSamplingScore(n);return tthis.config.sessionExpirationMs(),i=n-this.automaticSession.renewalDate>this.config.sessionRenewalMs();t||i?(this.automaticSession.isFirst=undefined,this.renew()):(this.automaticSession.renewalDate=+new Date,this.setCookie(this.automaticSession.id,this.automaticSession.acquisitionDate,this.automaticSession.renewalDate))},t.prototype.backup=function(){this.setStorage(this.automaticSession.id,this.automaticSession.acquisitionDate,this.automaticSession.renewalDate)},t.prototype.initializeAutomaticSession=function(){var t=n.Util.getCookie("ai_session"),i;t&&typeof t.split=="function"?this.initializeAutomaticSessionWithData(t):(i=n.Util.getStorage("ai_session"),i&&this.initializeAutomaticSessionWithData(i));this.automaticSession.id||(this.automaticSession.isFirst=!0,this.renew())},t.prototype.initializeAutomaticSessionWithData=function(t){var i=t.split("|"),r,u;i.length>0&&(this.automaticSession.id=i[0]);try{i.length>1&&(r=+i[1],this.automaticSession.acquisitionDate=+new Date(r),this.automaticSession.acquisitionDate=this.automaticSession.acquisitionDate>0?this.automaticSession.acquisitionDate:0);i.length>2&&(u=+i[2],this.automaticSession.renewalDate=+new Date(u),this.automaticSession.renewalDate=this.automaticSession.renewalDate>0?this.automaticSession.renewalDate:0)}catch(f){n._InternalLogging.throwInternalNonUserActionable(n.LoggingSeverity.CRITICAL,new n._InternalLogMessage(n._InternalMessageId.NONUSRACT_ErrorParsingAISessionCookie,"Error parsing ai_session cookie, session will be reset: "+n.Util.getExceptionName(f),{exception:n.Util.dump(f)}))}this.automaticSession.renewalDate==0&&n._InternalLogging.throwInternalNonUserActionable(n.LoggingSeverity.WARNING,new n._InternalLogMessage(n._InternalMessageId.NONUSRACT_SessionRenewalDateIsZero,"AI session renewal date is 0, session will be reset."))},t.prototype.renew=function(){var t=+new Date;this.automaticSession.id=n.Util.newId();this.automaticSession.acquisitionDate=t;this.automaticSession.renewalDate=t;this.setCookie(this.automaticSession.id,this.automaticSession.acquisitionDate,this.automaticSession.renewalDate);n.Util.canUseLocalStorage()||n._InternalLogging.throwInternalNonUserActionable(n.LoggingSeverity.WARNING,new n._InternalLogMessage(n._InternalMessageId.NONUSRACT_BrowserDoesNotSupportLocalStorage,"Browser does not support local storage. Session durations will be inaccurate."))},t.prototype.setCookie=function(t,i,r){var f=i+this.config.sessionExpirationMs(),e=r+this.config.sessionRenewalMs(),u=new Date,s=[t,i,r],o;f0&&(this.id=e[0]));this.config=i;this.id||(this.id=n.Util.newId(),u=new Date,o=n.Util.toISOStringForIE8(u),this.accountAcquisitionDate=o,u.setTime(u.getTime()+31536e6),h=[this.id,o],c=this.config.cookieDomain?this.config.cookieDomain():undefined,n.Util.setCookie(t.userCookieName,h.join(t.cookieSeparator)+";expires="+u.toUTCString(),c),n.Util.removeStorage("ai_session"));this.accountId=i.accountId?i.accountId():undefined;f=n.Util.getCookie(t.authUserCookieName);f&&(f=decodeURI(f),r=f.split(t.cookieSeparator),r[0]&&(this.authenticatedId=r[0]),r.length>1&&r[1]&&(this.accountId=r[1]))}return t.prototype.setAuthenticatedUserContext=function(i,r){var f=!this.validateUserInput(i)||r&&!this.validateUserInput(r),u;if(f){n._InternalLogging.throwInternalUserActionable(n.LoggingSeverity.WARNING,new n._InternalLogMessage(n._InternalMessageId.USRACT_SetAuthContextFailedAccountName,"Setting auth user context failed. User auth/account id should be of type string, and not contain commas, semi-colons, equal signs, spaces, or vertical-bars."));return}this.authenticatedId=i;u=this.authenticatedId;r&&(this.accountId=r,u=[this.authenticatedId,this.accountId].join(t.cookieSeparator));n.Util.setCookie(t.authUserCookieName,encodeURI(u),this.config.cookieDomain())},t.prototype.clearAuthenticatedUserContext=function(){this.authenticatedId=null;this.accountId=null;n.Util.deleteCookie(t.authUserCookieName)},t.prototype.validateUserInput=function(n){return typeof n!="string"||!n||n.match(/,|;|=| |\|/)?!1:!0},t.cookieSeparator="|",t.userCookieName="ai_user",t.authUserCookieName="ai_authUser",t}();t.User=i})(t=n.Context||(n.Context={}))})(t=n.ApplicationInsights||(n.ApplicationInsights={}))}(Microsoft||(Microsoft={})),function(n){var t;(function(n){"use strict";var t=function(){function t(){}return t.reset=function(){t.isEnabled()&&n.Util.setSessionStorage(t.ITEMS_QUEUED_KEY,"0")},t.isEnabled=function(){return t.enabled&&t.appInsights!=null&&n.Util.canUseSessionStorage()},t.getIssuesReported=function(){return!t.isEnabled()||isNaN(+n.Util.getSessionStorage(t.ISSUES_REPORTED_KEY))?0:+n.Util.getSessionStorage(t.ISSUES_REPORTED_KEY)},t.incrementItemsQueued=function(){try{if(t.isEnabled()){var i=t.getNumberOfLostItems();++i;n.Util.setSessionStorage(t.ITEMS_QUEUED_KEY,i.toString())}}catch(r){}},t.decrementItemsQueued=function(i){try{if(t.isEnabled()){var r=t.getNumberOfLostItems();r-=i;r<0&&(r=0);n.Util.setSessionStorage(t.ITEMS_QUEUED_KEY,r.toString())}}catch(u){}},t.getNumberOfLostItems=function(){var i=0;try{t.isEnabled()&&(i=isNaN(+n.Util.getSessionStorage(t.ITEMS_QUEUED_KEY))?0:+n.Util.getSessionStorage(t.ITEMS_QUEUED_KEY))}catch(r){i=0}return i},t.reportLostItems=function(){try{if(t.isEnabled()&&t.getIssuesReported()0){t.appInsights.trackTrace("AI (Internal): Internal report DATALOSS: "+t.getNumberOfLostItems(),null);t.appInsights.flush();var i=t.getIssuesReported();++i;n.Util.setSessionStorage(t.ISSUES_REPORTED_KEY,i.toString())}}catch(r){n._InternalLogging.throwInternalNonUserActionable(n.LoggingSeverity.CRITICAL,new n._InternalLogMessage(n._InternalMessageId.NONUSRACT_FailedToReportDataLoss,"Failed to report data loss: "+n.Util.getExceptionName(r),{exception:n.Util.dump(r)}))}finally{try{t.reset()}catch(r){}}},t.enabled=!1,t.LIMIT_PER_SESSION=10,t.ITEMS_QUEUED_KEY="AI_itemsQueued",t.ISSUES_REPORTED_KEY="AI_lossIssuesReported",t}();n.DataLossAnalyzer=t})(t=n.ApplicationInsights||(n.ApplicationInsights={}))}(Microsoft||(Microsoft={})),function(n){var t;(function(n){"use strict";var t=function(){function t(n){if(this._buffer=[],this._lastSend=0,this._config=n,this._sender=null,typeof XMLHttpRequest!="undefined"){var t=new XMLHttpRequest;"withCredentials"in t?this._sender=this._xhrSender:typeof XDomainRequest!="undefined"&&(this._sender=this._xdrSender)}}return t.prototype.send=function(t){var r=this,i;try{if(this._config.disableTelemetry())return;if(!t){n._InternalLogging.throwInternalNonUserActionable(n.LoggingSeverity.CRITICAL,new n._InternalLogMessage(n._InternalMessageId.NONUSRACT_CannotSendEmptyTelemetry,"Cannot send empty telemetry"));return}if(!this._sender){n._InternalLogging.throwInternalNonUserActionable(n.LoggingSeverity.CRITICAL,new n._InternalLogMessage(n._InternalMessageId.NONUSRACT_SenderNotInitialized,"Sender was not initialized"));return}i=n.Serializer.serialize(t);this._getSizeInBytes(this._buffer)+i.length>this._config.maxBatchSizeInBytes()&&this.triggerSend();this._buffer.push(i);this._timeoutHandle||(this._timeoutHandle=setTimeout(function(){r._timeoutHandle=null;r.triggerSend()},this._config.maxBatchInterval()));n.DataLossAnalyzer.incrementItemsQueued()}catch(u){n._InternalLogging.throwInternalNonUserActionable(n.LoggingSeverity.CRITICAL,new n._InternalLogMessage(n._InternalMessageId.NONUSRACT_FailedAddingTelemetryToBuffer,"Failed adding telemetry to the sender's buffer, some telemetry will be lost: "+n.Util.getExceptionName(u),{exception:n.Util.dump(u)}))}},t.prototype._getSizeInBytes=function(n){var r=0,t,i;if(n&&n.length)for(t=0;t9)&&n._InternalLogging.throwInternalNonUserActionable(n.LoggingSeverity.CRITICAL,new n._InternalLogMessage(n._InternalMessageId.NONUSRACT_TransmissionFailed,"Telemetry transmission failed, some telemetry will be lost: "+n.Util.getExceptionName(u),{exception:n.Util.dump(u)}))}},t.prototype._xhrSender=function(i,r,u){var f=new XMLHttpRequest;f[n.AjaxMonitor.DisabledPropertyName]=!0;f.open("POST",this._config.endpointUrl(),r);f.setRequestHeader("Content-type","application/json");f.onreadystatechange=function(){return t._xhrReadyStateChange(f,i,u)};f.onerror=function(n){return t._onError(i,f.responseText||f.response||"",n)};f.send(i)},t.prototype._xdrSender=function(n){var i=new XDomainRequest;i.onload=function(){return t._xdrOnLoad(i,n)};i.onerror=function(r){return t._onError(n,i.responseText||"",r)};i.open("POST",this._config.endpointUrl());i.send(n)},t._xhrReadyStateChange=function(n,i,r){n.readyState===4&&((n.status<200||n.status>=300)&&n.status!==0?t._onError(i,n.responseText||n.response||""):t._onSuccess(i,r))},t._xdrOnLoad=function(n,i){n&&(n.responseText+""=="200"||n.responseText==="")?t._onSuccess(i,0):t._onError(i,n&&n.responseText||"")},t._onError=function(t,i){n._InternalLogging.throwInternalNonUserActionable(n.LoggingSeverity.WARNING,new n._InternalLogMessage(n._InternalMessageId.NONUSRACT_OnError,"Failed to send telemetry.",{message:i}))},t._onSuccess=function(t,i){n.DataLossAnalyzer.decrementItemsQueued(i)},t}();n.Sender=t})(t=n.ApplicationInsights||(n.ApplicationInsights={}))}(Microsoft||(Microsoft={})),function(n){var t;(function(n){"use strict";var t=function(){function t(){this.hashCodeGeneragor=new n.HashCodeScoreGenerator}return t.prototype.isEnabled=function(n,t){return this.hashCodeGeneragor.getHashCodeScore(n)=0&&(i=i.replace(/[^0-9a-zA-Z-._()\/ ]/g,"_"),n._InternalLogging.throwInternalUserActionable(n.LoggingSeverity.WARNING,new n._InternalLogMessage(n._InternalMessageId.USRACT_IllegalCharsInName,"name contains illegal characters. Illegal characters have been replaced with '_'.",{newName:i}))),i.length>t.MAX_NAME_LENGTH&&(i=i.substring(0,t.MAX_NAME_LENGTH),n._InternalLogging.throwInternalUserActionable(n.LoggingSeverity.WARNING,new n._InternalLogMessage(n._InternalMessageId.USRACT_NameTooLong,"name is too long. It has been truncated to "+t.MAX_NAME_LENGTH+" characters.",{name:i})))),i},t.sanitizeString=function(i){return i&&(i=n.Util.trim(i),i.toString().length>t.MAX_STRING_LENGTH&&(i=i.toString().substring(0,t.MAX_STRING_LENGTH),n._InternalLogging.throwInternalUserActionable(n.LoggingSeverity.WARNING,new n._InternalLogMessage(n._InternalMessageId.USRACT_StringValueTooLong,"string value is too long. It has been truncated to "+t.MAX_STRING_LENGTH+" characters.",{value:i})))),i},t.sanitizeUrl=function(i){return i&&i.length>t.MAX_URL_LENGTH&&(i=i.substring(0,t.MAX_URL_LENGTH),n._InternalLogging.throwInternalUserActionable(n.LoggingSeverity.WARNING,new n._InternalLogMessage(n._InternalMessageId.USRACT_UrlTooLong,"url is too long, it has been trucated to "+t.MAX_URL_LENGTH+" characters.",{url:i}))),i},t.sanitizeMessage=function(i){return i&&i.length>t.MAX_MESSAGE_LENGTH&&(i=i.substring(0,t.MAX_MESSAGE_LENGTH),n._InternalLogging.throwInternalUserActionable(n.LoggingSeverity.WARNING,new n._InternalLogMessage(n._InternalMessageId.USRACT_MessageTruncated,"message is too long, it has been trucated to "+t.MAX_MESSAGE_LENGTH+" characters.",{message:i}))),i},t.sanitizeException=function(i){return i&&i.length>t.MAX_EXCEPTION_LENGTH&&(i=i.substring(0,t.MAX_EXCEPTION_LENGTH),n._InternalLogging.throwInternalUserActionable(n.LoggingSeverity.WARNING,new n._InternalLogMessage(n._InternalMessageId.USRACT_ExceptionTruncated,"exception is too long, it has been trucated to "+t.MAX_EXCEPTION_LENGTH+" characters.",{exception:i}))),i},t.sanitizeProperties=function(n){var r,i,u;if(n){r={};for(i in n)u=t.sanitizeString(n[i]),i=t.sanitizeKeyAndAddUniqueness(i,r),r[i]=u;n=r}return n},t.sanitizeMeasurements=function(n){var r,i,u;if(n){r={};for(i in n)u=n[i],i=t.sanitizeKeyAndAddUniqueness(i,r),r[i]=u;n=r}return n},t.padNumber=function(n){var t="00"+n;return t.substr(t.length-3)},t.MAX_NAME_LENGTH=150,t.MAX_STRING_LENGTH=1024,t.MAX_URL_LENGTH=2048,t.MAX_MESSAGE_LENGTH=32768,t.MAX_EXCEPTION_LENGTH=32768,t}();t.DataSanitizer=i})(i=t.Common||(t.Common={}))})(t=n.Telemetry||(n.Telemetry={}))})(t=n.ApplicationInsights||(n.ApplicationInsights={}))}(Microsoft||(Microsoft={})),function(n){var t;(function(n){var t;(function(t){"use strict";var i=function(i){function r(r,u){i.call(this);this.aiDataContract={ver:n.FieldType.Required,message:n.FieldType.Required,severityLevel:n.FieldType.Default,measurements:n.FieldType.Default,properties:n.FieldType.Default};r=r||n.Util.NotSpecified;this.message=t.Common.DataSanitizer.sanitizeMessage(r);this.properties=t.Common.DataSanitizer.sanitizeProperties(u)}return __extends(r,i),r.envelopeType="Microsoft.ApplicationInsights.{0}.Message",r.dataType="MessageData",r}(AI.MessageData);t.Trace=i})(t=n.Telemetry||(n.Telemetry={}))})(t=n.ApplicationInsights||(n.ApplicationInsights={}))}(Microsoft||(Microsoft={})),function(n){"use strict";var t=function(n){function t(){this.ver=2;this.properties={};this.measurements={};n.call(this)}return __extends(t,n),t}(Microsoft.Telemetry.Domain);n.EventData=t}(AI||(AI={})),function(n){var t;(function(n){var t;(function(t){"use strict";var i=function(t){function i(i,r,u){t.call(this);this.aiDataContract={ver:n.FieldType.Required,name:n.FieldType.Required,properties:n.FieldType.Default,measurements:n.FieldType.Default};this.name=n.Telemetry.Common.DataSanitizer.sanitizeString(i);this.properties=n.Telemetry.Common.DataSanitizer.sanitizeProperties(r);this.measurements=n.Telemetry.Common.DataSanitizer.sanitizeMeasurements(u)}return __extends(i,t),i.envelopeType="Microsoft.ApplicationInsights.{0}.Event",i.dataType="EventData",i}(AI.EventData);t.Event=i})(t=n.Telemetry||(n.Telemetry={}))})(t=n.ApplicationInsights||(n.ApplicationInsights={}))}(Microsoft||(Microsoft={})),function(n){"use strict";var t=function(){function n(){this.hasFullStack=!0;this.parsedStack=[]}return n}();n.ExceptionDetails=t}(AI||(AI={})),function(n){"use strict";var t=function(n){function t(){this.ver=2;this.exceptions=[];this.properties={};this.measurements={};n.call(this)}return __extends(t,n),t}(Microsoft.Telemetry.Domain);n.ExceptionData=t}(AI||(AI={})),function(n){"use strict";var t=function(){function n(){}return n}();n.StackFrame=t}(AI||(AI={})),function(n){var t;(function(n){var t;(function(t){"use strict";var u=function(t){function i(i,u,f,e){t.call(this);this.aiDataContract={ver:n.FieldType.Required,handledAt:n.FieldType.Required,exceptions:n.FieldType.Required,severityLevel:n.FieldType.Default,properties:n.FieldType.Default,measurements:n.FieldType.Default};this.properties=n.Telemetry.Common.DataSanitizer.sanitizeProperties(f);this.measurements=n.Telemetry.Common.DataSanitizer.sanitizeMeasurements(e);this.handledAt=u||"unhandled";this.exceptions=[new r(i)]}return __extends(i,t),i.CreateSimpleException=function(n,t,i,r,u,f,e){return{handledAt:e||"unhandled",exceptions:[{hasFullStack:!0,message:n,stack:u,typeName:t,parsedStack:[{level:0,assembly:i,fileName:r,line:f,method:"unknown"}]}]}},i.envelopeType="Microsoft.ApplicationInsights.{0}.Exception",i.dataType="ExceptionData",i}(AI.ExceptionData),r,i;t.Exception=u;r=function(r){function u(i){r.call(this);this.aiDataContract={id:n.FieldType.Default,outerId:n.FieldType.Default,typeName:n.FieldType.Required,message:n.FieldType.Required,hasFullStack:n.FieldType.Default,stack:n.FieldType.Default,parsedStack:n.FieldType.Array};this.typeName=t.Common.DataSanitizer.sanitizeString(i.name||n.Util.NotSpecified);this.message=t.Common.DataSanitizer.sanitizeMessage(i.message||n.Util.NotSpecified);var u=i.stack;this.parsedStack=this.parseStack(u);this.stack=t.Common.DataSanitizer.sanitizeException(u);this.hasFullStack=n.Util.isArray(this.parsedStack)&&this.parsedStack.length>0}return __extends(u,r),u.prototype.parseStack=function(n){var t=undefined,e,l,o,r,a,s,h,p,w,b;if(typeof n=="string"){for(e=n.split("\n"),t=[],l=0,o=0,r=0;r<=e.length;r++)a=e[r],i.regex.test(a)&&(s=new i(e[r],l++),o+=s.sizeInBytes,t.push(s));if(h=32768,o>h)for(var u=0,f=t.length-1,v=0,c=u,y=f;uh){b=y-c+1;t.splice(c,b);break}c=u;y=f;u++;f--}}return t},u}(AI.ExceptionDetails);i=function(t){function i(r,u){t.call(this);this.sizeInBytes=0;this.aiDataContract={level:n.FieldType.Required,method:n.FieldType.Required,assembly:n.FieldType.Default,fileName:n.FieldType.Default,line:n.FieldType.Default};this.level=u;this.method="";this.assembly=n.Util.trim(r);var f=r.match(i.regex);f&&f.length>=5&&(this.method=n.Util.trim(f[2])||this.method,this.fileName=n.Util.trim(f[4]),this.line=parseInt(f[5])||0);this.sizeInBytes+=this.method.length;this.sizeInBytes+=this.fileName.length;this.sizeInBytes+=this.assembly.length;this.sizeInBytes+=i.baseSize;this.sizeInBytes+=this.level.toString().length;this.sizeInBytes+=this.line.toString().length}return __extends(i,t),i.regex=/^([\s]+at)?(.*?)(\@|\s\(|\s)([^\(\@\n]+):([0-9]+):([0-9]+)(\)?)$/,i.baseSize=58,i}(AI.StackFrame);t._StackFrame=i})(t=n.Telemetry||(n.Telemetry={}))})(t=n.ApplicationInsights||(n.ApplicationInsights={}))}(Microsoft||(Microsoft={})),function(n){"use strict";var t=function(n){function t(){this.ver=2;this.metrics=[];this.properties={};n.call(this)}return __extends(t,n),t}(Microsoft.Telemetry.Domain);n.MetricData=t}(AI||(AI={})),function(n){"use strict";(function(n){n[n.Measurement=0]="Measurement";n[n.Aggregation=1]="Aggregation"})(n.DataPointType||(n.DataPointType={}));var t=n.DataPointType}(AI||(AI={})),function(n){"use strict";var t=function(){function t(){this.kind=n.DataPointType.Measurement}return t}();n.DataPoint=t}(AI||(AI={})),function(n){var t;(function(n){var t;(function(t){var i;(function(t){"use strict";var i=function(t){function i(){t.apply(this,arguments);this.aiDataContract={name:n.FieldType.Required,kind:n.FieldType.Default,value:n.FieldType.Required,count:n.FieldType.Default,min:n.FieldType.Default,max:n.FieldType.Default,stdDev:n.FieldType.Default}}return __extends(i,t),i}(AI.DataPoint);t.DataPoint=i})(i=t.Common||(t.Common={}))})(t=n.Telemetry||(n.Telemetry={}))})(t=n.ApplicationInsights||(n.ApplicationInsights={}))}(Microsoft||(Microsoft={})),function(n){var t;(function(t){var i;(function(i){"use strict";var r=function(r){function u(u,f,e,o,s,h){r.call(this);this.aiDataContract={ver:t.FieldType.Required,metrics:t.FieldType.Required,properties:t.FieldType.Default};var c=new n.ApplicationInsights.Telemetry.Common.DataPoint;c.count=e>0?e:undefined;c.max=isNaN(s)||s===null?undefined:s;c.min=isNaN(o)||o===null?undefined:o;c.name=i.Common.DataSanitizer.sanitizeString(u);c.value=f;this.metrics=[c];this.properties=t.Telemetry.Common.DataSanitizer.sanitizeProperties(h)}return __extends(u,r),u.envelopeType="Microsoft.ApplicationInsights.{0}.Metric",u.dataType="MetricData",u}(AI.MetricData);i.Metric=r})(i=t.Telemetry||(t.Telemetry={}))})(t=n.ApplicationInsights||(n.ApplicationInsights={}))}(Microsoft||(Microsoft={})),function(n){"use strict";var t=function(n){function t(){this.ver=2;this.properties={};this.measurements={};n.call(this)}return __extends(t,n),t}(n.EventData);n.PageViewData=t}(AI||(AI={})),function(n){var t;(function(n){var t;(function(t){"use strict";var i=function(i){function r(r,u,f,e,o){i.call(this);this.aiDataContract={ver:n.FieldType.Required,name:n.FieldType.Default,url:n.FieldType.Default,duration:n.FieldType.Default,properties:n.FieldType.Default,measurements:n.FieldType.Default};this.url=t.Common.DataSanitizer.sanitizeUrl(u);this.name=t.Common.DataSanitizer.sanitizeString(r||n.Util.NotSpecified);isNaN(f)||(this.duration=n.Util.msToTimeSpan(f));this.properties=n.Telemetry.Common.DataSanitizer.sanitizeProperties(e);this.measurements=n.Telemetry.Common.DataSanitizer.sanitizeMeasurements(o)}return __extends(r,i),r.envelopeType="Microsoft.ApplicationInsights.{0}.Pageview",r.dataType="PageviewData",r}(AI.PageViewData);t.PageView=i})(t=n.Telemetry||(n.Telemetry={}))})(t=n.ApplicationInsights||(n.ApplicationInsights={}))}(Microsoft||(Microsoft={})),function(n){"use strict";var t=function(n){function t(){this.ver=2;this.properties={};this.measurements={};n.call(this)}return __extends(t,n),t}(n.PageViewData);n.PageViewPerfData=t}(AI||(AI={})),function(n){var t;(function(n){var t;(function(t){"use strict";var i=function(i){function r(u,f,e,o,s){var h;if(i.call(this),this.aiDataContract={ver:n.FieldType.Required,name:n.FieldType.Default,url:n.FieldType.Default,duration:n.FieldType.Default,perfTotal:n.FieldType.Default,networkConnect:n.FieldType.Default,sentRequest:n.FieldType.Default,receivedResponse:n.FieldType.Default,domProcessing:n.FieldType.Default,properties:n.FieldType.Default,measurements:n.FieldType.Default},this.isValid=!1,h=r.getPerformanceTiming(),h){var c=r.getDuration(h.navigationStart,h.loadEventEnd),l=r.getDuration(h.navigationStart,h.connectEnd),a=r.getDuration(h.requestStart,h.responseStart),v=r.getDuration(h.responseStart,h.responseEnd),y=r.getDuration(h.responseEnd,h.loadEventEnd);c==0?n._InternalLogging.throwInternalNonUserActionable(n.LoggingSeverity.WARNING,new n._InternalLogMessage(n._InternalMessageId.NONUSRACT_ErrorPVCalc,"error calculating page view performance.",{total:c,network:l,request:a,response:v,dom:y})):c0&&n.navigationStart>0&&n.responseStart>0&&n.requestStart>0&&n.loadEventEnd>0&&n.responseEnd>0&&n.connectEnd>0&&n.domLoading>0},r.getDuration=function(n,t){var i=0;return isNaN(n)||isNaN(t)||(i=Math.max(t-n,0)),i},r.envelopeType="Microsoft.ApplicationInsights.{0}.PageviewPerformance",r.dataType="PageviewPerformanceData",r}(AI.PageViewPerfData);t.PageViewPerformance=i})(t=n.Telemetry||(n.Telemetry={}))})(t=n.ApplicationInsights||(n.ApplicationInsights={}))}(Microsoft||(Microsoft={})),function(n){var t;(function(n){"use strict";var t=function(){function t(t){this._config=t;this._sender=new n.Sender(t);typeof window!="undefined"&&(this._sessionManager=new n.Context._SessionManager(t),this.application=new n.Context.Application,this.device=new n.Context.Device,this.internal=new n.Context.Internal,this.location=new n.Context.Location,this.user=new n.Context.User(t),this.operation=new n.Context.Operation,this.session=new n.Context.Session,this.sample=new n.Context.Sample(t.sampleRate()))}return t.prototype.addTelemetryInitializer=function(n){this.telemetryInitializers=this.telemetryInitializers||[];this.telemetryInitializers.push(n)},t.prototype.track=function(t){return t?(t.name===n.Telemetry.PageView.envelopeType&&n._InternalLogging.resetInternalMessageCount(),this.session&&typeof this.session.id!="string"&&this._sessionManager.update(),this._track(t)):n._InternalLogging.throwInternalUserActionable(n.LoggingSeverity.CRITICAL,new n._InternalLogMessage(n._InternalMessageId.USRACT_TrackArgumentsNotSpecified,"cannot call .track() with a null or undefined argument")),t},t.prototype._track=function(t){var i,f,r,u,o;this.session&&(typeof this.session.id=="string"?this._applySessionContext(t,this.session):this._applySessionContext(t,this._sessionManager.automaticSession));this._applyApplicationContext(t,this.application);this._applyDeviceContext(t,this.device);this._applyInternalContext(t,this.internal);this._applyLocationContext(t,this.location);this._applySampleContext(t,this.sample);this._applyUserContext(t,this.user);this._applyOperationContext(t,this.operation);t.iKey=this._config.instrumentationKey();i=!1;try{for(this.telemetryInitializers=this.telemetryInitializers||[],f=this.telemetryInitializers.length,r=0;rl&&(clearInterval(a),s||(o.appInsights.sendPageViewInternal(i,r,l,u,f),o.appInsights.flush()))}catch(v){n._InternalLogging.throwInternalNonUserActionable(n.LoggingSeverity.CRITICAL,new n._InternalLogMessage(n._InternalMessageId.NONUSRACT_TrackPVFailedCalc,"trackPageView failed on page load calculation: "+n.Util.getExceptionName(v),{exception:n.Util.dump(v)}))}},100)},i}();t.PageViewManager=i})(t=n.Telemetry||(n.Telemetry={}))})(t=n.ApplicationInsights||(n.ApplicationInsights={}))}(Microsoft||(Microsoft={})),function(n){var t;(function(n){var t;(function(t){"use strict";var r=function(){function t(n){this.prevPageVisitDataKeyName="prevPageVisitData";this.pageVisitTimeTrackingHandler=n}return t.prototype.trackPreviousPageVisit=function(t,i){try{var r=this.restartPageVisitTimer(t,i);r&&this.pageVisitTimeTrackingHandler(r.pageName,r.pageUrl,r.pageVisitTime)}catch(u){n._InternalLogging.warnToConsole("Auto track page visit time failed, metric will not be collected: "+n.Util.dump(u))}},t.prototype.restartPageVisitTimer=function(t,i){try{var r=this.stopPageVisitTimer();return this.startPageVisitTimer(t,i),r}catch(u){return n._InternalLogging.warnToConsole("Call to restart failed: "+n.Util.dump(u)),null}},t.prototype.startPageVisitTimer=function(t,r){try{if(n.Util.canUseSessionStorage()){if(n.Util.getSessionStorage(this.prevPageVisitDataKeyName)!=null)throw new Error("Cannot call startPageVisit consecutively without first calling stopPageVisit");var u=new i(t,r),f=JSON.stringify(u);n.Util.setSessionStorage(this.prevPageVisitDataKeyName,f)}}catch(e){n._InternalLogging.warnToConsole("Call to start failed: "+n.Util.dump(e))}},t.prototype.stopPageVisitTimer=function(){var r,i,t;try{return n.Util.canUseSessionStorage()?(r=Date.now(),i=n.Util.getSessionStorage(this.prevPageVisitDataKeyName),i?(t=JSON.parse(i),t.pageVisitTime=r-t.pageVisitStartTime,n.Util.removeSessionStorage(this.prevPageVisitDataKeyName),t):null):null}catch(u){return n._InternalLogging.warnToConsole("Stop page visit timer failed: "+n.Util.dump(u)),null}},t}(),i;t.PageVisitTimeManager=r;i=function(){function n(n,t){this.pageVisitStartTime=Date.now();this.pageName=n;this.pageUrl=t}return n}();t.PageVisitData=i})(t=n.Telemetry||(n.Telemetry={}))})(t=n.ApplicationInsights||(n.ApplicationInsights={}))}(Microsoft||(Microsoft={})),function(n){"use strict";(function(n){n[n.SQL=0]="SQL";n[n.Http=1]="Http";n[n.Other=2]="Other"})(n.DependencyKind||(n.DependencyKind={}));var t=n.DependencyKind}(AI||(AI={})),function(n){"use strict";(function(n){n[n.Undefined=0]="Undefined";n[n.Aic=1]="Aic";n[n.Apmc=2]="Apmc"})(n.DependencySourceType||(n.DependencySourceType={}));var t=n.DependencySourceType}(AI||(AI={})),function(n){"use strict";var t=function(t){function i(){this.ver=2;this.kind=n.DataPointType.Aggregation;this.dependencyKind=n.DependencyKind.Other;this.success=!0;this.dependencySource=n.DependencySourceType.Apmc;this.properties={};t.call(this)}return __extends(i,t),i}(Microsoft.Telemetry.Domain);n.RemoteDependencyData=t}(AI||(AI={})),function(n){var t;(function(n){var t;(function(t){"use strict";var i=function(t){function i(i,r,u,f,e,o){t.call(this);this.aiDataContract={id:n.FieldType.Required,ver:n.FieldType.Required,name:n.FieldType.Default,kind:n.FieldType.Required,value:n.FieldType.Default,count:n.FieldType.Default,min:n.FieldType.Default,max:n.FieldType.Default,stdDev:n.FieldType.Default,dependencyKind:n.FieldType.Default,success:n.FieldType.Default,async:n.FieldType.Default,dependencySource:n.FieldType.Default,commandName:n.FieldType.Default,dependencyTypeName:n.FieldType.Default,properties:n.FieldType.Default,resultCode:n.FieldType.Default};this.id=i;this.name=r;this.commandName=u;this.value=f;this.success=e;this.resultCode=o+"";this.dependencyKind=AI.DependencyKind.Http;this.dependencyTypeName="Ajax"}return __extends(i,t),i.envelopeType="Microsoft.ApplicationInsights.{0}.RemoteDependency",i.dataType="RemoteDependencyData",i}(AI.RemoteDependencyData);t.RemoteDependencyData=i})(t=n.Telemetry||(n.Telemetry={}))})(t=n.ApplicationInsights||(n.ApplicationInsights={}))}(Microsoft||(Microsoft={})),function(n){var t;(function(t){"use strict";var r,i;t.Version="0.22.9";r=function(){function r(u){var f=this,e,o,s;if(this._trackAjaxAttempts=0,this.config=u||{},e=r.defaultConfig,e!==undefined)for(o in e)this.config[o]===undefined&&(this.config[o]=e[o]);t._InternalLogging.verboseLogging=function(){return f.config.verboseLogging};t._InternalLogging.enableDebugExceptions=function(){return f.config.enableDebug};s={instrumentationKey:function(){return f.config.instrumentationKey},accountId:function(){return f.config.accountId},sessionRenewalMs:function(){return f.config.sessionRenewalMs},sessionExpirationMs:function(){return f.config.sessionExpirationMs},endpointUrl:function(){return f.config.endpointUrl},emitLineDelimitedJson:function(){return f.config.emitLineDelimitedJson},maxBatchSizeInBytes:function(){return f.config.maxBatchSizeInBytes},maxBatchInterval:function(){return f.config.maxBatchInterval},disableTelemetry:function(){return f.config.disableTelemetry},sampleRate:function(){return f.config.samplingPercentage},cookieDomain:function(){return f.config.cookieDomain}};this.context=new t.TelemetryContext(s);this._pageViewManager=new n.ApplicationInsights.Telemetry.PageViewManager(this,this.config.overridePageViewDuration);this._eventTracking=new i("trackEvent");this._eventTracking.action=function(n,i,r,u,e){e?isNaN(e.duration)&&(e.duration=r):e={duration:r};var o=new t.Telemetry.Event(n,u,e),s=new t.Telemetry.Common.Data(t.Telemetry.Event.dataType,o),h=new t.Telemetry.Common.Envelope(s,t.Telemetry.Event.envelopeType);f.context.track(h)};this._pageTracking=new i("trackPageView");this._pageTracking.action=function(n,t,i,r,u){f.sendPageViewInternal(n,t,i,r,u)};this._pageVisitTimeManager=new t.Telemetry.PageVisitTimeManager(function(n,t,i){return f.trackPageVisitTime(n,t,i)});this.config.disableAjaxTracking||new n.ApplicationInsights.AjaxMonitor(this)}return r.prototype.sendPageViewInternal=function(n,i,r,u,f){var e=new t.Telemetry.PageView(n,i,r,u,f),o=new t.Telemetry.Common.Data(t.Telemetry.PageView.dataType,e),s=new t.Telemetry.Common.Envelope(o,t.Telemetry.PageView.envelopeType);this.context.track(s);this._trackAjaxAttempts=0},r.prototype.sendPageViewPerformanceInternal=function(n){var i=new t.Telemetry.Common.Data(t.Telemetry.PageViewPerformance.dataType,n),r=new t.Telemetry.Common.Envelope(i,t.Telemetry.PageViewPerformance.envelopeType);this.context.track(r)},r.prototype.startTrackPage=function(n){try{typeof n!="string"&&(n=window.document&&window.document.title||"");this._pageTracking.start(n)}catch(i){t._InternalLogging.throwInternalNonUserActionable(t.LoggingSeverity.CRITICAL,new t._InternalLogMessage(t._InternalMessageId.NONUSRACT_StartTrackFailed,"startTrackPage failed, page view may not be collected: "+t.Util.getExceptionName(i),{exception:t.Util.dump(i)}))}},r.prototype.stopTrackPage=function(n,i,r,u){try{typeof n!="string"&&(n=window.document&&window.document.title||"");typeof i!="string"&&(i=window.location&&window.location.href||"");this._pageTracking.stop(n,i,r,u);this.config.autoTrackPageVisitTime&&this._pageVisitTimeManager.trackPreviousPageVisit(n,i)}catch(f){t._InternalLogging.throwInternalNonUserActionable(t.LoggingSeverity.CRITICAL,new t._InternalLogMessage(t._InternalMessageId.NONUSRACT_StopTrackFailed,"stopTrackPage failed, page view will not be collected: "+t.Util.getExceptionName(f),{exception:t.Util.dump(f)}))}},r.prototype.trackPageView=function(n,i,r,u,f){try{this._pageViewManager.trackPageView(n,i,r,u,f);this.config.autoTrackPageVisitTime&&this._pageVisitTimeManager.trackPreviousPageVisit(n,i)}catch(e){t._InternalLogging.throwInternalNonUserActionable(t.LoggingSeverity.CRITICAL,new t._InternalLogMessage(t._InternalMessageId.NONUSRACT_TrackPVFailed,"trackPageView failed, page view will not be collected: "+t.Util.getExceptionName(e),{exception:t.Util.dump(e)}))}},r.prototype.startTrackEvent=function(n){try{this._eventTracking.start(n)}catch(i){t._InternalLogging.throwInternalNonUserActionable(t.LoggingSeverity.CRITICAL,new t._InternalLogMessage(t._InternalMessageId.NONUSRACT_StartTrackEventFailed,"startTrackEvent failed, event will not be collected: "+t.Util.getExceptionName(i),{exception:t.Util.dump(i)}))}},r.prototype.stopTrackEvent=function(n,i,r){try{this._eventTracking.stop(n,undefined,i,r)}catch(u){t._InternalLogging.throwInternalNonUserActionable(t.LoggingSeverity.CRITICAL,new t._InternalLogMessage(t._InternalMessageId.NONUSRACT_StopTrackEventFailed,"stopTrackEvent failed, event will not be collected: "+t.Util.getExceptionName(u),{exception:t.Util.dump(u)}))}},r.prototype.trackEvent=function(n,i,r){try{var f=new t.Telemetry.Event(n,i,r),e=new t.Telemetry.Common.Data(t.Telemetry.Event.dataType,f),o=new t.Telemetry.Common.Envelope(e,t.Telemetry.Event.envelopeType);this.context.track(o)}catch(u){t._InternalLogging.throwInternalNonUserActionable(t.LoggingSeverity.CRITICAL,new t._InternalLogMessage(t._InternalMessageId.NONUSRACT_TrackEventFailed,"trackEvent failed, event will not be collected: "+t.Util.getExceptionName(u),{exception:t.Util.dump(u)}))}},r.prototype.trackAjax=function(n,i,r,u,f,e){if(this.config.maxAjaxCallsPerView===-1||this._trackAjaxAttempts0?n.maxBatchSizeInBytes:1e6,n.maxBatchInterval=isNaN(n.maxBatchInterval)?15e3:n.maxBatchInterval,n.enableDebug=t.Util.stringToBoolOrDefault(n.enableDebug),n.disableExceptionTracking=n.disableExceptionTracking!==undefined&&n.disableExceptionTracking!==null?t.Util.stringToBoolOrDefault(n.disableExceptionTracking):!1,n.disableTelemetry=t.Util.stringToBoolOrDefault(n.disableTelemetry),n.verboseLogging=t.Util.stringToBoolOrDefault(n.verboseLogging),n.emitLineDelimitedJson=t.Util.stringToBoolOrDefault(n.emitLineDelimitedJson),n.diagnosticLogInterval=n.diagnosticLogInterval||1e4,n.autoTrackPageVisitTime=t.Util.stringToBoolOrDefault(n.autoTrackPageVisitTime),(isNaN(n.samplingPercentage)||n.samplingPercentage<=0||n.samplingPercentage>=100)&&(n.samplingPercentage=100),n.disableAjaxTracking=n.disableAjaxTracking!==undefined&&n.disableAjaxTracking!==null?t.Util.stringToBoolOrDefault(n.disableAjaxTracking):!1,n.maxAjaxCallsPerView=isNaN(n.maxAjaxCallsPerView)?500:n.maxAjaxCallsPerView,n.disableCorrelationHeaders=n.disableCorrelationHeaders!==undefined&&n.disableCorrelationHeaders!==null?t.Util.stringToBoolOrDefault(n.disableCorrelationHeaders):!0,n.disableFlushOnBeforeUnload=n.disableFlushOnBeforeUnload!==undefined&&n.disableFlushOnBeforeUnload!==null?t.Util.stringToBoolOrDefault(n.disableFlushOnBeforeUnload):!1,n},i}();t.Initialization=i})(t=n.ApplicationInsights||(n.ApplicationInsights={}))}(Microsoft||(Microsoft={})),function(n){var t;(function(){"use strict";var r,u;try{if(typeof window!="undefined"&&typeof JSON!="undefined")if(r="appInsights",window[r]===undefined)n.ApplicationInsights.AppInsights.defaultConfig=n.ApplicationInsights.Initialization.getDefaultConfig();else{var f=window[r]||{},t=new n.ApplicationInsights.Initialization(f),i=t.loadAppInsights();for(u in i)f[u]=i[u];t.emptyQueue();t.pollInteralLogs(i);t.addHousekeepingBeforeUnload(i)}}catch(e){n.ApplicationInsights._InternalLogging.warnToConsole("Failed to initialize AppInsights JS SDK: "+e.message)}})(t=n.ApplicationInsights||(n.ApplicationInsights={}))}(Microsoft||(Microsoft={})) \ No newline at end of file From 20df2bd947d83fea4d5691385433398a68296a0d Mon Sep 17 00:00:00 2001 From: Damon Tivel Date: Tue, 21 Aug 2018 16:07:23 -0700 Subject: [PATCH 07/15] MonitoringProcessor: improve throughput (#342) Progress on https://github.com/NuGet/NuGetGallery/issues/6327. --- src/Ng/Jobs/MonitoringProcessorJob.cs | 32 +++--- src/Ng/Scripts/Catalog2Monitoring.cmd | 30 ++++-- src/Ng/Scripts/MonitoringProcessor.cmd | 35 ++++-- .../IPackageTimestampMetadataResource.cs | 4 +- .../PackageTimestampMetadataResourceV2.cs | 21 ++-- .../Validation/Result/TestResult.cs | 6 +- .../Validation/Test/AggregateValidator.cs | 6 +- .../Validation/Test/IAggregateValidator.cs | 4 +- .../Validation/Test/IValidator.cs | 4 +- .../Validation/Test/PackageValidator.cs | 9 +- .../RegistrationExistsValidator.cs | 16 ++- .../Registration/RegistrationIdValidator.cs | 18 ++-- .../RegistrationIndexValidator.cs | 52 ++++++--- .../Registration/RegistrationLeafValidator.cs | 71 +++++++----- .../RegistrationListedValidator.cs | 18 ++-- ...rationRequireLicenseAcceptanceValidator.cs | 11 +- .../Registration/RegistrationValidator.cs | 24 +++-- .../RegistrationVersionValidator.cs | 17 +-- .../Validation/Test/ValidationContext.cs | 40 ++++++- .../Validation/Test/Validator.cs | 22 ++-- tests/NgTests/NgTests.csproj | 1 + .../Validators/DummyAggregateValidator.cs | 4 +- tests/NgTests/Validators/DummyValidator.cs | 4 +- .../RegistrationIndexValidatorTests.cs | 28 ++--- .../RegistrationLeafValidatorTests.cs | 36 +++---- .../Validators/ValidationContextTests.cs | 101 ++++++++++++++++++ tests/NgTests/Validators/ValidatorTests.cs | 18 ++-- 27 files changed, 432 insertions(+), 200 deletions(-) create mode 100644 tests/NgTests/Validators/ValidationContextTests.cs diff --git a/src/Ng/Jobs/MonitoringProcessorJob.cs b/src/Ng/Jobs/MonitoringProcessorJob.cs index d04ce0402..71733f77f 100644 --- a/src/Ng/Jobs/MonitoringProcessorJob.cs +++ b/src/Ng/Jobs/MonitoringProcessorJob.cs @@ -6,7 +6,6 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; -using Microsoft.ApplicationInsights; using Microsoft.Extensions.Logging; using NuGet.Packaging.Core; using NuGet.Protocol; @@ -74,23 +73,25 @@ protected override void Init(IDictionary arguments, Cancellation protected override async Task RunInternal(CancellationToken cancellationToken) { - await ParallelAsync.Repeat(() => ProcessPackages(cancellationToken)); + await ParallelAsync.Repeat(() => ProcessPackagesAsync(cancellationToken)); } - private async Task ProcessPackages(CancellationToken token) + private async Task ProcessPackagesAsync(CancellationToken token) { StorageQueueMessage queueMessage = null; do { Logger.LogInformation("Fetching next queue message."); queueMessage = await _queue.GetNextAsync(token); - await HandleQueueMessage(queueMessage, token); + await HandleQueueMessageAsync(queueMessage, token); } while (queueMessage != null); Logger.LogInformation("No messages left in queue."); } - private async Task HandleQueueMessage(StorageQueueMessage queueMessage, CancellationToken token) + private async Task HandleQueueMessageAsync( + StorageQueueMessage queueMessage, + CancellationToken token) { if (queueMessage == null) { @@ -102,7 +103,7 @@ private async Task HandleQueueMessage(StorageQueueMessage> FetchCatalogIndexEntriesFromRegistration(FeedPackageIdentity feedPackage, CancellationToken token) + private async Task> FetchCatalogIndexEntriesFromRegistrationAsync( + FeedPackageIdentity feedPackage, + CancellationToken token) { var id = feedPackage.Id; var version = NuGetVersion.Parse(feedPackage.Version); @@ -189,7 +194,10 @@ private async Task> FetchCatalogIndexEntriesFromR }; } - private async Task SaveFailedPackageMonitoringStatus(PackageValidatorContext queuedContext, Exception exception, CancellationToken token) + private async Task SaveFailedPackageMonitoringStatusAsync( + PackageValidatorContext queuedContext, + Exception exception, + CancellationToken token) { var feedPackage = new FeedPackageIdentity(queuedContext.Package.Id, queuedContext.Package.Version); diff --git a/src/Ng/Scripts/Catalog2Monitoring.cmd b/src/Ng/Scripts/Catalog2Monitoring.cmd index e984385d6..30ed2b2d5 100644 --- a/src/Ng/Scripts/Catalog2Monitoring.cmd +++ b/src/Ng/Scripts/Catalog2Monitoring.cmd @@ -1,14 +1,30 @@ @echo OFF - + cd Ng :Top - echo "Starting job - #{Jobs.catalog2monitoring.Title}" +echo "Starting job - #{Jobs.catalog2monitoring.Title}" + +title #{Jobs.catalog2monitoring.Title} - title #{Jobs.catalog2monitoring.Title} - - start /w .\Ng.exe catalog2monitoring -source #{Jobs.common.v3.Source} -index #{Jobs.common.v3.index} -gallery #{Jobs.common.v3.f2c.Gallery} -endpointsToTest "#{Jobs.endpointmonitoring.EndpointsToTest}" -statusFolder #{Jobs.endpointmonitoring.StatusFolder} -storageType azure -storageAccountName #{Jobs.common.v3.c2r.StorageAccountName} -storageKeyValue #{Jobs.common.v3.c2r.StorageAccountKey} -storageContainer #{Jobs.endpointmonitoring.MonitoringContainer} -storageQueueName #{Jobs.endpointmonitoring.PackageValidatorQueue} -instrumentationkey #{Jobs.common.v3.Logging.InstrumentationKey} -vaultName #{Deployment.Azure.KeyVault.VaultName} -clientId #{Deployment.Azure.KeyVault.ClientId} -certificateThumbprint #{Deployment.Azure.KeyVault.CertificateThumbprint} -verbose true -interval #{Jobs.common.v3.Interval} +start /w .\Ng.exe catalog2monitoring ^ + -source #{Jobs.common.v3.Source} ^ + -index #{Jobs.common.v3.index} ^ + -gallery #{Jobs.common.v3.f2c.Gallery} ^ + -endpointsToTest "#{Jobs.endpointmonitoring.EndpointsToTest}" ^ + -statusFolder #{Jobs.endpointmonitoring.StatusFolder} ^ + -storageType azure ^ + -storageAccountName #{Jobs.common.v3.c2r.StorageAccountName} ^ + -storageKeyValue #{Jobs.common.v3.c2r.StorageAccountKey} ^ + -storageContainer #{Jobs.endpointmonitoring.MonitoringContainer} ^ + -storageQueueName #{Jobs.endpointmonitoring.PackageValidatorQueue} ^ + -instrumentationkey #{Jobs.common.v3.Logging.InstrumentationKey} ^ + -vaultName #{Deployment.Azure.KeyVault.VaultName} ^ + -clientId #{Deployment.Azure.KeyVault.ClientId} ^ + -certificateThumbprint #{Deployment.Azure.KeyVault.CertificateThumbprint} ^ + -verbose true ^ + -interval #{Jobs.common.v3.Interval} - echo "Finished #{Jobs.catalog2monitoring.Title}" +echo "Finished #{Jobs.catalog2monitoring.Title}" - goto Top \ No newline at end of file +goto Top \ No newline at end of file diff --git a/src/Ng/Scripts/MonitoringProcessor.cmd b/src/Ng/Scripts/MonitoringProcessor.cmd index a2ce2c2a1..3afd0b734 100644 --- a/src/Ng/Scripts/MonitoringProcessor.cmd +++ b/src/Ng/Scripts/MonitoringProcessor.cmd @@ -1,14 +1,35 @@ @echo OFF - + cd Ng :Top - echo "Starting job - #{Jobs.monitoringprocessor.Title}" +echo "Starting job - #{Jobs.monitoringprocessor.Title}" + +title #{Jobs.monitoringprocessor.Title} - title #{Jobs.monitoringprocessor.Title} - - start /w .\Ng.exe monitoringprocessor -source #{Jobs.common.v3.Source} -index #{Jobs.common.v3.index} -gallery #{Jobs.common.v3.f2c.Gallery} -endpointsToTest "#{Jobs.endpointmonitoring.EndpointsToTest}" -statusFolder #{Jobs.endpointmonitoring.StatusFolder} -storageType azure -storageAccountName #{Jobs.common.v3.c2r.StorageAccountName} -storageKeyValue #{Jobs.common.v3.c2r.StorageAccountKey} -storageContainer #{Jobs.endpointmonitoring.MonitoringContainer} -storageQueueName #{Jobs.endpointmonitoring.PackageValidatorQueue} -storageTypeAuditing azure -storageAccountNameAuditing #{Jobs.feed2catalogv3.AuditingStorageAccountName} -storageKeyValueAuditing #{Jobs.feed2catalogv3.AuditingStorageAccountKey} -storageContainerAuditing auditing -storagePathAuditing package -instrumentationkey #{Jobs.common.v3.Logging.InstrumentationKey} -vaultName #{Deployment.Azure.KeyVault.VaultName} -clientId #{Deployment.Azure.KeyVault.ClientId} -certificateThumbprint #{Deployment.Azure.KeyVault.CertificateThumbprint} -verbose true -interval #{Jobs.common.v3.Interval} +start /w .\Ng.exe monitoringprocessor ^ + -source #{Jobs.common.v3.Source} ^ + -index #{Jobs.common.v3.index} ^ + -gallery #{Jobs.common.v3.f2c.Gallery} ^ + -endpointsToTest "#{Jobs.endpointmonitoring.EndpointsToTest}" ^ + -statusFolder #{Jobs.endpointmonitoring.StatusFolder} ^ + -storageType azure ^ + -storageAccountName #{Jobs.common.v3.c2r.StorageAccountName} ^ + -storageKeyValue #{Jobs.common.v3.c2r.StorageAccountKey} ^ + -storageContainer #{Jobs.endpointmonitoring.MonitoringContainer} ^ + -storageQueueName #{Jobs.endpointmonitoring.PackageValidatorQueue} ^ + -storageTypeAuditing azure ^ + -storageAccountNameAuditing #{Jobs.feed2catalogv3.AuditingStorageAccountName} ^ + -storageKeyValueAuditing #{Jobs.feed2catalogv3.AuditingStorageAccountKey} ^ + -storageContainerAuditing auditing ^ + -storagePathAuditing package ^ + -instrumentationkey #{Jobs.common.v3.Logging.InstrumentationKey} ^ + -vaultName #{Deployment.Azure.KeyVault.VaultName} ^ + -clientId #{Deployment.Azure.KeyVault.ClientId} ^ + -certificateThumbprint #{Deployment.Azure.KeyVault.CertificateThumbprint} ^ + -verbose true ^ + -interval #{Jobs.common.v3.Interval} - echo "Finished #{Jobs.monitoringprocessor.Title}" +echo "Finished #{Jobs.monitoringprocessor.Title}" - goto Top \ No newline at end of file +goto Top \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Resources/IPackageTimestampMetadataResource.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Resources/IPackageTimestampMetadataResource.cs index 6d8d004f6..8688064a9 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Resources/IPackageTimestampMetadataResource.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Resources/IPackageTimestampMetadataResource.cs @@ -8,6 +8,6 @@ namespace NuGet.Services.Metadata.Catalog.Monitoring { public interface IPackageTimestampMetadataResource : INuGetResource { - Task GetAsync(ValidationContext data); + Task GetAsync(ValidationContext context); } -} +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Resources/PackageTimestampMetadataResourceV2.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Resources/PackageTimestampMetadataResourceV2.cs index 31472e603..994684d73 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Resources/PackageTimestampMetadataResourceV2.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Resources/PackageTimestampMetadataResourceV2.cs @@ -18,7 +18,7 @@ public class PackageTimestampMetadataResourceV2 : IPackageTimestampMetadataResou _source = source; _logger = logger; } - + private readonly string _source; private readonly ILogger _logger; @@ -26,23 +26,28 @@ public class PackageTimestampMetadataResourceV2 : IPackageTimestampMetadataResou /// Parses the feed for the package specified by the and returns a . /// If the package is missing from the feed, returns the package's deletion audit record timestamp. /// - public async Task GetAsync(ValidationContext data) + public async Task GetAsync(ValidationContext context) { - var feedPackageDetails = await FeedHelpers.GetPackage(data.Client, _source, data.Package.Id, - data.Package.Version.ToString()); + var feedPackageDetails = await FeedHelpers.GetPackage( + context.Client, + _source, + context.Package.Id, + context.Package.Version.ToString()); if (feedPackageDetails != null) { - return PackageTimestampMetadata.CreateForPackageExistingOnFeed(feedPackageDetails.CreatedDate, feedPackageDetails.LastEditedDate); + return PackageTimestampMetadata.CreateForPackageExistingOnFeed( + feedPackageDetails.CreatedDate, + feedPackageDetails.LastEditedDate); } DateTime? deleted = null; - if (data.DeletionAuditEntries.Any()) + if (context.DeletionAuditEntries.Any()) { - deleted = data.DeletionAuditEntries.Max(entry => entry.TimestampUtc); + deleted = context.DeletionAuditEntries.Max(entry => entry.TimestampUtc); } return PackageTimestampMetadata.CreateForPackageMissingFromFeed(deleted); } } -} +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Result/TestResult.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Result/TestResult.cs index abc2a3cd3..f11112d06 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Result/TestResult.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Result/TestResult.cs @@ -17,8 +17,8 @@ public enum TestResult /// Fail, /// - /// The test was skipped ( threw or returned false). + /// The test was skipped ( threw or returned false). /// Skip - }; -} + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/AggregateValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/AggregateValidator.cs index 67d91f5e4..9f80096f1 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/AggregateValidator.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/AggregateValidator.cs @@ -44,11 +44,11 @@ public AggregateValidator(ValidatorFactory factory, ILogger /// /// The to validate. /// A which contains the results of the validation. - public async Task ValidateAsync(ValidationContext data) + public async Task ValidateAsync(ValidationContext context) { return new AggregateValidationResult( this, - await Task.WhenAll(_validators.Select(v => v.ValidateAsync(data)))); + await Task.WhenAll(_validators.Select(v => v.ValidateAsync(context)))); } } -} +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/IAggregateValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/IAggregateValidator.cs index 8987010e8..ac5e57565 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/IAggregateValidator.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/IAggregateValidator.cs @@ -13,6 +13,6 @@ public interface IAggregateValidator : IValidatorIdentity /// /// Runs each and returns an containing all results. /// - Task ValidateAsync(ValidationContext data); + Task ValidateAsync(ValidationContext context); } -} +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/IValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/IValidator.cs index 7d1e62179..428e861a9 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/IValidator.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/IValidator.cs @@ -14,7 +14,7 @@ public interface IValidator : IValidatorIdentity /// Validates a package. /// /// A which contains the results of the validation. - Task ValidateAsync(ValidationContext data); + Task ValidateAsync(ValidationContext context); } /// @@ -24,4 +24,4 @@ public interface IValidator : IValidatorIdentity public interface IValidator : IValidator where T : EndpointValidator { } -} +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidator.cs index be7477a1f..17838bab8 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidator.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidator.cs @@ -7,10 +7,9 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using Newtonsoft.Json.Linq; using NuGet.Packaging.Core; -using NuGet.Services.Metadata.Catalog.Persistence; using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Persistence; using NuGet.Versioning; namespace NuGet.Services.Metadata.Catalog.Monitoring @@ -31,12 +30,14 @@ public class PackageValidator StorageFactory auditingStorageFactory, ILogger logger) { - if (aggregateValidators == null || !aggregateValidators.Any()) + var validators = aggregateValidators?.ToList(); + + if (aggregateValidators == null || !validators.Any()) { throw new ArgumentException("Must supply at least one endpoint!", nameof(aggregateValidators)); } - AggregateValidators = aggregateValidators.ToList(); + AggregateValidators = validators; _auditingStorageFactory = auditingStorageFactory ?? throw new ArgumentNullException(nameof(auditingStorageFactory)); _logger = logger; } diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationExistsValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationExistsValidator.cs index 0aa302585..954482bb7 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationExistsValidator.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationExistsValidator.cs @@ -12,18 +12,24 @@ namespace NuGet.Services.Metadata.Catalog.Monitoring public class RegistrationExistsValidator : RegistrationLeafValidator { public RegistrationExistsValidator( - IDictionary feedToSource, + IDictionary feedToSource, ILogger logger) : base(feedToSource, logger) { } - public override Task ShouldRunLeaf(ValidationContext data, PackageRegistrationLeafMetadata v2, PackageRegistrationLeafMetadata v3) + public override Task ShouldRunLeafAsync( + ValidationContext context, + PackageRegistrationLeafMetadata v2, + PackageRegistrationLeafMetadata v3) { return Task.FromResult(true); } - public override Task CompareLeaf(ValidationContext data, PackageRegistrationLeafMetadata v2, PackageRegistrationLeafMetadata v3) + public override Task CompareLeafAsync( + ValidationContext context, + PackageRegistrationLeafMetadata v2, + PackageRegistrationLeafMetadata v3) { var v2Exists = v2 != null; var v3Exists = v3 != null; @@ -36,7 +42,7 @@ public override Task CompareLeaf(ValidationContext data, PackageRegistrationLeaf // See https://github.com/NuGet/NuGetGallery/issues/4475 if (v3Exists && !(v3 is PackageRegistrationIndexMetadata)) { - Logger.LogInformation("{PackageId} {PackageVersion} doesn't exist in V2 but has a leaf node in V3!", data.Package.Id, data.Package.Version); + Logger.LogInformation("{PackageId} {PackageVersion} doesn't exist in V2 but has a leaf node in V3!", context.Package.Id, context.Package.Version); return completedTask; } @@ -50,4 +56,4 @@ public override Task CompareLeaf(ValidationContext data, PackageRegistrationLeaf return completedTask; } } -} +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationIdValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationIdValidator.cs index 5e2561dd7..bf10247fc 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationIdValidator.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationIdValidator.cs @@ -12,23 +12,21 @@ namespace NuGet.Services.Metadata.Catalog.Monitoring public class RegistrationIdValidator : RegistrationIndexValidator { public RegistrationIdValidator( - IDictionary feedToSource, - ILogger logger) + IDictionary feedToSource, + ILogger logger) : base(feedToSource, logger) { } - public override Task ShouldRunIndex(ValidationContext data, PackageRegistrationIndexMetadata v2, PackageRegistrationIndexMetadata v3) - { - return Task.FromResult(v2 != null && v3 != null); - } - - public override Task CompareIndex(ValidationContext data, PackageRegistrationIndexMetadata v2, PackageRegistrationIndexMetadata v3) + public override Task CompareIndexAsync( + ValidationContext context, + PackageRegistrationIndexMetadata v2, + PackageRegistrationIndexMetadata v3) { if (!v2.Id.Equals(v3.Id, System.StringComparison.OrdinalIgnoreCase)) { throw new MetadataFieldInconsistencyException( - v2, v3, + v2, v3, nameof(PackageRegistrationIndexMetadata.Id), m => m.Id); } @@ -36,4 +34,4 @@ public override Task CompareIndex(ValidationContext data, PackageRegistrationInd return Task.FromResult(0); } } -} +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationIndexValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationIndexValidator.cs index ae166a9b8..6f8eec8a0 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationIndexValidator.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationIndexValidator.cs @@ -18,34 +18,54 @@ public abstract class RegistrationIndexValidator : RegistrationValidator { } - protected override async Task ShouldRun(ValidationContext data) + protected override async Task ShouldRunAsync(ValidationContext context) { - return - await base.ShouldRun(data) && + var shouldRunTask = context.GetCachedResultAsync( + Keys.ShouldRunAsync, + new Lazy>(() => base.ShouldRunAsync(context))); + var v2Index = await context.GetCachedResultAsync( + Keys.GetIndexAsyncV2, + new Lazy>(() => GetIndexAsync(V2Resource, context))); + var v3Index = await context.GetCachedResultAsync( + Keys.GetIndexAsyncV3, + new Lazy>(() => GetIndexAsync(V3Resource, context))); + var shouldRunIndexTask = context.GetCachedResultAsync( + Keys.ShouldRunIndexAsync, + new Lazy>(() => ShouldRunIndexAsync(context, v2Index, v3Index))); - await ShouldRunIndex( - data, - await GetIndex(V2Resource, data), - await GetIndex(V3Resource, data)); + return await shouldRunTask && await shouldRunIndexTask; } - protected override async Task RunInternal(ValidationContext data) + protected override async Task RunInternalAsync(ValidationContext context) { + var v2Index = await context.GetCachedResultAsync( + Keys.GetIndexAsyncV2, + new Lazy>(() => GetIndexAsync(V2Resource, context))); + var v3Index = await context.GetCachedResultAsync( + Keys.GetIndexAsyncV3, + new Lazy>(() => GetIndexAsync(V3Resource, context))); + try { - await CompareIndex( - data, - await GetIndex(V2Resource, data), - await GetIndex(V3Resource, data)); + await CompareIndexAsync(context, v2Index, v3Index); } catch (Exception e) { - throw new ValidationException("Registration index metadata does not match the FindPackagesById metadata!", e); + throw new ValidationException("Registration index metadata does not match the FindPackagesById metadata!", e); } } - public abstract Task ShouldRunIndex(ValidationContext data, PackageRegistrationIndexMetadata v2, PackageRegistrationIndexMetadata v3); + public Task ShouldRunIndexAsync( + ValidationContext context, + PackageRegistrationIndexMetadata v2, + PackageRegistrationIndexMetadata v3) + { + return Task.FromResult(v2 != null && v3 != null); + } - public abstract Task CompareIndex(ValidationContext data, PackageRegistrationIndexMetadata v2, PackageRegistrationIndexMetadata v3); + public abstract Task CompareIndexAsync( + ValidationContext context, + PackageRegistrationIndexMetadata v2, + PackageRegistrationIndexMetadata v3); } -} +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationLeafValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationLeafValidator.cs index a84f49b6f..39b206f80 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationLeafValidator.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationLeafValidator.cs @@ -14,49 +14,64 @@ namespace NuGet.Services.Metadata.Catalog.Monitoring public abstract class RegistrationLeafValidator : RegistrationValidator { public RegistrationLeafValidator( - IDictionary feedToSource, + IDictionary feedToSource, ILogger logger) : base(feedToSource, logger) { } - protected override async Task ShouldRun(ValidationContext data) + protected override async Task ShouldRunAsync(ValidationContext context) { - return - await base.ShouldRun(data) && + var shouldRunTask = context.GetCachedResultAsync( + Keys.ShouldRunAsync, + new Lazy>(() => base.ShouldRunAsync(context))); + var v2Index = await context.GetCachedResultAsync( + Keys.GetIndexAsyncV2, + new Lazy>(() => GetIndexAsync(V2Resource, context))); + var v3Index = await context.GetCachedResultAsync( + Keys.GetIndexAsyncV3, + new Lazy>(() => GetIndexAsync(V3Resource, context))); + var v2Leaf = await context.GetCachedResultAsync( + Keys.GetLeafAsyncV2, + new Lazy>(() => GetLeafAsync(V2Resource, context))); + var v3Leaf = await context.GetCachedResultAsync( + Keys.GetLeafAsyncV3, + new Lazy>(() => GetLeafAsync(V3Resource, context))); - await ShouldRunLeaf( - data, - await GetIndex(V2Resource, data), - await GetIndex(V3Resource, data)) && - - await ShouldRunLeaf( - data, - await GetLeaf(V2Resource, data), - await GetLeaf(V3Resource, data)); + return await shouldRunTask + && await ShouldRunLeafAsync(context, v2Index, v3Index) + && await ShouldRunLeafAsync(context, v2Leaf, v3Leaf); } - protected override async Task RunInternal(ValidationContext data) + protected override async Task RunInternalAsync(ValidationContext context) { var exceptions = new List(); + var v2Index = await context.GetCachedResultAsync( + Keys.GetIndexAsyncV2, + new Lazy>(() => GetIndexAsync(V2Resource, context))); + var v3Index = await context.GetCachedResultAsync( + Keys.GetIndexAsyncV3, + new Lazy>(() => GetIndexAsync(V3Resource, context))); + try { - await CompareLeaf( - data, - await GetIndex(V2Resource, data), - await GetIndex(V3Resource, data)); + await CompareLeafAsync(context, v2Index, v3Index); } catch (Exception e) { exceptions.Add(new ValidationException("Registration index metadata does not match the FindPackagesById metadata!", e)); } + var v2Leaf = await context.GetCachedResultAsync( + Keys.GetLeafAsyncV2, + new Lazy>(() => GetLeafAsync(V2Resource, context))); + var v3Leaf = await context.GetCachedResultAsync( + Keys.GetLeafAsyncV3, + new Lazy>(() => GetLeafAsync(V3Resource, context))); + try { - await CompareLeaf( - data, - await GetLeaf(V2Resource, data), - await GetLeaf(V3Resource, data)); + await CompareLeafAsync(context, v2Leaf, v3Leaf); } catch (Exception e) { @@ -69,8 +84,14 @@ protected override async Task RunInternal(ValidationContext data) } } - public abstract Task ShouldRunLeaf(ValidationContext data, PackageRegistrationLeafMetadata v2, PackageRegistrationLeafMetadata v3); + public abstract Task ShouldRunLeafAsync( + ValidationContext context, + PackageRegistrationLeafMetadata v2, + PackageRegistrationLeafMetadata v3); - public abstract Task CompareLeaf(ValidationContext data, PackageRegistrationLeafMetadata v2, PackageRegistrationLeafMetadata v3); + public abstract Task CompareLeafAsync( + ValidationContext context, + PackageRegistrationLeafMetadata v2, + PackageRegistrationLeafMetadata v3); } -} +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationListedValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationListedValidator.cs index 9b339cadc..abc293b20 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationListedValidator.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationListedValidator.cs @@ -12,23 +12,29 @@ namespace NuGet.Services.Metadata.Catalog.Monitoring public class RegistrationListedValidator : RegistrationLeafValidator { public RegistrationListedValidator( - IDictionary feedToSource, - ILogger logger) + IDictionary feedToSource, + ILogger logger) : base(feedToSource, logger) { } - public override Task ShouldRunLeaf(ValidationContext data, PackageRegistrationLeafMetadata v2, PackageRegistrationLeafMetadata v3) + public override Task ShouldRunLeafAsync( + ValidationContext context, + PackageRegistrationLeafMetadata v2, + PackageRegistrationLeafMetadata v3) { return Task.FromResult(v2 != null && v3 != null); } - public override Task CompareLeaf(ValidationContext data, PackageRegistrationLeafMetadata v2, PackageRegistrationLeafMetadata v3) + public override Task CompareLeafAsync( + ValidationContext context, + PackageRegistrationLeafMetadata v2, + PackageRegistrationLeafMetadata v3) { if (v2.Listed != v3.Listed) { throw new MetadataFieldInconsistencyException( - v2, v3, + v2, v3, nameof(PackageRegistrationLeafMetadata.Listed), m => m.Listed); } @@ -36,4 +42,4 @@ public override Task CompareLeaf(ValidationContext data, PackageRegistrationLeaf return Task.FromResult(0); } } -} +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationRequireLicenseAcceptanceValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationRequireLicenseAcceptanceValidator.cs index d79314eee..754f69686 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationRequireLicenseAcceptanceValidator.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationRequireLicenseAcceptanceValidator.cs @@ -12,18 +12,13 @@ namespace NuGet.Services.Metadata.Catalog.Monitoring public class RegistrationRequireLicenseAcceptanceValidator : RegistrationIndexValidator { public RegistrationRequireLicenseAcceptanceValidator( - IDictionary feedToSource, + IDictionary feedToSource, ILogger logger) : base(feedToSource, logger) { } - public override Task ShouldRunIndex(ValidationContext data, PackageRegistrationIndexMetadata v2, PackageRegistrationIndexMetadata v3) - { - return Task.FromResult(v2 != null && v3 != null); - } - - public override Task CompareIndex(ValidationContext data, PackageRegistrationIndexMetadata v2, PackageRegistrationIndexMetadata v3) + public override Task CompareIndexAsync(ValidationContext context, PackageRegistrationIndexMetadata v2, PackageRegistrationIndexMetadata v3) { var isEqual = v2.RequireLicenseAcceptance == v3.RequireLicenseAcceptance; @@ -38,4 +33,4 @@ public override Task CompareIndex(ValidationContext data, PackageRegistrationInd return Task.FromResult(0); } } -} +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationValidator.cs index b94e57711..3b705188d 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationValidator.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationValidator.cs @@ -22,18 +22,28 @@ public abstract class RegistrationValidator : Validator V3Resource = feedToSource[FeedType.HttpV3].GetResource(); } - protected async Task GetIndex( + protected async Task GetIndexAsync( IPackageRegistrationMetadataResource resource, - ValidationContext data) + ValidationContext context) { - return await resource.GetIndexAsync(data.Package, Logger.AsCommon(), data.CancellationToken); + return await resource.GetIndexAsync(context.Package, Logger.AsCommon(), context.CancellationToken); } - protected async Task GetLeaf( + protected async Task GetLeafAsync( IPackageRegistrationMetadataResource resource, - ValidationContext data) + ValidationContext context) { - return await resource.GetLeafAsync(data.Package, Logger.AsCommon(), data.CancellationToken); + return await resource.GetLeafAsync(context.Package, Logger.AsCommon(), context.CancellationToken); + } + + protected static class Keys + { + internal static readonly string GetIndexAsyncV2 = $"{nameof(RegistrationValidator)}_{nameof(GetIndexAsync)}_V2"; + internal static readonly string GetIndexAsyncV3 = $"{nameof(RegistrationValidator)}_{nameof(GetIndexAsync)}_V3"; + internal static readonly string GetLeafAsyncV2 = $"{nameof(RegistrationValidator)}_{nameof(GetLeafAsync)}_V2"; + internal static readonly string GetLeafAsyncV3 = $"{nameof(RegistrationValidator)}_{nameof(GetLeafAsync)}_V3"; + internal static readonly string ShouldRunAsync = $"{nameof(Validator)}_{nameof(ShouldRunAsync)}"; + internal static readonly string ShouldRunIndexAsync = $"{nameof(RegistrationIndexValidator)}_{nameof(ShouldRunIndexAsync)}"; } } -} +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationVersionValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationVersionValidator.cs index ee7b7906b..0f259d658 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationVersionValidator.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationVersionValidator.cs @@ -12,22 +12,15 @@ namespace NuGet.Services.Metadata.Catalog.Monitoring public class RegistrationVersionValidator : RegistrationIndexValidator { public RegistrationVersionValidator( - IDictionary feedToSource, - ILogger logger) + IDictionary feedToSource, + ILogger logger) : base(feedToSource, logger) { } - public override Task ShouldRunIndex(ValidationContext data, PackageRegistrationIndexMetadata v2, PackageRegistrationIndexMetadata v3) + public override Task CompareIndexAsync(ValidationContext context, PackageRegistrationIndexMetadata v2, PackageRegistrationIndexMetadata v3) { - return Task.FromResult(v2 != null && v3 != null); - } - - public override Task CompareIndex(ValidationContext data, PackageRegistrationIndexMetadata v2, PackageRegistrationIndexMetadata v3) - { - var isEqual = - v2.Version == - v3.Version; + var isEqual = v2.Version == v3.Version; if (!isEqual) { @@ -40,4 +33,4 @@ public override Task CompareIndex(ValidationContext data, PackageRegistrationInd return Task.FromResult(0); } } -} +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/ValidationContext.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/ValidationContext.cs index 7a556cd8f..c9342164f 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/ValidationContext.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/ValidationContext.cs @@ -1,8 +1,11 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Threading; +using System.Threading.Tasks; using NuGet.Packaging.Core; using NuGet.Services.Metadata.Catalog.Helpers; @@ -13,6 +16,10 @@ namespace NuGet.Services.Metadata.Catalog.Monitoring /// public class ValidationContext { + private readonly ConcurrentDictionary>> _boolCache; + private readonly ConcurrentDictionary>> _indexCache; + private readonly ConcurrentDictionary>> _leafCache; + /// /// The to run the test on. /// @@ -43,14 +50,18 @@ public class ValidationContext /// public ValidationContext() { + _boolCache = new ConcurrentDictionary>>(); + _indexCache = new ConcurrentDictionary>>(); + _leafCache = new ConcurrentDictionary>>(); } public ValidationContext( - PackageIdentity package, - IEnumerable entries, - IEnumerable deletionAuditEntries, - CollectorHttpClient client, + PackageIdentity package, + IEnumerable entries, + IEnumerable deletionAuditEntries, + CollectorHttpClient client, CancellationToken token) + : this() { Package = package; Entries = entries; @@ -58,5 +69,24 @@ public ValidationContext() Client = client; CancellationToken = token; } + + public Task GetCachedResultAsync(string key, Lazy> lazyTask) + { + return _boolCache.GetOrAdd(key, lazyTask).Value; + } + + public Task GetCachedResultAsync( + string key, + Lazy> lazyTask) + { + return _indexCache.GetOrAdd(key, lazyTask).Value; + } + + public Task GetCachedResultAsync( + string key, + Lazy> lazyTask) + { + return _leafCache.GetOrAdd(key, lazyTask).Value; + } } -} +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Validator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Validator.cs index 2545ee7d0..87009c161 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Validator.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Validator.cs @@ -44,14 +44,14 @@ protected Validator() CommonLogger = logger.AsCommon(); } - public async Task ValidateAsync(ValidationContext data) + public async Task ValidateAsync(ValidationContext context) { try { bool shouldRun = false; try { - shouldRun = await ShouldRun(data); + shouldRun = await ShouldRunAsync(context); } catch (Exception e) { @@ -60,7 +60,7 @@ public async Task ValidateAsync(ValidationContext data) if (shouldRun) { - await RunInternal(data); + await RunInternalAsync(context); } else { @@ -78,20 +78,20 @@ public async Task ValidateAsync(ValidationContext data) /// /// Checks that the current batch of catalog entries contains the entry that was created from the current state of the V2 feed. /// - protected virtual async Task ShouldRun(ValidationContext data) + protected virtual async Task ShouldRunAsync(ValidationContext context) { - var timestampV2 = await _timestampMetadataResourceV2.GetAsync(data); - var timestampCatalog = await PackageTimestampMetadata.FromCatalogEntries(data.Client, data.Entries); - + var timestampV2 = await _timestampMetadataResourceV2.GetAsync(context); + var timestampCatalog = await PackageTimestampMetadata.FromCatalogEntries(context.Client, context.Entries); + if (!timestampV2.Last.HasValue) { - throw new TimestampComparisonException(timestampV2, timestampCatalog, + throw new TimestampComparisonException(timestampV2, timestampCatalog, "Cannot get timestamp data for package from the V2 feed!"); } if (!timestampCatalog.Last.HasValue) { - throw new TimestampComparisonException(timestampV2, timestampCatalog, + throw new TimestampComparisonException(timestampV2, timestampCatalog, "Cannot get timestamp data for package from the catalog!"); } @@ -106,7 +106,7 @@ protected virtual async Task ShouldRun(ValidationContext data) return timestampCatalog.Last == timestampV2.Last; } - protected abstract Task RunInternal(ValidationContext data); + protected abstract Task RunInternalAsync(ValidationContext context); } /// @@ -121,4 +121,4 @@ public abstract class Validator : Validator, IValidator where T : Endpoint { } } -} +} \ No newline at end of file diff --git a/tests/NgTests/NgTests.csproj b/tests/NgTests/NgTests.csproj index af928906f..d920cbcb9 100644 --- a/tests/NgTests/NgTests.csproj +++ b/tests/NgTests/NgTests.csproj @@ -319,6 +319,7 @@ + diff --git a/tests/NgTests/Validators/DummyAggregateValidator.cs b/tests/NgTests/Validators/DummyAggregateValidator.cs index 55fdf2650..2828a1377 100644 --- a/tests/NgTests/Validators/DummyAggregateValidator.cs +++ b/tests/NgTests/Validators/DummyAggregateValidator.cs @@ -23,9 +23,9 @@ public AggregateValidationResult Validate() return new AggregateValidationResult(this, _results); } - public Task ValidateAsync(ValidationContext data) + public Task ValidateAsync(ValidationContext context) { return Task.FromResult(Validate()); } } -} +} \ No newline at end of file diff --git a/tests/NgTests/Validators/DummyValidator.cs b/tests/NgTests/Validators/DummyValidator.cs index 10780ddcc..b8a9f28cc 100644 --- a/tests/NgTests/Validators/DummyValidator.cs +++ b/tests/NgTests/Validators/DummyValidator.cs @@ -20,7 +20,7 @@ public DummyValidator(TestResult result, Exception e) _exception = e; } - public Task ValidateAsync(ValidationContext data) + public Task ValidateAsync(ValidationContext context) { return Task.FromResult(Validate()); } @@ -30,4 +30,4 @@ public ValidationResult Validate() return new ValidationResult(this, _result, _exception); } } -} +} \ No newline at end of file diff --git a/tests/NgTests/Validators/RegistrationIndexValidatorTests.cs b/tests/NgTests/Validators/RegistrationIndexValidatorTests.cs index f5f19492d..393f6b5ad 100644 --- a/tests/NgTests/Validators/RegistrationIndexValidatorTests.cs +++ b/tests/NgTests/Validators/RegistrationIndexValidatorTests.cs @@ -21,9 +21,9 @@ private static IEnumerable ValidatorTestData(Func ValidatorSpecialTestData(Func( - () => validator.CompareIndex(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); + () => validator.CompareIndexAsync(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); } [Theory] @@ -85,7 +85,7 @@ public class TheCompareIndexMethod PackageRegistrationIndexMetadata v3, bool shouldPass) { - var compareTask = Task.Run(async () => await validator.CompareIndex(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); + var compareTask = Task.Run(async () => await validator.CompareIndexAsync(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); if (shouldPass) { @@ -99,7 +99,7 @@ public class TheCompareIndexMethod } } - public class TheShouldRunIndexMethod + public class TheShouldRunIndexAsyncMethod { public static IEnumerable ValidatorRunIndexTestData => ValidatorTestData(t => ValidatorTestUtility.GetPairs(t.CreateIndexes)); @@ -112,7 +112,7 @@ public class TheShouldRunIndexMethod PackageRegistrationIndexMetadata v2, PackageRegistrationIndexMetadata v3) { - Assert.Equal(true, await validator.ShouldRunIndex(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); + Assert.True(await validator.ShouldRunIndexAsync(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); } [Theory] @@ -122,8 +122,8 @@ public class TheShouldRunIndexMethod PackageRegistrationIndexMetadata v2, PackageRegistrationIndexMetadata v3) { - Assert.Equal(false, await validator.ShouldRunIndex(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); + Assert.False(await validator.ShouldRunIndexAsync(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); } } } -} +} \ No newline at end of file diff --git a/tests/NgTests/Validators/RegistrationLeafValidatorTests.cs b/tests/NgTests/Validators/RegistrationLeafValidatorTests.cs index 8da3077dd..5d31d45a8 100644 --- a/tests/NgTests/Validators/RegistrationLeafValidatorTests.cs +++ b/tests/NgTests/Validators/RegistrationLeafValidatorTests.cs @@ -21,9 +21,9 @@ private static IEnumerable ValidatorTestData(Func ValidatorSpecialTestData(Func( - () => validator.CompareLeaf(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); + () => validator.CompareLeafAsync(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); } [Theory] @@ -87,7 +87,7 @@ public class OnIndex PackageRegistrationIndexMetadata v3, bool shouldPass) { - var compareTask = Task.Run(async () => await validator.CompareLeaf(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); + var compareTask = Task.Run(async () => await validator.CompareLeafAsync(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); if (shouldPass) { @@ -116,7 +116,7 @@ public class OnLeaf PackageRegistrationLeafMetadata v2, PackageRegistrationLeafMetadata v3) { - await validator.CompareLeaf(ValidatorTestUtility.GetFakeValidationContext(), v2, v3); + await validator.CompareLeafAsync(ValidatorTestUtility.GetFakeValidationContext(), v2, v3); } [Theory] @@ -127,7 +127,7 @@ public class OnLeaf PackageRegistrationLeafMetadata v3) { await Assert.ThrowsAnyAsync( - () => validator.CompareLeaf(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); + () => validator.CompareLeafAsync(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); } [Theory] @@ -138,7 +138,7 @@ public class OnLeaf PackageRegistrationLeafMetadata v3, bool shouldPass) { - var compareTask = Task.Run(async () => await validator.CompareLeaf(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); + var compareTask = Task.Run(async () => await validator.CompareLeafAsync(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); if (shouldPass) { @@ -168,7 +168,7 @@ public class OnIndex PackageRegistrationIndexMetadata v2, PackageRegistrationIndexMetadata v3) { - Assert.Equal(true, await validator.ShouldRunLeaf(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); + Assert.Equal(true, await validator.ShouldRunLeafAsync(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); } [Theory] @@ -178,7 +178,7 @@ public class OnIndex PackageRegistrationIndexMetadata v2, PackageRegistrationIndexMetadata v3) { - Assert.Equal(false, await validator.ShouldRunLeaf(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); + Assert.Equal(false, await validator.ShouldRunLeafAsync(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); } } @@ -195,7 +195,7 @@ public class OnLeaf PackageRegistrationLeafMetadata v2, PackageRegistrationLeafMetadata v3) { - Assert.Equal(true, await validator.ShouldRunLeaf(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); + Assert.Equal(true, await validator.ShouldRunLeafAsync(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); } [Theory] @@ -205,9 +205,9 @@ public class OnLeaf PackageRegistrationLeafMetadata v2, PackageRegistrationLeafMetadata v3) { - Assert.Equal(false, await validator.ShouldRunLeaf(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); + Assert.Equal(false, await validator.ShouldRunLeafAsync(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); } } } } -} +} \ No newline at end of file diff --git a/tests/NgTests/Validators/ValidationContextTests.cs b/tests/NgTests/Validators/ValidationContextTests.cs new file mode 100644 index 000000000..1a49525a1 --- /dev/null +++ b/tests/NgTests/Validators/ValidationContextTests.cs @@ -0,0 +1,101 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using NuGet.Packaging.Core; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Monitoring; +using NuGet.Versioning; +using Xunit; + +namespace NgTests.Validators +{ + public class ValidationContextTests + { + private const string _key = "a"; + + [Fact] + public void DefaultConstructor_InitializesProperties() + { + var context = new ValidationContext(); + + Assert.Null(context.Package); + Assert.Null(context.Entries); + Assert.Null(context.DeletionAuditEntries); + Assert.Null(context.Client); + Assert.Equal(CancellationToken.None, context.CancellationToken); + } + + [Fact] + public void Constructor_InitializesProperties() + { + var package = new PackageIdentity("A", new NuGetVersion(1, 0, 0)); + var catalogIndexEntries = Enumerable.Empty(); + var deletionAuditEntries = Enumerable.Empty(); + var client = new CollectorHttpClient(); + var cancellationToken = new CancellationToken(canceled: true); + + var context = new ValidationContext( + package, + catalogIndexEntries, + deletionAuditEntries, + client, + cancellationToken); + + Assert.Same(package, context.Package); + Assert.Same(catalogIndexEntries, context.Entries); + Assert.Same(deletionAuditEntries, context.DeletionAuditEntries); + Assert.Same(client, context.Client); + Assert.Equal(cancellationToken, context.CancellationToken); + } + + [Fact] + public void GetCachedResultAsync_ReturnsMemoizedBoolTask() + { + var context = new ValidationContext(); + + var task1 = context.GetCachedResultAsync(_key, new Lazy>(() => Task.FromResult(true))); + var task2 = context.GetCachedResultAsync(_key, new Lazy>(() => Task.FromResult(true))); + + Assert.Same(task1, task2); + } + + [Fact] + public void GetCachedResultAsync_ReturnsMemoizedIndexTask() + { + var context = new ValidationContext(); + + var task1 = context.GetCachedResultAsync( + _key, + new Lazy>( + () => Task.FromResult(null))); + var task2 = context.GetCachedResultAsync( + _key, + new Lazy>( + () => Task.FromResult(null))); + + Assert.Same(task1, task2); + } + + [Fact] + public void GetMemoizedLeafAsync_ReturnsMemoizedLeafTask() + { + var context = new ValidationContext(); + + var task1 = context.GetCachedResultAsync( + _key, + new Lazy>( + () => Task.FromResult(null))); + var task2 = context.GetCachedResultAsync( + _key, + new Lazy>( + () => Task.FromResult(null))); + + Assert.Same(task1, task2); + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Validators/ValidatorTests.cs b/tests/NgTests/Validators/ValidatorTests.cs index 7d9806252..92c3c2778 100644 --- a/tests/NgTests/Validators/ValidatorTests.cs +++ b/tests/NgTests/Validators/ValidatorTests.cs @@ -19,10 +19,10 @@ public async Task Run_ReturnsPass() var validator = new TestableValidator(shouldRun, runInternal); - var data = new ValidationContext(); + var context = new ValidationContext(); // Act - var result = await validator.ValidateAsync(data); + var result = await validator.ValidateAsync(context); // Assert Assert.Same(validator, result.Validator); @@ -39,10 +39,10 @@ public async Task Run_ReturnsSkip() var validator = new TestableValidator(shouldRun, runInternal); - var data = new ValidationContext(); + var context = new ValidationContext(); // Act - var result = await validator.ValidateAsync(data); + var result = await validator.ValidateAsync(context); // Assert Assert.Same(validator, result.Validator); @@ -61,10 +61,10 @@ public async Task Run_ReturnsFail() var validator = new TestableValidator(shouldRun, runInternal); - var data = new ValidationContext(); + var context = new ValidationContext(); // Act - var result = await validator.ValidateAsync(data); + var result = await validator.ValidateAsync(context); // Assert Assert.Same(validator, result.Validator); @@ -81,12 +81,12 @@ public TestableValidator(Func shouldRun, Action runInternal) _runInternal = runInternal; } - protected override Task ShouldRun(ValidationContext data) + protected override Task ShouldRunAsync(ValidationContext context) { return Task.FromResult(_shouldRun()); } - protected override Task RunInternal(ValidationContext data) + protected override Task RunInternalAsync(ValidationContext context) { _runInternal(); @@ -96,4 +96,4 @@ protected override Task RunInternal(ValidationContext data) private Func _shouldRun; private Action _runInternal; } -} +} \ No newline at end of file From 606ca272986b9eef0ef1c5d20fbda745408bece8 Mon Sep 17 00:00:00 2001 From: Damon Tivel Date: Wed, 22 Aug 2018 12:04:39 -0700 Subject: [PATCH 08/15] Catalog2Dnx: improve throughput (#335) Progress on https://github.com/NuGet/NuGetGallery/issues/6267. --- src/Catalog/Dnx/DnxCatalogCollector.cs | 224 ++++++++++++--- src/Catalog/Dnx/DnxConstants.cs | 20 ++ src/Catalog/Dnx/DnxMaker.cs | 85 +++++- src/Catalog/Helpers/PackageUtility.cs | 4 +- .../NuGet.Services.Metadata.Catalog.csproj | 6 + src/Catalog/PackageCatalogItemCreator.cs | 33 ++- src/Catalog/Persistence/AggregateStorage.cs | 34 ++- .../Persistence/AggregateStorageFactory.cs | 19 +- .../Persistence/AzureCloudBlockBlob.cs | 10 +- src/Catalog/Persistence/AzureStorage.cs | 188 ++++++++++--- .../Persistence/AzureStorageFactory.cs | 39 ++- src/Catalog/Persistence/FileStorage.cs | 40 +-- src/Catalog/Persistence/FileStorageFactory.cs | 12 +- src/Catalog/Persistence/IAzureStorage.cs | 4 +- src/Catalog/Persistence/ICloudBlockBlob.cs | 1 + src/Catalog/Persistence/IStorage.cs | 9 + .../OptimisticConcurrencyControlToken.cs | 65 +++++ src/Catalog/Persistence/Storage.cs | 88 +++--- src/Catalog/Persistence/StorageConstants.cs | 11 + src/Catalog/Persistence/StorageFactory.cs | 3 +- .../FlatContainerPackagePathProvider.cs | 9 +- .../PackagesFolderPackagePathProvider.cs | 5 +- src/Catalog/Registration/RecordingStorage.cs | 17 ++ src/Catalog/Telemetry/TelemetryConstants.cs | 1 + src/Catalog/app.config | 4 + src/Catalog/packages.config | 1 + src/Ng/Arguments.cs | 11 +- src/Ng/CommandHelpers.cs | 35 ++- src/Ng/Jobs/Catalog2DnxJob.cs | 31 ++- src/Ng/Scripts/Catalog2DnxV3.cmd | 5 + src/Ng/Scripts/Catalog2DnxV3China.cmd | 5 + ...ervices.Metadata.Catalog.Monitoring.csproj | 20 +- .../app.config | 4 + .../packages.config | 14 +- src/V3PerPackage/EnqueueCommand.cs | 5 +- src/V3PerPackage/PerBatchProcessor.cs | 8 +- tests/CatalogTests/CatalogTests.csproj | 4 + .../Helpers/PackageUtilityTests.cs | 14 +- .../PackageCatalogItemCreatorTests.cs | 50 +++- .../Persistence/AggregateStorageTests.cs | 47 ++++ .../Persistence/AzureCloudBlockBlobTests.cs | 15 +- .../Persistence/FileStorageTests.cs | 43 +++ .../OptimisticConcurrencyControlTokenTests.cs | 59 ++++ .../Registration/RecordingStorageTests.cs | 43 +++ tests/NgTests/AggregateStorageTests.cs | 17 +- tests/NgTests/Data/Catalogs.cs | 19 ++ .../Data/TestCatalogEntries.Designer.cs | 50 ++++ tests/NgTests/Data/TestCatalogEntries.resx | 91 +++++++ tests/NgTests/DnxCatalogCollectorTests.cs | 256 +++++++++++++++--- tests/NgTests/DnxMakerTests.cs | 240 ++++++++++++++-- tests/NgTests/Infrastructure/MemoryStorage.cs | 24 +- 51 files changed, 1701 insertions(+), 341 deletions(-) create mode 100644 src/Catalog/Dnx/DnxConstants.cs create mode 100644 src/Catalog/Persistence/OptimisticConcurrencyControlToken.cs create mode 100644 src/Catalog/Persistence/StorageConstants.cs create mode 100644 tests/CatalogTests/Persistence/AggregateStorageTests.cs create mode 100644 tests/CatalogTests/Persistence/FileStorageTests.cs create mode 100644 tests/CatalogTests/Persistence/OptimisticConcurrencyControlTokenTests.cs create mode 100644 tests/CatalogTests/Registration/RecordingStorageTests.cs diff --git a/src/Catalog/Dnx/DnxCatalogCollector.cs b/src/Catalog/Dnx/DnxCatalogCollector.cs index 27975c287..7c28e974a 100644 --- a/src/Catalog/Dnx/DnxCatalogCollector.cs +++ b/src/Catalog/Dnx/DnxCatalogCollector.cs @@ -21,13 +21,17 @@ namespace NuGet.Services.Metadata.Catalog.Dnx public class DnxCatalogCollector : CommitCollector { private readonly StorageFactory _storageFactory; + private readonly IAzureStorage _sourceStorage; private readonly DnxMaker _dnxMaker; private readonly ILogger _logger; private readonly int _maxDegreeOfParallelism; + private readonly Uri _contentBaseAddress; public DnxCatalogCollector( Uri index, StorageFactory storageFactory, + IAzureStorage preferredPackageSourceStorage, + Uri contentBaseAddress, ITelemetryService telemetryService, ILogger logger, int maxDegreeOfParallelism, @@ -36,6 +40,8 @@ public class DnxCatalogCollector : CommitCollector : base(index, telemetryService, handlerFunc, httpClientTimeout) { _storageFactory = storageFactory ?? throw new ArgumentNullException(nameof(storageFactory)); + _sourceStorage = preferredPackageSourceStorage; + _contentBaseAddress = contentBaseAddress; _dnxMaker = new DnxMaker(storageFactory); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -49,8 +55,6 @@ public class DnxCatalogCollector : CommitCollector _maxDegreeOfParallelism = maxDegreeOfParallelism; } - public Uri ContentBaseAddress { get; set; } - protected override async Task OnProcessBatch( CollectorHttpClient client, IEnumerable items, @@ -62,7 +66,7 @@ public class DnxCatalogCollector : CommitCollector var catalogEntries = items.Select( item => new CatalogEntry( item["nuget:id"].ToString().ToLowerInvariant(), - NuGetVersionUtility.NormalizeVersion(item["nuget:version"].ToString().ToLowerInvariant()), + NuGetVersionUtility.NormalizeVersion(item["nuget:version"].ToString()).ToLowerInvariant(), item["@type"].ToString().Replace("nuget:", Schema.Prefixes.NuGet), item)) .ToList(); @@ -79,35 +83,56 @@ public class DnxCatalogCollector : CommitCollector return true; } - private async Task> ProcessCatalogEntriesAsync(CollectorHttpClient client, IEnumerable catalogEntries, CancellationToken cancellationToken) + private async Task> ProcessCatalogEntriesAsync( + CollectorHttpClient client, + IEnumerable catalogEntries, + CancellationToken cancellationToken) { var processedCatalogEntries = new ConcurrentBag(); await catalogEntries.ForEachAsync(_maxDegreeOfParallelism, async catalogEntry => { var packageId = catalogEntry.PackageId; - var packageVersion = catalogEntry.PackageVersion; + var normalizedPackageVersion = catalogEntry.NormalizedPackageVersion; if (catalogEntry.EntryType == Schema.DataTypes.PackageDetails.ToString()) { - var properties = GetTelemetryProperties(packageId, packageVersion); + var properties = GetTelemetryProperties(catalogEntry); using (_telemetryService.TrackDuration(TelemetryConstants.ProcessPackageDetailsSeconds, properties)) { - var sourceUri = new Uri(ContentBaseAddress, $"{packageId}.{packageVersion}.nupkg"); + var packageFileName = PackageUtility.GetPackageFileName( + packageId, + normalizedPackageVersion); + var sourceUri = new Uri(_contentBaseAddress, packageFileName); var destinationStorage = _storageFactory.Create(packageId); - var destinationUri = destinationStorage.GetUri(DnxMaker.GetRelativeAddressNupkg(packageId, packageVersion)); + var destinationRelativeUri = DnxMaker.GetRelativeAddressNupkg( + packageId, + normalizedPackageVersion); + var destinationUri = destinationStorage.GetUri(destinationRelativeUri); var isNupkgSynchronized = await destinationStorage.AreSynchronized(sourceUri, destinationUri); - var isPackageInIndex = await _dnxMaker.HasPackageInIndexAsync(destinationStorage, packageId, packageVersion, cancellationToken); - - if (isNupkgSynchronized && isPackageInIndex) + var isPackageInIndex = await _dnxMaker.HasPackageInIndexAsync( + destinationStorage, + packageId, + normalizedPackageVersion, + cancellationToken); + var areRequiredPropertiesPresent = await AreRequiredPropertiesPresentAsync(destinationStorage, destinationUri); + + if (isNupkgSynchronized && isPackageInIndex && areRequiredPropertiesPresent) { - _logger.LogInformation("No changes detected: {Id}/{Version}", packageId, packageVersion); + _logger.LogInformation("No changes detected: {Id}/{Version}", packageId, normalizedPackageVersion); + return; } - if (isNupkgSynchronized || await ProcessPackageDetailsAsync(client, packageId, packageVersion, sourceUri, cancellationToken)) + if ((isNupkgSynchronized && areRequiredPropertiesPresent) + || await ProcessPackageDetailsAsync( + client, + packageId, + normalizedPackageVersion, + sourceUri, + cancellationToken)) { processedCatalogEntries.Add(catalogEntry); } @@ -115,11 +140,11 @@ private async Task> ProcessCatalogEntriesAsync(Collect } else if (catalogEntry.EntryType == Schema.DataTypes.PackageDelete.ToString()) { - var properties = GetTelemetryProperties(packageId, packageVersion); + var properties = GetTelemetryProperties(catalogEntry); using (_telemetryService.TrackDuration(TelemetryConstants.ProcessPackageDeleteSeconds, properties)) { - await ProcessPackageDeleteAsync(packageId, packageVersion, cancellationToken); + await ProcessPackageDeleteAsync(packageId, normalizedPackageVersion, cancellationToken); processedCatalogEntries.Add(catalogEntry); } @@ -129,7 +154,24 @@ private async Task> ProcessCatalogEntriesAsync(Collect return processedCatalogEntries; } - private async Task UpdatePackageVersionIndexAsync(IEnumerable catalogEntries, CancellationToken cancellationToken) + private async Task AreRequiredPropertiesPresentAsync(Storage destinationStorage, Uri destinationUri) + { + var azureStorage = destinationStorage as IAzureStorage; + + if (azureStorage == null) + { + return true; + } + + return await azureStorage.HasPropertiesAsync( + destinationUri, + DnxConstants.ApplicationOctetStreamContentType, + DnxConstants.DefaultCacheControl); + } + + private async Task UpdatePackageVersionIndexAsync( + IEnumerable catalogEntries, + CancellationToken cancellationToken) { var catalogEntryGroups = catalogEntries.GroupBy(catalogEntry => catalogEntry.PackageId); @@ -150,11 +192,11 @@ private async Task UpdatePackageVersionIndexAsync(IEnumerable cata { if (catalogEntry.EntryType == Schema.DataTypes.PackageDetails.ToString()) { - versions.Add(NuGetVersion.Parse(catalogEntry.PackageVersion)); + versions.Add(NuGetVersion.Parse(catalogEntry.NormalizedPackageVersion)); } else if (catalogEntry.EntryType == Schema.DataTypes.PackageDelete.ToString()) { - versions.Remove(NuGetVersion.Parse(catalogEntry.PackageVersion)); + versions.Remove(NuGetVersion.Parse(catalogEntry.NormalizedPackageVersion)); } } }, cancellationToken); @@ -162,12 +204,109 @@ private async Task UpdatePackageVersionIndexAsync(IEnumerable cata foreach (var catalogEntry in catalogEntryGroup) { - _logger.LogInformation("Commit: {Id}/{Version}", packageId, catalogEntry.PackageVersion); + _logger.LogInformation("Commit: {Id}/{Version}", packageId, catalogEntry.NormalizedPackageVersion); } }); } private async Task ProcessPackageDetailsAsync( + HttpClient client, + string packageId, + string normalizedPackageVersion, + Uri sourceUri, + CancellationToken cancellationToken) + { + if (await ProcessPackageDetailsViaStorageAsync( + packageId, + normalizedPackageVersion, + cancellationToken)) + { + return true; + } + + _telemetryService.TrackMetric( + TelemetryConstants.UsePackageSourceFallback, + metric: 1, + properties: GetTelemetryProperties(packageId, normalizedPackageVersion)); + + return await ProcessPackageDetailsViaHttpAsync( + client, + packageId, + normalizedPackageVersion, + sourceUri, + cancellationToken); + } + + private async Task ProcessPackageDetailsViaStorageAsync( + string packageId, + string normalizedPackageVersion, + CancellationToken cancellationToken) + { + if (_sourceStorage == null) + { + return false; + } + + var packageFileName = PackageUtility.GetPackageFileName(packageId, normalizedPackageVersion); + var sourceUri = _sourceStorage.ResolveUri(packageFileName); + + var sourceBlob = await _sourceStorage.GetCloudBlockBlobReferenceAsync(sourceUri); + + if (await sourceBlob.ExistsAsync(cancellationToken)) + { + // It's possible (though unlikely) that the blob may change between reads. Reading a blob with a + // single GET request returns the whole blob in a consistent state, but we're reading the blob many + // different times. To detect the blob changing between reads, we check the ETag again later. + // If the ETag's differ, we'll fall back to using a single HTTP GET request. + var token1 = await _sourceStorage.GetOptimisticConcurrencyControlTokenAsync(sourceUri, cancellationToken); + + var nuspec = await GetNuspecAsync(sourceBlob, packageId, cancellationToken); + + if (string.IsNullOrEmpty(nuspec)) + { + _logger.LogWarning( + "No .nuspec available for {Id}/{Version}. Falling back to HTTP processing.", + packageId, + normalizedPackageVersion); + } + else + { + await _dnxMaker.AddPackageAsync( + _sourceStorage, + nuspec, + packageId, + normalizedPackageVersion, + cancellationToken); + + var token2 = await _sourceStorage.GetOptimisticConcurrencyControlTokenAsync(sourceUri, cancellationToken); + + if (token1 == token2) + { + _logger.LogInformation("Added .nupkg and .nuspec for package {Id}/{Version}", packageId, normalizedPackageVersion); + + return true; + } + else + { + _telemetryService.TrackMetric( + TelemetryConstants.BlobModified, + metric: 1, + properties: GetTelemetryProperties(packageId, normalizedPackageVersion)); + } + } + } + else + { + _telemetryService.TrackMetric( + TelemetryConstants.NonExistentBlob, + metric: 1, + properties: GetTelemetryProperties(packageId, normalizedPackageVersion)); + } + + return false; + } + + private async Task ProcessPackageDetailsViaHttpAsync( HttpClient client, string id, string version, @@ -210,28 +349,33 @@ private async Task UpdatePackageVersionIndexAsync(IEnumerable cata return true; } - private async Task ProcessPackageDeleteAsync(string id, string version, CancellationToken cancellationToken) + private async Task ProcessPackageDeleteAsync( + string packageId, + string normalizedPackageVersion, + CancellationToken cancellationToken) { await _dnxMaker.UpdatePackageVersionIndexAsync( - id, - versions => versions.Remove(NuGetVersion.Parse(version)), + packageId, + versions => versions.Remove(NuGetVersion.Parse(normalizedPackageVersion)), cancellationToken); - await _dnxMaker.DeletePackageAsync(id, version, cancellationToken); + await _dnxMaker.DeletePackageAsync(packageId, normalizedPackageVersion, cancellationToken); - _logger.LogInformation("Commit delete: {Id}/{Version}", id, version); + _logger.LogInformation("Commit delete: {Id}/{Version}", packageId, normalizedPackageVersion); } - private static void AssertNoMultipleEntriesForSamePackageIdentity(DateTime commitTimeStamp, IEnumerable catalogEntries) + private static void AssertNoMultipleEntriesForSamePackageIdentity( + DateTime commitTimeStamp, + IEnumerable catalogEntries) { var catalogEntriesForSamePackageIdentity = catalogEntries.GroupBy( catalogEntry => new { catalogEntry.PackageId, - catalogEntry.PackageVersion + catalogEntry.NormalizedPackageVersion }) .Where(group => group.Count() > 1) - .Select(group => $"{group.Key.PackageId} {group.Key.PackageVersion}"); + .Select(group => $"{group.Key.PackageId} {group.Key.NormalizedPackageVersion}"); if (catalogEntriesForSamePackageIdentity.Any()) { @@ -241,6 +385,17 @@ private static void AssertNoMultipleEntriesForSamePackageIdentity(DateTime commi } } + private static async Task GetNuspecAsync( + ICloudBlockBlob sourceBlob, + string packageId, + CancellationToken cancellationToken) + { + using (var stream = await sourceBlob.GetStreamAsync(cancellationToken)) + { + return GetNuspec(stream, packageId); + } + } + private static string GetNuspec(Stream stream, string id) { string name = $"{id}.nuspec"; @@ -274,26 +429,31 @@ private static string GetNuspec(Stream stream, string id) return null; } - private static Dictionary GetTelemetryProperties(string packageId, string packageVersion) + private static Dictionary GetTelemetryProperties(CatalogEntry catalogEntry) + { + return GetTelemetryProperties(catalogEntry.PackageId, catalogEntry.NormalizedPackageVersion); + } + + private static Dictionary GetTelemetryProperties(string packageId, string normalizedPackageVersion) { return new Dictionary() { { TelemetryConstants.Id, packageId }, - { TelemetryConstants.Version, packageVersion } + { TelemetryConstants.Version, normalizedPackageVersion } }; } private sealed class CatalogEntry { internal string PackageId { get; } - internal string PackageVersion { get; } + internal string NormalizedPackageVersion { get; } internal string EntryType { get; } internal JToken Entry { get; } - internal CatalogEntry(string packageId, string packageVersion, string entryType, JToken entry) + internal CatalogEntry(string packageId, string normalizedPackageVersion, string entryType, JToken entry) { PackageId = packageId; - PackageVersion = packageVersion; + NormalizedPackageVersion = normalizedPackageVersion; EntryType = entryType; Entry = entry; } diff --git a/src/Catalog/Dnx/DnxConstants.cs b/src/Catalog/Dnx/DnxConstants.cs new file mode 100644 index 000000000..385f520a7 --- /dev/null +++ b/src/Catalog/Dnx/DnxConstants.cs @@ -0,0 +1,20 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using NuGet.Services.Metadata.Catalog.Persistence; + +namespace NuGet.Services.Metadata.Catalog.Dnx +{ + internal static class DnxConstants + { + internal const string ApplicationOctetStreamContentType = "application/octet-stream"; + internal const string DefaultCacheControl = "max-age=120"; + + internal static readonly IReadOnlyDictionary RequiredBlobProperties = new Dictionary() + { + { StorageConstants.CacheControl, DefaultCacheControl }, + { StorageConstants.ContentType, ApplicationOctetStreamContentType } + }; + } +} \ No newline at end of file diff --git a/src/Catalog/Dnx/DnxMaker.cs b/src/Catalog/Dnx/DnxMaker.cs index a5b3dc378..478d7a509 100644 --- a/src/Catalog/Dnx/DnxMaker.cs +++ b/src/Catalog/Dnx/DnxMaker.cs @@ -26,8 +26,8 @@ public DnxMaker(StorageFactory storageFactory) public async Task AddPackageAsync( Stream nupkgStream, string nuspec, - string id, - string version, + string packageId, + string normalizedPackageVersion, CancellationToken cancellationToken) { if (nupkgStream == null) @@ -40,23 +40,57 @@ public DnxMaker(StorageFactory storageFactory) throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(nuspec)); } - if (string.IsNullOrEmpty(id)) + if (string.IsNullOrEmpty(packageId)) { - throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(id)); + throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(packageId)); } - if (string.IsNullOrEmpty(version)) + if (string.IsNullOrEmpty(normalizedPackageVersion)) { - throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(version)); + throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(normalizedPackageVersion)); } cancellationToken.ThrowIfCancellationRequested(); - var storage = _storageFactory.Create(id); - var normalizedVersion = NuGetVersionUtility.NormalizeVersion(version); + var storage = _storageFactory.Create(packageId); + var nuspecUri = await SaveNuspecAsync(storage, packageId, normalizedPackageVersion, nuspec, cancellationToken); + var nupkgUri = await SaveNupkgAsync(nupkgStream, storage, packageId, normalizedPackageVersion, cancellationToken); + + return new DnxEntry(nupkgUri, nuspecUri); + } + + public async Task AddPackageAsync( + IStorage sourceStorage, + string nuspec, + string packageId, + string normalizedPackageVersion, + CancellationToken cancellationToken) + { + if (sourceStorage == null) + { + throw new ArgumentNullException(nameof(sourceStorage)); + } + + if (string.IsNullOrEmpty(nuspec)) + { + throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(nuspec)); + } - var nuspecUri = await SaveNuspecAsync(storage, id, normalizedVersion, nuspec, cancellationToken); - var nupkgUri = await SaveNupkgAsync(nupkgStream, storage, id, normalizedVersion, cancellationToken); + if (string.IsNullOrEmpty(packageId)) + { + throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(packageId)); + } + + if (string.IsNullOrEmpty(normalizedPackageVersion)) + { + throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(normalizedPackageVersion)); + } + + cancellationToken.ThrowIfCancellationRequested(); + + var destinationStorage = _storageFactory.Create(packageId); + var nuspecUri = await SaveNuspecAsync(destinationStorage, packageId, normalizedPackageVersion, nuspec, cancellationToken); + var nupkgUri = await CopyNupkgAsync(sourceStorage, destinationStorage, packageId, normalizedPackageVersion, cancellationToken); return new DnxEntry(nupkgUri, nuspecUri); } @@ -111,8 +145,9 @@ private async Task SaveNuspecAsync(Storage storage, string id, string versi { var relativeAddress = GetRelativeAddressNuspec(id, version); var nuspecUri = new Uri(storage.BaseAddress, relativeAddress); + var content = new StringStorageContent(nuspec, "text/xml", DnxConstants.DefaultCacheControl); - await storage.SaveAsync(nuspecUri, new StringStorageContent(nuspec, "text/xml", "max-age=120"), cancellationToken); + await storage.SaveAsync(nuspecUri, content, cancellationToken); return nuspecUri; } @@ -195,10 +230,36 @@ private StorageContent CreateContent(IEnumerable versions) private async Task SaveNupkgAsync(Stream nupkgStream, Storage storage, string id, string version, CancellationToken cancellationToken) { Uri nupkgUri = new Uri(storage.BaseAddress, GetRelativeAddressNupkg(id, version)); - await storage.SaveAsync(nupkgUri, new StreamStorageContent(nupkgStream, "application/octet-stream", "max-age=120"), cancellationToken); + var content = new StreamStorageContent( + nupkgStream, + DnxConstants.ApplicationOctetStreamContentType, + DnxConstants.DefaultCacheControl); + + await storage.SaveAsync(nupkgUri, content, cancellationToken); + return nupkgUri; } + private async Task CopyNupkgAsync( + IStorage sourceStorage, + Storage destinationStorage, + string id, string version, CancellationToken cancellationToken) + { + var packageFileName = PackageUtility.GetPackageFileName(id, version); + var sourceUri = sourceStorage.ResolveUri(packageFileName); + var destinationRelativeUri = GetRelativeAddressNupkg(id, version); + var destinationUri = destinationStorage.ResolveUri(destinationRelativeUri); + + await sourceStorage.CopyAsync( + sourceUri, + destinationStorage, + destinationUri, + DnxConstants.RequiredBlobProperties, + cancellationToken); + + return destinationUri; + } + private async Task DeleteNuspecAsync(Storage storage, string id, string version, CancellationToken cancellationToken) { string relativeAddress = GetRelativeAddressNuspec(id, version); diff --git a/src/Catalog/Helpers/PackageUtility.cs b/src/Catalog/Helpers/PackageUtility.cs index 31dc2b80a..22cf8dd8c 100644 --- a/src/Catalog/Helpers/PackageUtility.cs +++ b/src/Catalog/Helpers/PackageUtility.cs @@ -7,7 +7,7 @@ namespace NuGet.Services.Metadata.Catalog.Helpers { public static class PackageUtility { - public static string GetPackageFileNameLowercase(string packageId, string packageVersion) + public static string GetPackageFileName(string packageId, string packageVersion) { if (string.IsNullOrEmpty(packageId)) { @@ -19,7 +19,7 @@ public static string GetPackageFileNameLowercase(string packageId, string packag throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(packageVersion)); } - return $"{packageId.ToLowerInvariant()}.{packageVersion.ToLowerInvariant()}.nupkg"; + return $"{packageId}.{packageVersion}.nupkg"; } } } \ No newline at end of file diff --git a/src/Catalog/NuGet.Services.Metadata.Catalog.csproj b/src/Catalog/NuGet.Services.Metadata.Catalog.csproj index a0d574452..b8eb0d5b6 100644 --- a/src/Catalog/NuGet.Services.Metadata.Catalog.csproj +++ b/src/Catalog/NuGet.Services.Metadata.Catalog.csproj @@ -93,6 +93,9 @@ ..\..\packages\WindowsAzure.Storage.8.2.1\lib\net45\Microsoft.WindowsAzure.Storage.dll + + ..\..\packages\Microsoft.Azure.Storage.DataMovement.0.6.0\lib\net45\Microsoft.WindowsAzure.Storage.DataMovement.dll + ..\..\packages\Newtonsoft.Json.9.0.1\lib\net45\Newtonsoft.Json.dll True @@ -173,6 +176,7 @@ + @@ -192,6 +196,8 @@ + + diff --git a/src/Catalog/PackageCatalogItemCreator.cs b/src/Catalog/PackageCatalogItemCreator.cs index dada18f2d..7fc5841de 100644 --- a/src/Catalog/PackageCatalogItemCreator.cs +++ b/src/Catalog/PackageCatalogItemCreator.cs @@ -82,25 +82,30 @@ public sealed class PackageCatalogItemCreator : IPackageCatalogItemCreator return item; } - private async Task GetPackageViaStorageAsync(FeedPackageDetails packageItem, CancellationToken cancellationToken) + private async Task GetPackageViaStorageAsync( + FeedPackageDetails packageItem, + CancellationToken cancellationToken) { PackageCatalogItem item = null; - var packageFileName = PackageUtility.GetPackageFileNameLowercase(packageItem.PackageId, packageItem.PackageVersion); - var blob = await _storage.GetCloudBlockBlobReferenceAsync(packageFileName); + var packageId = packageItem.PackageId.ToLowerInvariant(); + var packageVersion = packageItem.PackageVersion.ToLowerInvariant(); + var packageFileName = PackageUtility.GetPackageFileName(packageId, packageVersion); + var blobUri = _storage.ResolveUri(packageFileName); + var blob = await _storage.GetCloudBlockBlobReferenceAsync(blobUri); - if (blob == null) + if (!await blob.ExistsAsync(cancellationToken)) { _telemetryService.TrackMetric( TelemetryConstants.NonExistentBlob, metric: 1, - properties: GetProperties(packageItem, blob)); + properties: GetProperties(packageId, packageVersion, blob)); return item; } using (_telemetryService.TrackDuration( TelemetryConstants.PackageBlobReadSeconds, - GetProperties(packageItem, blob: null))) + GetProperties(packageId, packageVersion, blob: null))) { await blob.FetchAttributesAsync(cancellationToken); @@ -146,7 +151,7 @@ private async Task GetPackageViaStorageAsync(FeedPackageDeta _telemetryService.TrackMetric( TelemetryConstants.BlobModified, metric: 1, - properties: GetProperties(packageItem, blob)); + properties: GetProperties(packageId, packageVersion, blob)); } } } @@ -155,7 +160,7 @@ private async Task GetPackageViaStorageAsync(FeedPackageDeta _telemetryService.TrackMetric( TelemetryConstants.NonExistentPackageHash, metric: 1, - properties: GetProperties(packageItem, blob)); + properties: GetProperties(packageId, packageVersion, blob)); } } @@ -223,11 +228,19 @@ private async Task GetPackageViaHttpAsync(FeedPackageDetails } private static Dictionary GetProperties(FeedPackageDetails packageItem, ICloudBlockBlob blob) + { + return GetProperties( + packageItem.PackageId.ToLowerInvariant(), + packageItem.PackageVersion.ToLowerInvariant(), + blob); + } + + private static Dictionary GetProperties(string packageId, string packageVersion, ICloudBlockBlob blob) { var properties = new Dictionary() { - { TelemetryConstants.Id, packageItem.PackageId.ToLowerInvariant() }, - { TelemetryConstants.Version, packageItem.PackageVersion.ToLowerInvariant() } + { TelemetryConstants.Id, packageId }, + { TelemetryConstants.Version, packageVersion } }; if (blob != null) diff --git a/src/Catalog/Persistence/AggregateStorage.cs b/src/Catalog/Persistence/AggregateStorage.cs index b786db82a..08fc30304 100644 --- a/src/Catalog/Persistence/AggregateStorage.cs +++ b/src/Catalog/Persistence/AggregateStorage.cs @@ -11,16 +11,16 @@ namespace NuGet.Services.Metadata.Catalog.Persistence public class AggregateStorage : Storage { public delegate StorageContent WriteSecondaryStorageContentInterceptor( - Uri primaryStorageBaseUri, - Uri primaryResourceUri, + Uri primaryStorageBaseUri, + Uri primaryResourceUri, Uri secondaryStorageBaseUri, - Uri secondaryResourceUri, + Uri secondaryResourceUri, StorageContent content); private readonly Storage _primaryStorage; private readonly Storage[] _secondaryStorage; private readonly WriteSecondaryStorageContentInterceptor _writeSecondaryStorageContentInterceptor; - + public AggregateStorage(Uri baseAddress, Storage primaryStorage, Storage[] secondaryStorage, WriteSecondaryStorageContentInterceptor writeSecondaryStorageContentInterceptor) : base(baseAddress) @@ -28,11 +28,21 @@ public class AggregateStorage : Storage _primaryStorage = primaryStorage; _secondaryStorage = secondaryStorage; _writeSecondaryStorageContentInterceptor = writeSecondaryStorageContentInterceptor; - + BaseAddress = _primaryStorage.BaseAddress; } - - protected override Task OnSave(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) + + protected override Task OnCopyAsync( + Uri sourceUri, + IStorage destinationStorage, + Uri destinationUri, + IReadOnlyDictionary destinationProperties, + CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + + protected override Task OnSaveAsync(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) { var tasks = new List(); tasks.Add(_primaryStorage.SaveAsync(resourceUri, content, cancellationToken)); @@ -46,9 +56,9 @@ protected override Task OnSave(Uri resourceUri, StorageContent content, Cancella if (_writeSecondaryStorageContentInterceptor != null) { secondaryContent = _writeSecondaryStorageContentInterceptor( - _primaryStorage.BaseAddress, - resourceUri, - storage.BaseAddress, + _primaryStorage.BaseAddress, + resourceUri, + storage.BaseAddress, secondaryResourceUri, content); } @@ -58,12 +68,12 @@ protected override Task OnSave(Uri resourceUri, StorageContent content, Cancella return Task.WhenAll(tasks); } - protected override Task OnLoad(Uri resourceUri, CancellationToken cancellationToken) + protected override Task OnLoadAsync(Uri resourceUri, CancellationToken cancellationToken) { return _primaryStorage.LoadAsync(resourceUri, cancellationToken); } - protected override Task OnDelete(Uri resourceUri, CancellationToken cancellationToken) + protected override Task OnDeleteAsync(Uri resourceUri, CancellationToken cancellationToken) { var tasks = new List(); tasks.Add(_primaryStorage.DeleteAsync(resourceUri, cancellationToken)); diff --git a/src/Catalog/Persistence/AggregateStorageFactory.cs b/src/Catalog/Persistence/AggregateStorageFactory.cs index 0469e29fb..cb82f0a4b 100644 --- a/src/Catalog/Persistence/AggregateStorageFactory.cs +++ b/src/Catalog/Persistence/AggregateStorageFactory.cs @@ -10,13 +10,23 @@ public class AggregateStorageFactory : StorageFactory { private readonly AggregateStorage.WriteSecondaryStorageContentInterceptor _writeSecondaryStorageContentInterceptor; - public AggregateStorageFactory(StorageFactory primaryStorageFactory, StorageFactory[] secondaryStorageFactories) - : this(primaryStorageFactory, secondaryStorageFactories, null) + public AggregateStorageFactory( + StorageFactory primaryStorageFactory, + StorageFactory[] secondaryStorageFactories, + bool verbose) + : this( + primaryStorageFactory, + secondaryStorageFactories, + writeSecondaryStorageContentInterceptor: null, + verbose: verbose) { } - public AggregateStorageFactory(StorageFactory primaryStorageFactory, StorageFactory[] secondaryStorageFactories, - AggregateStorage.WriteSecondaryStorageContentInterceptor writeSecondaryStorageContentInterceptor) + public AggregateStorageFactory( + StorageFactory primaryStorageFactory, + StorageFactory[] secondaryStorageFactories, + AggregateStorage.WriteSecondaryStorageContentInterceptor writeSecondaryStorageContentInterceptor, + bool verbose) { PrimaryStorageFactory = primaryStorageFactory; SecondaryStorageFactories = secondaryStorageFactories; @@ -24,6 +34,7 @@ public AggregateStorageFactory(StorageFactory primaryStorageFactory, StorageFact BaseAddress = PrimaryStorageFactory.BaseAddress; DestinationAddress = PrimaryStorageFactory.DestinationAddress; + Verbose = verbose; } public override Storage Create(string name = null) diff --git a/src/Catalog/Persistence/AzureCloudBlockBlob.cs b/src/Catalog/Persistence/AzureCloudBlockBlob.cs index ce074449a..efe6272b9 100644 --- a/src/Catalog/Persistence/AzureCloudBlockBlob.cs +++ b/src/Catalog/Persistence/AzureCloudBlockBlob.cs @@ -20,12 +20,12 @@ public sealed class AzureCloudBlockBlob : ICloudBlockBlob public AzureCloudBlockBlob(CloudBlockBlob blob) { - if (blob == null) - { - throw new ArgumentNullException(nameof(blob)); - } + _blob = blob ?? throw new ArgumentNullException(nameof(blob)); + } - _blob = blob; + public Task ExistsAsync(CancellationToken cancellationToken) + { + return _blob.ExistsAsync(); } public async Task FetchAttributesAsync(CancellationToken cancellationToken) diff --git a/src/Catalog/Persistence/AzureStorage.cs b/src/Catalog/Persistence/AzureStorage.cs index 228d44b70..ec7373a2e 100644 --- a/src/Catalog/Persistence/AzureStorage.cs +++ b/src/Catalog/Persistence/AzureStorage.cs @@ -11,37 +11,60 @@ using System.Threading.Tasks; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; +using Microsoft.WindowsAzure.Storage.DataMovement; using Microsoft.WindowsAzure.Storage.RetryPolicies; namespace NuGet.Services.Metadata.Catalog.Persistence { public class AzureStorage : Storage, IAzureStorage { + private readonly bool _compressContent; private readonly CloudBlobDirectory _directory; private readonly BlobRequestOptions _blobRequestOptions; + private readonly bool _useServerSideCopy; public static readonly TimeSpan DefaultServerTimeout = TimeSpan.FromSeconds(30); public static readonly TimeSpan DefaultMaxExecutionTime = TimeSpan.FromMinutes(10); - public AzureStorage(CloudStorageAccount account, - string containerName, - string path, - Uri baseAddress) - : this(account, containerName, path, baseAddress, DefaultMaxExecutionTime, DefaultServerTimeout) + public AzureStorage( + CloudStorageAccount account, + string containerName, + string path, + Uri baseAddress, + bool useServerSideCopy, + bool compressContent, + bool verbose) + : this( + account, + containerName, + path, + baseAddress, + DefaultMaxExecutionTime, + DefaultServerTimeout, + useServerSideCopy, + compressContent, + verbose) { } - public AzureStorage(CloudStorageAccount account, - string containerName, - string path, - Uri baseAddress, - TimeSpan maxExecutionTime, - TimeSpan serverTimeout) + public AzureStorage( + CloudStorageAccount account, + string containerName, + string path, + Uri baseAddress, + TimeSpan maxExecutionTime, + TimeSpan serverTimeout, + bool useServerSideCopy, + bool compressContent, + bool verbose) : this(account.CreateCloudBlobClient().GetContainerReference(containerName).GetDirectoryReference(path), baseAddress, maxExecutionTime, serverTimeout) { + _useServerSideCopy = useServerSideCopy; + _compressContent = compressContent; + Verbose = verbose; } private AzureStorage(CloudBlobDirectory directory, Uri baseAddress, TimeSpan maxExecutionTime, TimeSpan serverTimeout) @@ -59,8 +82,6 @@ private AzureStorage(CloudBlobDirectory directory, Uri baseAddress, TimeSpan max } } - ResetStatistics(); - _blobRequestOptions = new BlobRequestOptions() { ServerTimeout = serverTimeout, @@ -69,13 +90,26 @@ private AzureStorage(CloudBlobDirectory directory, Uri baseAddress, TimeSpan max }; } - public bool CompressContent + public override async Task GetOptimisticConcurrencyControlTokenAsync( + Uri resourceUri, + CancellationToken cancellationToken) { - get; - set; + if (resourceUri == null) + { + throw new ArgumentNullException(nameof(resourceUri)); + } + + cancellationToken.ThrowIfCancellationRequested(); + + string blobName = GetName(resourceUri); + CloudBlockBlob blob = _directory.GetBlockBlobReference(blobName); + + await blob.FetchAttributesAsync(cancellationToken); + + return new OptimisticConcurrencyControlToken(blob.Properties.ETag); } - static Uri GetDirectoryUri(CloudBlobDirectory directory) + private static Uri GetDirectoryUri(CloudBlobDirectory directory) { Uri uri = new UriBuilder(directory.Uri) { @@ -119,15 +153,70 @@ private StorageListItem GetStorageListItem(IListBlobItem listBlobItem) return new StorageListItem(listBlobItem.Uri, lastModified); } - // save - protected override async Task OnSave(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) + protected override async Task OnCopyAsync( + Uri sourceUri, + IStorage destinationStorage, + Uri destinationUri, + IReadOnlyDictionary destinationProperties, + CancellationToken cancellationToken) + { + var azureDestinationStorage = destinationStorage as AzureStorage; + + if (azureDestinationStorage == null) + { + throw new NotImplementedException("Copying is only supported from Azure storage to Azure storage."); + } + + string sourceName = GetName(sourceUri); + string destinationName = azureDestinationStorage.GetName(destinationUri); + + CloudBlockBlob sourceBlob = _directory.GetBlockBlobReference(sourceName); + CloudBlockBlob destinationBlob = azureDestinationStorage._directory.GetBlockBlobReference(destinationName); + + var context = new SingleTransferContext(); + + if (destinationProperties?.Count > 0) + { + context.SetAttributesCallback = new SetAttributesCallback((destination) => + { + var blob = (CloudBlockBlob)destination; + + // The copy statement copied all properties from the source blob to the destination blob; however, + // there may be required properties on destination blob, all of which may have not already existed + // on the source blob at the time of copy. + foreach (var property in destinationProperties) + { + switch (property.Key) + { + case StorageConstants.CacheControl: + blob.Properties.CacheControl = property.Value; + break; + + case StorageConstants.ContentType: + blob.Properties.ContentType = property.Value; + break; + + default: + throw new NotImplementedException($"Storage property '{property.Value}' is not supported."); + } + } + }); + } + + context.ShouldOverwriteCallback = new ShouldOverwriteCallback((source, destination) => true); + + await TransferManager.CopyAsync(sourceBlob, destinationBlob, _useServerSideCopy, options: null, context: context); + } + + protected override async Task OnSaveAsync(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) { string name = GetName(resourceUri); CloudBlockBlob blob = _directory.GetBlockBlobReference(name); blob.Properties.ContentType = content.ContentType; blob.Properties.CacheControl = content.CacheControl; - if (CompressContent) + + if (_compressContent) { blob.Properties.ContentEncoding = "gzip"; using (Stream stream = content.GetContentStream()) @@ -140,12 +229,14 @@ protected override async Task OnSave(Uri resourceUri, StorageContent content, Ca } destinationStream.Seek(0, SeekOrigin.Begin); + await blob.UploadFromStreamAsync(destinationStream, - accessCondition: null, - options: _blobRequestOptions, - operationContext: null, - cancellationToken: cancellationToken); - Trace.WriteLine(String.Format("Saved compressed blob {0} to container {1}", blob.Uri.ToString(), _directory.Container.Name)); + accessCondition: null, + options: _blobRequestOptions, + operationContext: null, + cancellationToken: cancellationToken); + + Trace.WriteLine(string.Format("Saved compressed blob {0} to container {1}", blob.Uri.ToString(), _directory.Container.Name)); } } else @@ -153,13 +244,15 @@ protected override async Task OnSave(Uri resourceUri, StorageContent content, Ca using (Stream stream = content.GetContentStream()) { await blob.UploadFromStreamAsync(stream, - accessCondition: null, - options: _blobRequestOptions, - operationContext: null, - cancellationToken: cancellationToken); - Trace.WriteLine(String.Format("Saved uncompressed blob {0} to container {1}", blob.Uri.ToString(), _directory.Container.Name)); + accessCondition: null, + options: _blobRequestOptions, + operationContext: null, + cancellationToken: cancellationToken); } + + Trace.WriteLine(string.Format("Saved uncompressed blob {0} to container {1}", blob.Uri.ToString(), _directory.Container.Name)); } + await TryTakeBlobSnapshotAsync(blob); } @@ -202,8 +295,7 @@ private async Task TryTakeBlobSnapshotAsync(CloudBlockBlob blob) } } - // load - protected override async Task OnLoad(Uri resourceUri, CancellationToken cancellationToken) + protected override async Task OnLoadAsync(Uri resourceUri, CancellationToken cancellationToken) { // the Azure SDK will treat a starting / as an absolute URL, // while we may be working in a subdirectory of a storage container @@ -254,8 +346,7 @@ protected override async Task OnLoad(Uri resourceUri, Cancellati return null; } - // delete - protected override async Task OnDelete(Uri resourceUri, CancellationToken cancellationToken) + protected override async Task OnDeleteAsync(Uri resourceUri, CancellationToken cancellationToken) { string name = GetName(resourceUri); @@ -292,25 +383,34 @@ public override async Task AreSynchronized(Uri firstResourceUri, Uri secon return !(await source.ExistsAsync()); } - public async Task GetCloudBlockBlobReferenceAsync(string name) + public async Task GetCloudBlockBlobReferenceAsync(Uri blobUri) { - Uri uri = ResolveUri(name); - string blobName = GetName(uri); + string blobName = GetName(blobUri); CloudBlockBlob blob = _directory.GetBlockBlobReference(blobName); + var blobExists = await blob.ExistsAsync(); - if (await blob.ExistsAsync()) + if (Verbose && !blobExists) { - return new AzureCloudBlockBlob(blob); + Trace.WriteLine($"The blob {blobUri.AbsoluteUri} does not exist."); } - if (Verbose) + return new AzureCloudBlockBlob(blob); + } + + public async Task HasPropertiesAsync(Uri blobUri, string contentType, string cacheControl) + { + var blobName = GetName(blobUri); + var blob = _directory.GetBlockBlobReference(blobName); + + if (await blob.ExistsAsync()) { - Trace.WriteLine($"The blob {uri} does not exist."); + await blob.FetchAttributesAsync(); + + return string.Equals(blob.Properties.ContentType, contentType) + && string.Equals(blob.Properties.CacheControl, cacheControl); } - // We could return a reference even when the blob does not exist; - // however, there's currently no scenario for this. - return null; + return false; } } } \ No newline at end of file diff --git a/src/Catalog/Persistence/AzureStorageFactory.cs b/src/Catalog/Persistence/AzureStorageFactory.cs index 2518c4888..e1ffa2d47 100644 --- a/src/Catalog/Persistence/AzureStorageFactory.cs +++ b/src/Catalog/Persistence/AzureStorageFactory.cs @@ -14,19 +14,25 @@ public class AzureStorageFactory : StorageFactory private readonly Uri _differentBaseAddress = null; private readonly TimeSpan _maxExecutionTime; private readonly TimeSpan _serverTimeout; - - public AzureStorageFactory(CloudStorageAccount account, - string containerName, - TimeSpan maxExecutionTime, - TimeSpan serverTimeout, - string path = null, - Uri baseAddress = null) + private readonly bool _useServerSideCopy; + + public AzureStorageFactory( + CloudStorageAccount account, + string containerName, + TimeSpan maxExecutionTime, + TimeSpan serverTimeout, + string path, + Uri baseAddress, + bool useServerSideCopy, + bool compressContent, + bool verbose) { _account = account; _containerName = containerName; _path = null; _maxExecutionTime = maxExecutionTime; _serverTimeout = serverTimeout; + _useServerSideCopy = useServerSideCopy; if (path != null) { @@ -62,9 +68,12 @@ public class AzureStorageFactory : StorageFactory blobEndpointBuilder.Port = -1; DestinationAddress = new Uri(blobEndpointBuilder.Uri, containerName + "/" + _path ?? string.Empty); + + CompressContent = compressContent; + Verbose = verbose; } - public bool CompressContent { get; set; } + public bool CompressContent { get; } public override Storage Create(string name = null) { @@ -79,8 +88,16 @@ public override Storage Create(string name = null) newBase = new Uri(_differentBaseAddress, name + "/"); } - return new AzureStorage(_account, _containerName, path, newBase, _maxExecutionTime, _serverTimeout) - { Verbose = Verbose, CompressContent = CompressContent }; + return new AzureStorage( + _account, + _containerName, + path, + newBase, + _maxExecutionTime, + _serverTimeout, + _useServerSideCopy, + CompressContent, + Verbose); } } -} +} \ No newline at end of file diff --git a/src/Catalog/Persistence/FileStorage.cs b/src/Catalog/Persistence/FileStorage.cs index 050fec49b..530c0c892 100644 --- a/src/Catalog/Persistence/FileStorage.cs +++ b/src/Catalog/Persistence/FileStorage.cs @@ -12,10 +12,12 @@ namespace NuGet.Services.Metadata.Catalog.Persistence { public class FileStorage : Storage { - public FileStorage(string baseAddress, string path) - : this(new Uri(baseAddress), path) { } + public FileStorage(string baseAddress, string path, bool verbose) + : this(new Uri(baseAddress), path, verbose) + { + } - public FileStorage(Uri baseAddress, string path) + public FileStorage(Uri baseAddress, string path, bool verbose) : base(baseAddress) { Path = path; @@ -26,7 +28,7 @@ public FileStorage(Uri baseAddress, string path) directoryInfo.Create(); } - ResetStatistics(); + Verbose = verbose; } //File exists @@ -39,24 +41,30 @@ public override Task> ListAsync(CancellationToken c { DirectoryInfo directoryInfo = new DirectoryInfo(Path); var files = directoryInfo.GetFiles("*", SearchOption.AllDirectories) - .Select(file => + .Select(file => new StorageListItem(GetUri(file.FullName.Replace(Path, string.Empty)), file.LastWriteTimeUtc)); return Task.FromResult(files.AsEnumerable()); } public string Path - { + { get; set; } - // save - - protected override async Task OnSave(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) + protected override Task OnCopyAsync( + Uri sourceUri, + IStorage destinationStorage, + Uri destinationUri, + IReadOnlyDictionary destinationProperties, + CancellationToken cancellationToken) { - SaveCount++; + throw new NotImplementedException(); + } + protected override async Task OnSaveAsync(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) + { TraceMethod("SAVE", resourceUri); string name = GetName(resourceUri); @@ -88,13 +96,11 @@ protected override async Task OnSave(Uri resourceUri, StorageContent content, Ca using (FileStream stream = File.Create(path + name)) { - await content.GetContentStream().CopyToAsync(stream,4096, cancellationToken); + await content.GetContentStream().CopyToAsync(stream, 4096, cancellationToken); } } - // load - - protected override async Task OnLoad(Uri resourceUri, CancellationToken cancellationToken) + protected override async Task OnLoadAsync(Uri resourceUri, CancellationToken cancellationToken) { string name = GetName(resourceUri); @@ -125,9 +131,7 @@ protected override async Task OnLoad(Uri resourceUri, Cancellati return null; } - // delete - - protected override async Task OnDelete(Uri resourceUri, CancellationToken cancellationToken) + protected override async Task OnDeleteAsync(Uri resourceUri, CancellationToken cancellationToken) { string name = GetName(resourceUri); @@ -152,7 +156,7 @@ protected override async Task OnDelete(Uri resourceUri, CancellationToken cancel FileInfo fileInfo = new FileInfo(filename); if (fileInfo.Exists) { - await Task.Run(() => { fileInfo.Delete(); },cancellationToken); + await Task.Run(() => { fileInfo.Delete(); }, cancellationToken); } } } diff --git a/src/Catalog/Persistence/FileStorageFactory.cs b/src/Catalog/Persistence/FileStorageFactory.cs index a9cbfbc3e..be8296988 100644 --- a/src/Catalog/Persistence/FileStorageFactory.cs +++ b/src/Catalog/Persistence/FileStorageFactory.cs @@ -7,12 +7,14 @@ namespace NuGet.Services.Metadata.Catalog.Persistence { public class FileStorageFactory : StorageFactory { - string _path; - - public FileStorageFactory(Uri baseAddress, string path) + private readonly string _path; + + public FileStorageFactory(Uri baseAddress, string path, bool verbose) { BaseAddress = new Uri(baseAddress.ToString().TrimEnd('/') + '/'); _path = path.TrimEnd('\\') + '\\'; + + Verbose = verbose; } public override Storage Create(string name = null) @@ -20,7 +22,7 @@ public override Storage Create(string name = null) string fileSystemPath = (name == null) ? _path.Trim('\\') : _path + name; string uriPath = name ?? string.Empty; - return new FileStorage(BaseAddress + uriPath, fileSystemPath) { Verbose = Verbose }; + return new FileStorage(BaseAddress + uriPath, fileSystemPath, Verbose); } } -} +} \ No newline at end of file diff --git a/src/Catalog/Persistence/IAzureStorage.cs b/src/Catalog/Persistence/IAzureStorage.cs index 1ea3e2728..96c4e1733 100644 --- a/src/Catalog/Persistence/IAzureStorage.cs +++ b/src/Catalog/Persistence/IAzureStorage.cs @@ -1,12 +1,14 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; using System.Threading.Tasks; namespace NuGet.Services.Metadata.Catalog.Persistence { public interface IAzureStorage : IStorage { - Task GetCloudBlockBlobReferenceAsync(string name); + Task GetCloudBlockBlobReferenceAsync(Uri uri); + Task HasPropertiesAsync(Uri blobUri, string contentType, string cacheControl); } } \ No newline at end of file diff --git a/src/Catalog/Persistence/ICloudBlockBlob.cs b/src/Catalog/Persistence/ICloudBlockBlob.cs index 2166419f6..aff323fa7 100644 --- a/src/Catalog/Persistence/ICloudBlockBlob.cs +++ b/src/Catalog/Persistence/ICloudBlockBlob.cs @@ -14,6 +14,7 @@ public interface ICloudBlockBlob string ETag { get; } Uri Uri { get; } + Task ExistsAsync(CancellationToken cancellationToken); Task FetchAttributesAsync(CancellationToken cancellationToken); Task> GetMetadataAsync(CancellationToken cancellationToken); Task GetStreamAsync(CancellationToken cancellationToken); diff --git a/src/Catalog/Persistence/IStorage.cs b/src/Catalog/Persistence/IStorage.cs index dcf7a0cff..2c24dbb65 100644 --- a/src/Catalog/Persistence/IStorage.cs +++ b/src/Catalog/Persistence/IStorage.cs @@ -12,7 +12,16 @@ public interface IStorage { Uri BaseAddress { get; } + Task CopyAsync( + Uri sourceUri, + IStorage destinationStorage, + Uri destinationUri, + IReadOnlyDictionary destinationProperties, + CancellationToken cancellation); Task DeleteAsync(Uri resourceUri, CancellationToken cancellationToken); + Task GetOptimisticConcurrencyControlTokenAsync( + Uri resourceUri, + CancellationToken cancellationToken); Task> ListAsync(CancellationToken cancellationToken); Task LoadAsync(Uri resourceUri, CancellationToken cancellationToken); Task LoadStringAsync(Uri resourceUri, CancellationToken cancellationToken); diff --git a/src/Catalog/Persistence/OptimisticConcurrencyControlToken.cs b/src/Catalog/Persistence/OptimisticConcurrencyControlToken.cs new file mode 100644 index 000000000..0855ce6cd --- /dev/null +++ b/src/Catalog/Persistence/OptimisticConcurrencyControlToken.cs @@ -0,0 +1,65 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.Metadata.Catalog.Persistence +{ + public sealed class OptimisticConcurrencyControlToken : IEquatable + { + private readonly string _token; + + public static readonly OptimisticConcurrencyControlToken Null = new OptimisticConcurrencyControlToken(token: null); + + public OptimisticConcurrencyControlToken(string token) + { + _token = token; + } + + public bool Equals(OptimisticConcurrencyControlToken other) + { + var azureToken = other as OptimisticConcurrencyControlToken; + + if (azureToken == null) + { + return false; + } + + return _token == azureToken._token; + } + + public override bool Equals(object obj) + { + return Equals(obj as OptimisticConcurrencyControlToken); + } + + public override int GetHashCode() + { + return _token.GetHashCode(); + } + + public static bool operator ==( + OptimisticConcurrencyControlToken token1, + OptimisticConcurrencyControlToken token2) + { + if (((object)token1) == null || ((object)token2) == null) + { + return object.Equals(token1, token2); + } + + return token1.Equals(token2); + } + + public static bool operator !=( + OptimisticConcurrencyControlToken token1, + OptimisticConcurrencyControlToken token2) + { + if (((object)token1) == null || ((object)token2) == null) + { + return !object.Equals(token1, token2); + } + + return !token1.Equals(token2); + } + } +} \ No newline at end of file diff --git a/src/Catalog/Persistence/Storage.cs b/src/Catalog/Persistence/Storage.cs index 342abe50a..23a85279b 100644 --- a/src/Catalog/Persistence/Storage.cs +++ b/src/Catalog/Persistence/Storage.cs @@ -16,10 +16,7 @@ namespace NuGet.Services.Metadata.Catalog.Persistence public abstract class Storage : IStorage { public Uri BaseAddress { get; protected set; } - public bool Verbose { get; set; } - public int SaveCount { get; protected set; } - public int LoadCount { get; protected set; } - public int DeleteCount { get; protected set; } + public bool Verbose { get; protected set; } public Storage(Uri baseAddress) { @@ -32,21 +29,56 @@ public override string ToString() return BaseAddress.ToString(); } - protected abstract Task OnSave(Uri resourceUri, StorageContent content, CancellationToken cancellationToken); - protected abstract Task OnLoad(Uri resourceUri, CancellationToken cancellationToken); - protected abstract Task OnDelete(Uri resourceUri, CancellationToken cancellationToken); + protected abstract Task OnCopyAsync( + Uri sourceUri, + IStorage destinationStorage, + Uri destinationUri, + IReadOnlyDictionary destinationProperties, + CancellationToken cancellationToken); + protected abstract Task OnSaveAsync(Uri resourceUri, StorageContent content, CancellationToken cancellationToken); + protected abstract Task OnLoadAsync(Uri resourceUri, CancellationToken cancellationToken); + protected abstract Task OnDeleteAsync(Uri resourceUri, CancellationToken cancellationToken); + + public virtual Task GetOptimisticConcurrencyControlTokenAsync( + Uri resourceUri, + CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } - public async Task SaveAsync(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) + public async Task CopyAsync( + Uri sourceUri, + IStorage destinationStorage, + Uri destinationUri, + IReadOnlyDictionary destinationProperties, + CancellationToken cancellationToken) { - SaveCount++; + TraceMethod(nameof(CopyAsync), sourceUri); + + var stopwatch = Stopwatch.StartNew(); + + try + { + await OnCopyAsync(sourceUri, destinationStorage, destinationUri, destinationProperties, cancellationToken); + } + catch (Exception e) + { + TraceException(nameof(CopyAsync), sourceUri, e); + throw; + } + TraceExecutionTime(nameof(CopyAsync), sourceUri, stopwatch.ElapsedMilliseconds); + } + + public async Task SaveAsync(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) + { TraceMethod(nameof(SaveAsync), resourceUri); - Stopwatch sw = new Stopwatch(); - sw.Start(); + + var stopwatch = Stopwatch.StartNew(); try { - await OnSave(resourceUri, content, cancellationToken); + await OnSaveAsync(resourceUri, content, cancellationToken); } catch (Exception e) { @@ -54,22 +86,20 @@ public async Task SaveAsync(Uri resourceUri, StorageContent content, Cancellatio throw; } - sw.Stop(); - TraceExecutionTime(nameof(SaveAsync), resourceUri, sw.ElapsedMilliseconds); + TraceExecutionTime(nameof(SaveAsync), resourceUri, stopwatch.ElapsedMilliseconds); } public async Task LoadAsync(Uri resourceUri, CancellationToken cancellationToken) { - LoadCount++; StorageContent storageContent = null; TraceMethod(nameof(LoadAsync), resourceUri); - Stopwatch sw = new Stopwatch(); - sw.Start(); + + var stopwatch = Stopwatch.StartNew(); try { - storageContent = await OnLoad(resourceUri, cancellationToken); + storageContent = await OnLoadAsync(resourceUri, cancellationToken); } catch (Exception e) { @@ -77,22 +107,20 @@ public async Task LoadAsync(Uri resourceUri, CancellationToken c throw; } - sw.Stop(); - TraceExecutionTime(nameof(LoadAsync), resourceUri, sw.ElapsedMilliseconds); + TraceExecutionTime(nameof(LoadAsync), resourceUri, stopwatch.ElapsedMilliseconds); + return storageContent; } public async Task DeleteAsync(Uri resourceUri, CancellationToken cancellationToken) { - DeleteCount++; - TraceMethod(nameof(DeleteAsync), resourceUri); - Stopwatch sw = new Stopwatch(); - sw.Start(); + + var stopwatch = Stopwatch.StartNew(); try { - await OnDelete(resourceUri, cancellationToken); + await OnDeleteAsync(resourceUri, cancellationToken); } catch (StorageException e) { @@ -116,8 +144,7 @@ public async Task DeleteAsync(Uri resourceUri, CancellationToken cancellationTok throw; } - sw.Stop(); - TraceExecutionTime(nameof(DeleteAsync), resourceUri, sw.ElapsedMilliseconds); + TraceExecutionTime(nameof(DeleteAsync), resourceUri, stopwatch.ElapsedMilliseconds); } public async Task LoadStringAsync(Uri resourceUri, CancellationToken cancellationToken) @@ -141,13 +168,6 @@ public async Task LoadStringAsync(Uri resourceUri, CancellationToken can public abstract bool Exists(string fileName); - public void ResetStatistics() - { - SaveCount = 0; - LoadCount = 0; - DeleteCount = 0; - } - public Uri ResolveUri(string relativeUri) { return new Uri(BaseAddress, relativeUri); diff --git a/src/Catalog/Persistence/StorageConstants.cs b/src/Catalog/Persistence/StorageConstants.cs new file mode 100644 index 000000000..4374b678a --- /dev/null +++ b/src/Catalog/Persistence/StorageConstants.cs @@ -0,0 +1,11 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.Metadata.Catalog.Persistence +{ + public static class StorageConstants + { + public const string CacheControl = "CacheControl"; + public const string ContentType = "ContentType"; + } +} \ No newline at end of file diff --git a/src/Catalog/Persistence/StorageFactory.cs b/src/Catalog/Persistence/StorageFactory.cs index 9aa6e6842..533e437b4 100644 --- a/src/Catalog/Persistence/StorageFactory.cs +++ b/src/Catalog/Persistence/StorageFactory.cs @@ -1,5 +1,6 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + using System; namespace NuGet.Services.Metadata.Catalog.Persistence @@ -13,7 +14,7 @@ public abstract class StorageFactory : IStorageFactory // For telemetry only public Uri DestinationAddress { get; protected set; } - public bool Verbose { get; set; } + public bool Verbose { get; protected set; } public override string ToString() { diff --git a/src/Catalog/Registration/FlatContainerPackagePathProvider.cs b/src/Catalog/Registration/FlatContainerPackagePathProvider.cs index c52e5ad4e..46a58d867 100644 --- a/src/Catalog/Registration/FlatContainerPackagePathProvider.cs +++ b/src/Catalog/Registration/FlatContainerPackagePathProvider.cs @@ -13,13 +13,14 @@ public FlatContainerPackagePathProvider(string container) { _container = container; } + public string GetPackagePath(string id, string version) { - version = NuGetVersionUtility.NormalizeVersion(version); - - var packageFileName = PackageUtility.GetPackageFileNameLowercase(id, version); + var idLowerCase = id.ToLowerInvariant(); + var versionLowerCase = NuGetVersionUtility.NormalizeVersion(version).ToLowerInvariant(); + var packageFileName = PackageUtility.GetPackageFileName(idLowerCase, versionLowerCase); - return $"{_container}/{id.ToLowerInvariant()}/{version.ToLowerInvariant()}/{packageFileName}"; + return $"{_container}/{idLowerCase}/{versionLowerCase}/{packageFileName}"; } } } diff --git a/src/Catalog/Registration/PackagesFolderPackagePathProvider.cs b/src/Catalog/Registration/PackagesFolderPackagePathProvider.cs index 9b794b6db..7425f6ff4 100644 --- a/src/Catalog/Registration/PackagesFolderPackagePathProvider.cs +++ b/src/Catalog/Registration/PackagesFolderPackagePathProvider.cs @@ -9,9 +9,10 @@ public class PackagesFolderPackagePathProvider : IPackagePathProvider { public string GetPackagePath(string id, string version) { - version = NuGetVersionUtility.NormalizeVersion(version); + var idLowerCase = id.ToLowerInvariant(); + var versionLowerCase = NuGetVersionUtility.NormalizeVersion(version).ToLowerInvariant(); - var packageFileName = PackageUtility.GetPackageFileNameLowercase(id, version); + var packageFileName = PackageUtility.GetPackageFileName(idLowerCase, versionLowerCase); return $"packages/{packageFileName}"; } diff --git a/src/Catalog/Registration/RecordingStorage.cs b/src/Catalog/Registration/RecordingStorage.cs index 29b7ec391..a4455b0b4 100644 --- a/src/Catalog/Registration/RecordingStorage.cs +++ b/src/Catalog/Registration/RecordingStorage.cs @@ -24,6 +24,23 @@ public RecordingStorage(IStorage storage) public HashSet Loaded { get; private set; } public HashSet Saved { get; private set; } + public Task GetOptimisticConcurrencyControlTokenAsync( + Uri resourceUri, + CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + + public Task CopyAsync( + Uri sourceUri, + IStorage destinationStorage, + Uri destinationUri, + IReadOnlyDictionary destinationProperties, + CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + public Task SaveAsync(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) { Task result = _innerStorage.SaveAsync(resourceUri, content, cancellationToken); diff --git a/src/Catalog/Telemetry/TelemetryConstants.cs b/src/Catalog/Telemetry/TelemetryConstants.cs index 5f1092e6a..3325b0fa3 100644 --- a/src/Catalog/Telemetry/TelemetryConstants.cs +++ b/src/Catalog/Telemetry/TelemetryConstants.cs @@ -35,6 +35,7 @@ public static class TelemetryConstants public const string StatusCode = "StatusCode"; public const string Success = "Success"; public const string Uri = "Uri"; + public const string UsePackageSourceFallback = "UsePackageSourceFallback"; public const string Version = "Version"; } } \ No newline at end of file diff --git a/src/Catalog/app.config b/src/Catalog/app.config index e310fa8d7..f785a25e1 100644 --- a/src/Catalog/app.config +++ b/src/Catalog/app.config @@ -34,6 +34,10 @@ + + + + diff --git a/src/Catalog/packages.config b/src/Catalog/packages.config index c0e55404e..de74bac52 100644 --- a/src/Catalog/packages.config +++ b/src/Catalog/packages.config @@ -5,6 +5,7 @@ + diff --git a/src/Ng/Arguments.cs b/src/Ng/Arguments.cs index d9b51a7c6..6129049c2 100644 --- a/src/Ng/Arguments.cs +++ b/src/Ng/Arguments.cs @@ -51,6 +51,13 @@ public static class Arguments public const string StorageServerTimeoutInSeconds = "storageServerTimeoutInSeconds"; public const string HttpClientTimeoutInSeconds = "httpClientTimeoutInSeconds"; + public const string StorageAccountNamePreferredPackageSourceStorage = "storageAccountNamePreferredPackageSourceStorage"; + public const string StorageKeyValuePreferredPackageSourceStorage = "storageKeyValuePreferredPackageSourceStorage"; + public const string StorageContainerPreferredPackageSourceStorage = "storageContainerPreferredPackageSourceStorage"; + + public const string PreferAlternatePackageSourceStorage = "preferAlternatePackageSourceStorage"; + + public const string StorageUseServerSideCopy = "storageUseServerSideCopy"; #endregion #region Catalog2Lucene @@ -102,11 +109,7 @@ public static class Arguments public const string StorageKeyValueAuditing = "storageKeyValueAuditing"; public const string StoragePathAuditing = "storagePathAuditing"; public const string StorageTypeAuditing = "storageTypeAuditing"; - public const string StorageAccountNamePreferredPackageSourceStorage = "storageAccountNamePreferredPackageSourceStorage"; - public const string StorageKeyValuePreferredPackageSourceStorage = "storageKeyValuePreferredPackageSourceStorage"; - public const string StorageContainerPreferredPackageSourceStorage = "storageContainerPreferredPackageSourceStorage"; - public const string PreferAlternatePackageSourceStorage = "preferAlternatePackageSourceStorage"; public const string SkipCreatedPackagesProcessing = "skipCreatedPackagesProcessing"; #endregion diff --git a/src/Ng/CommandHelpers.cs b/src/Ng/CommandHelpers.cs index 8f17779f5..07c072c0f 100644 --- a/src/Ng/CommandHelpers.cs +++ b/src/Ng/CommandHelpers.cs @@ -114,10 +114,8 @@ public static RegistrationStorageFactories CreateRegistrationStorageFactories(ID var aggregateStorageFactory = new CatalogAggregateStorageFactory( storageFactory, new[] { compressedStorageFactory }, - secondaryStorageBaseUrlRewriter.Rewrite) - { - Verbose = verbose - }; + secondaryStorageBaseUrlRewriter.Rewrite, + verbose); legacyStorageFactory = aggregateStorageFactory; } @@ -139,6 +137,7 @@ public static CatalogStorageFactory CreateStorageFactory(IDictionary(argumentNameMap[Arguments.StorageSuffix]); var storageOperationMaxExecutionTime = MaxExecutionTime(arguments.GetOrDefault(argumentNameMap[Arguments.StorageOperationMaxExecutionTimeInSeconds])); var storageServerTimeout = MaxExecutionTime(arguments.GetOrDefault(argumentNameMap[Arguments.StorageServerTimeoutInSeconds])); + var storageUseServerSideCopy = arguments.GetOrDefault(argumentNameMap[Arguments.StorageUseServerSideCopy]); var credentials = new StorageCredentials(storageAccountName, storageKeyValue); var account = string.IsNullOrEmpty(storageSuffix) ? - new CloudStorageAccount(credentials, useHttps: true) : - new CloudStorageAccount(credentials, storageSuffix, useHttps: true); - return new CatalogAzureStorageFactory(account, - storageContainer, - storageOperationMaxExecutionTime, - storageServerTimeout, - storagePath, - storageBaseAddress) - { Verbose = verbose, CompressContent = compressed }; + new CloudStorageAccount(credentials, useHttps: true) : + new CloudStorageAccount(credentials, storageSuffix, useHttps: true); + + return new CatalogAzureStorageFactory( + account, + storageContainer, + storageOperationMaxExecutionTime, + storageServerTimeout, + storagePath, + storageBaseAddress, + storageUseServerSideCopy, + compressed, + verbose); } throw new ArgumentException($"Unrecognized storageType \"{storageType}\""); } diff --git a/src/Ng/Jobs/Catalog2DnxJob.cs b/src/Ng/Jobs/Catalog2DnxJob.cs index c27bc445f..316741eb1 100644 --- a/src/Ng/Jobs/Catalog2DnxJob.cs +++ b/src/Ng/Jobs/Catalog2DnxJob.cs @@ -10,6 +10,7 @@ using NuGet.Services.Configuration; using NuGet.Services.Metadata.Catalog; using NuGet.Services.Metadata.Catalog.Dnx; +using NuGet.Services.Metadata.Catalog.Persistence; namespace Ng.Jobs { @@ -45,7 +46,12 @@ public override string GetUsage() + $"[-{Arguments.Verbose} true|false] " + $"[-{Arguments.Interval} ]" + $"[-{Arguments.HttpClientTimeoutInSeconds} ]" - + $"[-{Arguments.StorageSuffix} ]"; + + $"[-{Arguments.StorageSuffix} ]" + + $"[-{Arguments.PreferAlternatePackageSourceStorage} true|false " + + $"-{Arguments.StorageAccountNamePreferredPackageSourceStorage} " + + $"-{Arguments.StorageKeyValuePreferredPackageSourceStorage} " + + $"-{Arguments.StorageContainerPreferredPackageSourceStorage} " + + $"-{Arguments.StorageUseServerSideCopy} true|false]"; } protected override void Init(IDictionary arguments, CancellationToken cancellationToken) @@ -57,20 +63,33 @@ protected override void Init(IDictionary arguments, Cancellation var httpClientTimeoutInSeconds = arguments.GetOrDefault(Arguments.HttpClientTimeoutInSeconds); var httpClientTimeout = httpClientTimeoutInSeconds.HasValue ? (TimeSpan?)TimeSpan.FromSeconds(httpClientTimeoutInSeconds.Value) : null; - Logger.LogInformation("CONFIG source: \"{ConfigSource}\" storage: \"{Storage}\"", source, storageFactory); + StorageFactory preferredPackageSourceStorageFactory = null; + IAzureStorage preferredPackageSourceStorage = null; + + var preferAlternatePackageSourceStorage = arguments.GetOrDefault(Arguments.PreferAlternatePackageSourceStorage, defaultValue: false); + + if (preferAlternatePackageSourceStorage) + { + preferredPackageSourceStorageFactory = CommandHelpers.CreateSuffixedStorageFactory("PreferredPackageSourceStorage", arguments, verbose); + preferredPackageSourceStorage = preferredPackageSourceStorageFactory.Create() as IAzureStorage; + } + + Logger.LogInformation("CONFIG source: \"{ConfigSource}\" storage: \"{Storage}\" preferred package source storage: \"{PreferredPackageSourceStorage}\"", + source, + storageFactory, + preferredPackageSourceStorageFactory); Logger.LogInformation("HTTP client timeout: {Timeout}", httpClientTimeout); _collector = new DnxCatalogCollector( new Uri(source), storageFactory, + preferredPackageSourceStorage, + contentBaseAddress == null ? null : new Uri(contentBaseAddress), TelemetryService, Logger, MaxDegreeOfParallelism, CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose), - httpClientTimeout) - { - ContentBaseAddress = contentBaseAddress == null ? null : new Uri(contentBaseAddress) - }; + httpClientTimeout); var storage = storageFactory.Create(); _front = new DurableCursor(storage.ResolveUri("cursor.json"), storage, MemoryCursor.MinValue); diff --git a/src/Ng/Scripts/Catalog2DnxV3.cmd b/src/Ng/Scripts/Catalog2DnxV3.cmd index 2e44be2d8..e9af1f345 100644 --- a/src/Ng/Scripts/Catalog2DnxV3.cmd +++ b/src/Ng/Scripts/Catalog2DnxV3.cmd @@ -15,6 +15,11 @@ start /w Ng.exe catalog2dnx ^ -storageAccountName #{Jobs.ngcatalog2dnx.StorageAccount.Name} ^ -storageKeyValue #{Jobs.ngcatalog2dnx.StorageAccount.Key} ^ -storageContainer #{Jobs.ngcatalog2dnx.Container} ^ + -preferAlternatePackageSourceStorage #{Jobs.ngcatalog2dnx.PreferAlternatePackageSourceStorage} ^ + -storageAccountNamePreferredPackageSourceStorage #{Jobs.ngcatalog2dnx.PreferredPackageSourceStorageAccountName} ^ + -storageKeyValuePreferredPackageSourceStorage #{Jobs.ngcatalog2dnx.PreferredPackageSourceStorageAccountKey} ^ + -storageContainerPreferredPackageSourceStorage #{Jobs.ngcatalog2dnx.PreferredPackageSourceStorageContainerName} ^ + -storageUseServerSideCopy #{Jobs.ngcatalog2dnx.StorageUseServerSideCopy} ^ -instrumentationkey #{Jobs.common.v3.Logging.InstrumentationKey} ^ -vaultName #{Deployment.Azure.KeyVault.VaultName} ^ -clientId #{Deployment.Azure.KeyVault.ClientId} ^ diff --git a/src/Ng/Scripts/Catalog2DnxV3China.cmd b/src/Ng/Scripts/Catalog2DnxV3China.cmd index cfb2e12a7..4d2984135 100644 --- a/src/Ng/Scripts/Catalog2DnxV3China.cmd +++ b/src/Ng/Scripts/Catalog2DnxV3China.cmd @@ -15,6 +15,11 @@ start /w Ng.exe catalog2dnx ^ -storageAccountName #{Jobs.common.China.v3.StorageAccountName} ^ -storageKeyValue #{Jobs.common.China.v3.StorageKey} ^ -storageContainer #{Jobs.ngcatalog2dnx.Container} ^ + -preferAlternatePackageSourceStorage #{Jobs.ngcatalog2dnx.PreferAlternatePackageSourceStorage} ^ + -storageAccountNamePreferredPackageSourceStorage #{Jobs.ngcatalog2dnx.PreferredPackageSourceStorageAccountName} ^ + -storageKeyValuePreferredPackageSourceStorage #{Jobs.ngcatalog2dnx.PreferredPackageSourceStorageAccountKey} ^ + -storageContainerPreferredPackageSourceStorage #{Jobs.ngcatalog2dnx.PreferredPackageSourceStorageContainerName} ^ + -storageUseServerSideCopy #{Jobs.China.ngcatalog2dnx.StorageUseServerSideCopy} ^ -instrumentationkey #{Jobs.common.v3.Logging.InstrumentationKey} ^ -vaultName #{Deployment.Azure.KeyVault.VaultName} ^ -clientId #{Deployment.Azure.KeyVault.ClientId} ^ diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj b/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj index b9643ef69..5d65dea40 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj @@ -36,14 +36,14 @@ ..\..\packages\Microsoft.Azure.KeyVault.Core.1.0.0\lib\net40\Microsoft.Azure.KeyVault.Core.dll - - ..\..\packages\Microsoft.Data.Edm.5.6.4\lib\net40\Microsoft.Data.Edm.dll + + ..\..\packages\Microsoft.Data.Edm.5.8.2\lib\net40\Microsoft.Data.Edm.dll - - ..\..\packages\Microsoft.Data.OData.5.6.4\lib\net40\Microsoft.Data.OData.dll + + ..\..\packages\Microsoft.Data.OData.5.8.2\lib\net40\Microsoft.Data.OData.dll - - ..\..\packages\Microsoft.Data.Services.Client.5.6.4\lib\net40\Microsoft.Data.Services.Client.dll + + ..\..\packages\Microsoft.Data.Services.Client.5.8.2\lib\net40\Microsoft.Data.Services.Client.dll ..\..\packages\Microsoft.Extensions.DependencyInjection.Abstractions.1.0.0\lib\netstandard1.0\Microsoft.Extensions.DependencyInjection.Abstractions.dll @@ -54,8 +54,8 @@ ..\..\packages\Microsoft.Extensions.Logging.Abstractions.1.0.0\lib\netstandard1.1\Microsoft.Extensions.Logging.Abstractions.dll - - ..\..\packages\WindowsAzure.Storage.7.1.2\lib\net40\Microsoft.WindowsAzure.Storage.dll + + ..\..\packages\WindowsAzure.Storage.8.2.1\lib\net45\Microsoft.WindowsAzure.Storage.dll ..\..\packages\Newtonsoft.Json.9.0.1\lib\net45\Newtonsoft.Json.dll @@ -115,8 +115,8 @@ - - ..\..\packages\System.Spatial.5.6.4\lib\net40\System.Spatial.dll + + ..\..\packages\System.Spatial.5.8.2\lib\net40\System.Spatial.dll diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/app.config b/src/NuGet.Services.Metadata.Catalog.Monitoring/app.config index 8da4767d3..506d0ddbe 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/app.config +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/app.config @@ -46,6 +46,10 @@ + + + + \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/packages.config b/src/NuGet.Services.Metadata.Catalog.Monitoring/packages.config index 7b4c4dea9..a2282424b 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/packages.config +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/packages.config @@ -2,9 +2,9 @@ - - - + + + @@ -28,16 +28,20 @@ + + + + - + - + \ No newline at end of file diff --git a/src/V3PerPackage/EnqueueCommand.cs b/src/V3PerPackage/EnqueueCommand.cs index 89f425d6a..8fce726a1 100644 --- a/src/V3PerPackage/EnqueueCommand.cs +++ b/src/V3PerPackage/EnqueueCommand.cs @@ -23,7 +23,8 @@ public async Task ExecuteAsync(bool restart) { var fileSystemStorage = new FileStorageFactory( new Uri("http://localhost/"), - Directory.GetCurrentDirectory()); + Directory.GetCurrentDirectory(), + verbose: false); var front = new DurableCursor( new Uri("http://localhost/cursor.json"), @@ -42,4 +43,4 @@ public async Task ExecuteAsync(bool restart) await _collector.Run(front, back, CancellationToken.None); } } -} +} \ No newline at end of file diff --git a/src/V3PerPackage/PerBatchProcessor.cs b/src/V3PerPackage/PerBatchProcessor.cs index 9d4927008..726dce597 100644 --- a/src/V3PerPackage/PerBatchProcessor.cs +++ b/src/V3PerPackage/PerBatchProcessor.cs @@ -218,20 +218,20 @@ private async Task ExecuteCatalog2DnxAsync(PerBatchContext context, IReadOnlyLis context.Process.FlatContainerStoragePath); var storageFactory = serviceProvider.GetRequiredService(); + IAzureStorage preferredPackageSourceStorage = null; var httpClientTimeout = TimeSpan.FromMinutes(10); var maxDegreeOfParallelism = ServicePointManager.DefaultConnectionLimit; var collector = new DnxCatalogCollector( catalogIndexUri, storageFactory, + preferredPackageSourceStorage, + context.Global.ContentBaseAddress, serviceProvider.GetRequiredService(), serviceProvider.GetRequiredService(), maxDegreeOfParallelism, () => serviceProvider.GetRequiredService(), - httpClientTimeout) - { - ContentBaseAddress = context.Global.ContentBaseAddress, - }; + httpClientTimeout); var lowercasePackageIds = packageContexts.Select(x => x.PackageId.ToLowerInvariant()); using (await _stringLocker.AcquireAsync(lowercasePackageIds, TimeSpan.FromMinutes(5))) diff --git a/tests/CatalogTests/CatalogTests.csproj b/tests/CatalogTests/CatalogTests.csproj index 799bf08b6..470807c27 100644 --- a/tests/CatalogTests/CatalogTests.csproj +++ b/tests/CatalogTests/CatalogTests.csproj @@ -202,7 +202,11 @@ + + + + diff --git a/tests/CatalogTests/Helpers/PackageUtilityTests.cs b/tests/CatalogTests/Helpers/PackageUtilityTests.cs index c035fd4a0..edf342f44 100644 --- a/tests/CatalogTests/Helpers/PackageUtilityTests.cs +++ b/tests/CatalogTests/Helpers/PackageUtilityTests.cs @@ -12,10 +12,10 @@ public class PackageUtilityTests [Theory] [InlineData(null)] [InlineData("")] - public void GetPackageFileNameLowercase_WhenPackageIdIsNullOrEmpty_Throws(string packageId) + public void GetPackageFileName_WhenPackageIdIsNullOrEmpty_Throws(string packageId) { var exception = Assert.Throws( - () => PackageUtility.GetPackageFileNameLowercase(packageId, packageVersion: "a")); + () => PackageUtility.GetPackageFileName(packageId, packageVersion: "a")); Assert.Equal("packageId", exception.ParamName); } @@ -23,10 +23,10 @@ public void GetPackageFileNameLowercase_WhenPackageIdIsNullOrEmpty_Throws(string [Theory] [InlineData(null)] [InlineData("")] - public void GetPackageFileNameLowercase_WhenPackageVersionIsNullOrEmpty_Throws(string packageVersion) + public void GetPackageFileName_WhenPackageVersionIsNullOrEmpty_Throws(string packageVersion) { var exception = Assert.Throws( - () => PackageUtility.GetPackageFileNameLowercase(packageId: "a", packageVersion: packageVersion)); + () => PackageUtility.GetPackageFileName(packageId: "a", packageVersion: packageVersion)); Assert.Equal("packageVersion", exception.ParamName); } @@ -34,11 +34,11 @@ public void GetPackageFileNameLowercase_WhenPackageVersionIsNullOrEmpty_Throws(s [Theory] [InlineData("a", "b")] [InlineData("A", "B")] - public void GetPackageFileNameLowercase_WithValidArguments_ReturnsFileName(string packageId, string packageVersion) + public void GetPackageFileName_WithValidArguments_ReturnsFileName(string packageId, string packageVersion) { - var packageFileName = PackageUtility.GetPackageFileNameLowercase(packageId, packageVersion); + var packageFileName = PackageUtility.GetPackageFileName(packageId, packageVersion); - Assert.Equal("a.b.nupkg", packageFileName); + Assert.Equal($"{packageId}.{packageVersion}.nupkg", packageFileName); } } } \ No newline at end of file diff --git a/tests/CatalogTests/PackageCatalogItemCreatorTests.cs b/tests/CatalogTests/PackageCatalogItemCreatorTests.cs index c3f8d2300..55471bf0d 100644 --- a/tests/CatalogTests/PackageCatalogItemCreatorTests.cs +++ b/tests/CatalogTests/PackageCatalogItemCreatorTests.cs @@ -145,13 +145,22 @@ public async Task CreateAsync_WhenHttpPackageSourceReturnsPackage_ReturnsInstanc public class WhenStorageIsIAzureStorage { [Fact] - public async Task CreateAsync_WhenAzureStorageReturnsNullBlob_ReturnsInstanceFromHttpPackageSource() + public async Task CreateAsync_WhenAzureStorageReturnsNonexistantBlob_ReturnsInstanceFromHttpPackageSource() { using (var test = new Test()) { + test.Storage.Setup(x => x.ResolveUri(test.PackageFileName)) + .Returns(test.ContentUri); + test.Storage.Setup(x => x.GetCloudBlockBlobReferenceAsync( - It.Is(name => name == test.PackageFileName))) - .ReturnsAsync((ICloudBlockBlob)null); + It.Is(uri => uri == test.ContentUri))) + .ReturnsAsync(test.Blob.Object); + + test.Blob.Setup(x => x.ExistsAsync(It.IsAny())) + .ReturnsAsync(false); + + test.Blob.SetupGet(x => x.Uri) + .Returns(test.ContentUri); var item = await test.Creator.CreateAsync( test.FeedPackageDetails, @@ -167,9 +176,10 @@ public async Task CreateAsync_WhenAzureStorageReturnsNullBlob_ReturnsInstanceFro Assert.Equal("NonExistentBlob", call.Name); Assert.Equal(1UL, call.Metric); - Assert.Equal(2, call.Properties.Count); + Assert.Equal(3, call.Properties.Count); Assert.Equal(test.PackageId.ToLowerInvariant(), call.Properties["Id"]); Assert.Equal(test.PackageVersion.ToLowerInvariant(), call.Properties["Version"]); + Assert.Equal(test.ContentUri.AbsoluteUri, call.Properties["Uri"]); } } @@ -178,10 +188,16 @@ public async Task CreateAsync_WhenBlobDoesNotHavePackageHash_ReturnsInstanceFrom { using (var test = new Test()) { + test.Storage.Setup(x => x.ResolveUri(test.PackageFileName)) + .Returns(test.ContentUri); + test.Storage.Setup(x => x.GetCloudBlockBlobReferenceAsync( - It.Is(name => name == test.PackageFileName))) + It.Is(uri => uri == test.ContentUri))) .ReturnsAsync(test.Blob.Object); + test.Blob.Setup(x => x.ExistsAsync(It.IsAny())) + .ReturnsAsync(true); + test.Blob.Setup(x => x.FetchAttributesAsync(It.IsAny())) .Returns(Task.FromResult(0)); @@ -192,7 +208,7 @@ public async Task CreateAsync_WhenBlobDoesNotHavePackageHash_ReturnsInstanceFrom .ReturnsAsync(new Dictionary()); test.Blob.SetupGet(x => x.Uri) - .Returns(test.PackageUri); + .Returns(test.ContentUri); var item = await test.Creator.CreateAsync( test.FeedPackageDetails, @@ -211,7 +227,7 @@ public async Task CreateAsync_WhenBlobDoesNotHavePackageHash_ReturnsInstanceFrom Assert.Equal(3, call.Properties.Count); Assert.Equal(test.PackageId.ToLowerInvariant(), call.Properties["Id"]); Assert.Equal(test.PackageVersion.ToLowerInvariant(), call.Properties["Version"]); - Assert.Equal(test.PackageUri.AbsoluteUri, call.Properties["Uri"]); + Assert.Equal(test.ContentUri.AbsoluteUri, call.Properties["Uri"]); } } @@ -220,10 +236,16 @@ public async Task CreateAsync_WhenBlobHasPackageHash_ReturnsInstanceFromBlobStor { using (var test = new Test()) { + test.Storage.Setup(x => x.ResolveUri(test.PackageFileName)) + .Returns(test.ContentUri); + test.Storage.Setup(x => x.GetCloudBlockBlobReferenceAsync( - It.Is(name => name == test.PackageFileName))) + It.Is(uri => uri == test.ContentUri))) .ReturnsAsync(test.Blob.Object); + test.Blob.Setup(x => x.ExistsAsync(It.IsAny())) + .ReturnsAsync(true); + test.Blob.Setup(x => x.FetchAttributesAsync(It.IsAny())) .Returns(Task.FromResult(0)); @@ -261,10 +283,16 @@ public async Task CreateAsync_WhenBlobChangesBetweenReads_ReturnsInstanceFromHtt { using (var test = new Test()) { + test.Storage.Setup(x => x.ResolveUri(test.PackageFileName)) + .Returns(test.ContentUri); + test.Storage.Setup(x => x.GetCloudBlockBlobReferenceAsync( - It.Is(name => name == test.PackageFileName))) + It.Is(uri => uri == test.ContentUri))) .ReturnsAsync(test.Blob.Object); + test.Blob.Setup(x => x.ExistsAsync(It.IsAny())) + .ReturnsAsync(true); + test.Blob.Setup(x => x.FetchAttributesAsync(It.IsAny())) .Returns(Task.FromResult(0)); @@ -285,7 +313,7 @@ public async Task CreateAsync_WhenBlobChangesBetweenReads_ReturnsInstanceFromHtt .Returns(Task.FromResult(0)); test.Blob.SetupGet(x => x.Uri) - .Returns(test.PackageUri); + .Returns(test.ContentUri); var item = await test.Creator.CreateAsync( test.FeedPackageDetails, @@ -304,7 +332,7 @@ public async Task CreateAsync_WhenBlobChangesBetweenReads_ReturnsInstanceFromHtt Assert.Equal(3, call.Properties.Count); Assert.Equal(test.PackageId.ToLowerInvariant(), call.Properties["Id"]); Assert.Equal(test.PackageVersion.ToLowerInvariant(), call.Properties["Version"]); - Assert.Equal(test.PackageUri.AbsoluteUri, call.Properties["Uri"]); + Assert.Equal(test.ContentUri.AbsoluteUri, call.Properties["Uri"]); } } } diff --git a/tests/CatalogTests/Persistence/AggregateStorageTests.cs b/tests/CatalogTests/Persistence/AggregateStorageTests.cs new file mode 100644 index 000000000..65beeda5f --- /dev/null +++ b/tests/CatalogTests/Persistence/AggregateStorageTests.cs @@ -0,0 +1,47 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using NgTests.Infrastructure; +using NuGet.Services.Metadata.Catalog.Persistence; +using Xunit; + +namespace CatalogTests.Persistence +{ + public class AggregateStorageTests + { + private readonly AggregateStorage _storage; + + public AggregateStorageTests() + { + _storage = CreateNewAggregateStorage(); + } + + [Fact] + public async Task CopyAsync_Always_Throws() + { + var destinationStorage = CreateNewAggregateStorage(); + var sourceFileUri = _storage.ResolveUri("a"); + var destinationFileUri = destinationStorage.ResolveUri("a"); + + await Assert.ThrowsAsync( + () => _storage.CopyAsync( + sourceFileUri, + destinationStorage, + destinationFileUri, + destinationProperties: null, + cancellationToken: CancellationToken.None)); + } + + private static AggregateStorage CreateNewAggregateStorage() + { + return new AggregateStorage( + new Uri("https://nuget.test"), + new MemoryStorage(), + secondaryStorage: null, + writeSecondaryStorageContentInterceptor: null); + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Persistence/AzureCloudBlockBlobTests.cs b/tests/CatalogTests/Persistence/AzureCloudBlockBlobTests.cs index 10ca0abd2..45680d5a7 100644 --- a/tests/CatalogTests/Persistence/AzureCloudBlockBlobTests.cs +++ b/tests/CatalogTests/Persistence/AzureCloudBlockBlobTests.cs @@ -11,7 +11,7 @@ using NuGet.Services.Metadata.Catalog.Persistence; using Xunit; -namespace CatalogTests +namespace CatalogTests.Persistence { public class AzureCloudBlockBlobTests { @@ -32,6 +32,19 @@ public void Constructor_WhenBlobIsNull_Throws() Assert.Equal("blob", exception.ParamName); } + [Fact] + public async Task ExistsAsync_CallsUnderlyingMethod() + { + _underlyingBlob.Setup(x => x.ExistsAsync()) + .ReturnsAsync(true); + + var blob = new AzureCloudBlockBlob(_underlyingBlob.Object); + + Assert.True(await blob.ExistsAsync(CancellationToken.None)); + + _underlyingBlob.VerifyAll(); + } + [Fact] public async Task FetchAttributesAsync_CallsUnderlyingMethod() { diff --git a/tests/CatalogTests/Persistence/FileStorageTests.cs b/tests/CatalogTests/Persistence/FileStorageTests.cs new file mode 100644 index 000000000..7c92be8f6 --- /dev/null +++ b/tests/CatalogTests/Persistence/FileStorageTests.cs @@ -0,0 +1,43 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using NuGet.Services.Metadata.Catalog.Persistence; +using Xunit; + +namespace CatalogTests.Persistence +{ + public class FileStorageTests + { + private readonly FileStorage _storage; + + public FileStorageTests() + { + _storage = CreateNewFileStorage(); + } + + [Fact] + public async Task CopyAsync_Always_Throws() + { + var destinationStorage = CreateNewFileStorage(); + var sourceFileUri = _storage.ResolveUri("a"); + var destinationFileUri = destinationStorage.ResolveUri("a"); + + await Assert.ThrowsAsync( + () => _storage.CopyAsync( + sourceFileUri, + destinationStorage, + destinationFileUri, + destinationProperties: null, + cancellationToken: CancellationToken.None)); + } + + private static FileStorage CreateNewFileStorage() + { + return new FileStorage(Path.GetTempPath(), Guid.NewGuid().ToString("N"), verbose: false); + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Persistence/OptimisticConcurrencyControlTokenTests.cs b/tests/CatalogTests/Persistence/OptimisticConcurrencyControlTokenTests.cs new file mode 100644 index 000000000..8361e0b30 --- /dev/null +++ b/tests/CatalogTests/Persistence/OptimisticConcurrencyControlTokenTests.cs @@ -0,0 +1,59 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using NuGet.Services.Metadata.Catalog.Persistence; +using Xunit; + +namespace CatalogTests.Persistence +{ + public class OptimisticConcurrencyControlTokenTests + { + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData("a")] + public void WhenTokensAreEqual_EqualityTestsSucceed(string innerToken) + { + var token1 = new OptimisticConcurrencyControlToken(innerToken); + var token2 = new OptimisticConcurrencyControlToken(innerToken); + + Assert.True(token1.Equals(token2)); + Assert.True(token1 == token2); + Assert.False(token1 != token2); + } + + [Theory] + [InlineData(null, "")] + [InlineData("", "a")] + [InlineData("a", "b")] + public void WhenTokensAreNotEqual_EqualityTestsFail(string innerToken1, string innerToken2) + { + var token1 = new OptimisticConcurrencyControlToken(innerToken1); + var token2 = new OptimisticConcurrencyControlToken(innerToken2); + + Assert.False(token1.Equals(token2)); + Assert.False(token1 == token2); + Assert.True(token1 != token2); + } + + [Fact] + public void Null_EqualsNullToken() + { + var token1 = OptimisticConcurrencyControlToken.Null; + var token2 = OptimisticConcurrencyControlToken.Null; + + Assert.True(token1.Equals(token2)); + Assert.True(token1 == token2); + Assert.False(token1 != token2); + } + + [Fact] + public void GetHashCode_EqualsInnerTokenGetHashCode() + { + var innerToken = "abc"; + var token = new OptimisticConcurrencyControlToken(innerToken); + + Assert.Equal(innerToken.GetHashCode(), token.GetHashCode()); + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Registration/RecordingStorageTests.cs b/tests/CatalogTests/Registration/RecordingStorageTests.cs new file mode 100644 index 000000000..fe92eff81 --- /dev/null +++ b/tests/CatalogTests/Registration/RecordingStorageTests.cs @@ -0,0 +1,43 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using NgTests.Infrastructure; +using NuGet.Services.Metadata.Catalog.Registration; +using Xunit; + +namespace CatalogTests.Registration +{ + public class RecordingStorageTests + { + private readonly RecordingStorage _storage; + + public RecordingStorageTests() + { + _storage = CreateNewRecordingStorage(); + } + + [Fact] + public async Task CopyAsync_Always_Throws() + { + var destinationStorage = CreateNewRecordingStorage(); + var sourceFileUri = _storage.ResolveUri("a"); + var destinationFileUri = destinationStorage.ResolveUri("a"); + + await Assert.ThrowsAsync( + () => _storage.CopyAsync( + sourceFileUri, + destinationStorage, + destinationFileUri, + destinationProperties: null, + cancellationToken: CancellationToken.None)); + } + + private static RecordingStorage CreateNewRecordingStorage() + { + return new RecordingStorage(new MemoryStorage()); + } + } +} \ No newline at end of file diff --git a/tests/NgTests/AggregateStorageTests.cs b/tests/NgTests/AggregateStorageTests.cs index c01200650..c9afe4ede 100644 --- a/tests/NgTests/AggregateStorageTests.cs +++ b/tests/NgTests/AggregateStorageTests.cs @@ -159,16 +159,9 @@ public async Task CorrectlyReplacesRegistrationBaseUrlsInSecondaryStorage() protected AggregateStorageFactory Create(MemoryStorage primaryStorage, MemoryStorage secondaryStorage, params MemoryStorage[] storages) { - var storageFactories = new List(); - storageFactories.Add(new TestStorageFactory(name => primaryStorage.WithName(name))); - storageFactories.Add(new TestStorageFactory(name => secondaryStorage.WithName(name))); - - foreach (var storage in storages) - { - storageFactories.Add(new TestStorageFactory(name => storage.WithName(name))); - } + const AggregateStorage.WriteSecondaryStorageContentInterceptor interceptor = null; - return new AggregateStorageFactory(storageFactories.First(), storageFactories.Skip(1).ToArray()); + return CreateWithInterceptor(interceptor, primaryStorage, secondaryStorage, storages); } protected AggregateStorageFactory CreateWithInterceptor(AggregateStorage.WriteSecondaryStorageContentInterceptor interceptor, MemoryStorage primaryStorage, MemoryStorage secondaryStorage, params MemoryStorage[] storages) @@ -182,7 +175,11 @@ protected AggregateStorageFactory CreateWithInterceptor(AggregateStorage.WriteSe storageFactories.Add(new TestStorageFactory(name => storage.WithName(name))); } - return new AggregateStorageFactory(storageFactories.First(), storageFactories.Skip(1).ToArray(), interceptor); + return new AggregateStorageFactory( + storageFactories.First(), + storageFactories.Skip(1).ToArray(), + interceptor, + verbose: false); } protected void AssertUriAndContentExists(MemoryStorage storage, Uri uri, string expectedContent) diff --git a/tests/NgTests/Data/Catalogs.cs b/tests/NgTests/Data/Catalogs.cs index 69bc4fc4e..56041e544 100644 --- a/tests/NgTests/Data/Catalogs.cs +++ b/tests/NgTests/Data/Catalogs.cs @@ -204,5 +204,24 @@ public static MemoryStorage CreateTestCatalogWithMultipleEntriesWithSamePackageI return catalogStorage; } + + public static MemoryStorage CreateTestCatalogWithOnePackage() + { + var catalogStorage = new MemoryStorage(); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "index.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageWithOnePackageIndex)); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "page0.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageWithOnePackagePage)); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "data/2015.10.12.10.08.54/listedpackage.1.0.0.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageWithThreePackagesListedPackage100)); + + return catalogStorage; + } } } \ No newline at end of file diff --git a/tests/NgTests/Data/TestCatalogEntries.Designer.cs b/tests/NgTests/Data/TestCatalogEntries.Designer.cs index 36da47895..2971298ba 100644 --- a/tests/NgTests/Data/TestCatalogEntries.Designer.cs +++ b/tests/NgTests/Data/TestCatalogEntries.Designer.cs @@ -192,6 +192,56 @@ public class TestCatalogEntries { } } + /// + /// Looks up a localized string similar to { + /// "@id": "http://tempuri.org/index.json", + /// "@type": [ + /// "CatalogRoot", + /// "AppendOnlyCatalog", + /// "Permalink" + /// ], + /// "commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + /// "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + /// "count": 1, + /// "items": [ + /// { + /// "@id": "http://tempuri.org/page0.json", + /// "@type": "CatalogPage", + /// "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + /// "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + /// "count": 4 + /// } + /// ], + /// "nuget:las [rest of string was truncated]";. + /// + public static string TestCatalogStorageWithOnePackageIndex { + get { + return ResourceManager.GetString("TestCatalogStorageWithOnePackageIndex", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "http://tempuri.org/page0.json", + /// "@type": "CatalogPage", + /// "commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + /// "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + /// "count": 1, + /// "items": [ + /// { + /// "@id": "http://tempuri.org/data/2015.10.12.10.08.54/listedpackage.1.0.0.json", + /// "@type": "nuget:PackageDetails", + /// "commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + /// "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + /// "nuget:id": "ListedPackage", + /// "nuget:ve [rest of string was truncated]";. + /// + public static string TestCatalogStorageWithOnePackagePage { + get { + return ResourceManager.GetString("TestCatalogStorageWithOnePackagePage", resourceCulture); + } + } + /// /// Looks up a localized string similar to { /// "@id": "http://tempuri.org/index.json", diff --git a/tests/NgTests/Data/TestCatalogEntries.resx b/tests/NgTests/Data/TestCatalogEntries.resx index b68021905..9351db962 100644 --- a/tests/NgTests/Data/TestCatalogEntries.resx +++ b/tests/NgTests/Data/TestCatalogEntries.resx @@ -468,6 +468,97 @@ "@type": "xsd:dateTime" } } +} + + + { + "@id": "http://tempuri.org/index.json", + "@type": [ + "CatalogRoot", + "AppendOnlyCatalog", + "Permalink" + ], + "commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + "count": 1, + "items": [ + { + "@id": "http://tempuri.org/page0.json", + "@type": "CatalogPage", + "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + "count": 4 + } + ], + "nuget:lastCreated": "2015-10-12T10:08:54.1506742Z", + "nuget:lastDeleted": "2015-10-12T10:08:54.1506742Z", + "nuget:lastEdited": "2015-10-12T10:08:54.1506742Z", + "@context": { + "@vocab": "http://schema.nuget.org/catalog#", + "nuget": "http://schema.nuget.org/schema#", + "items": { + "@id": "item", + "@container": "@set" + }, + "parent": { + "@type": "@id" + }, + "commitTimeStamp": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastCreated": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastEdited": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastDeleted": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + } + } +} + + + { + "@id": "http://tempuri.org/page0.json", + "@type": "CatalogPage", + "commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + "count": 1, + "items": [ + { + "@id": "http://tempuri.org/data/2015.10.12.10.08.54/listedpackage.1.0.0.json", + "@type": "nuget:PackageDetails", + "commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + "nuget:id": "ListedPackage", + "nuget:version": "1.0.0" + } + ], + "parent": "http://tempuri.org/index.json", + "@context": { + "@vocab": "http://schema.nuget.org/catalog#", + "nuget": "http://schema.nuget.org/schema#", + "items": { + "@id": "item", + "@container": "@set" + }, + "parent": { + "@type": "@id" + }, + "commitTimeStamp": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastCreated": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastEdited": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastDeleted": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + } + } } diff --git a/tests/NgTests/DnxCatalogCollectorTests.cs b/tests/NgTests/DnxCatalogCollectorTests.cs index 44e73a06d..f676964cb 100644 --- a/tests/NgTests/DnxCatalogCollectorTests.cs +++ b/tests/NgTests/DnxCatalogCollectorTests.cs @@ -28,6 +28,8 @@ public class DnxCatalogCollectorTests private const string _nuspecData = "nuspec data"; private const int _maxDegreeOfParallelism = 20; private static readonly HttpContent _noContent = new ByteArrayContent(new byte[0]); + private const IAzureStorage _nullPreferredPackageSourceStorage = null; + private static readonly Uri _contentBaseAddress = new Uri("http://tempuri.org/packages/"); private MemoryStorage _catalogToDnxStorage; private TestStorageFactory _catalogToDnxStorageFactory; @@ -50,13 +52,12 @@ public DnxCatalogCollectorTests() _target = new DnxCatalogCollector( new Uri("http://tempuri.org/index.json"), _catalogToDnxStorageFactory, - new Mock().Object, + _nullPreferredPackageSourceStorage, + _contentBaseAddress, + Mock.Of(), _logger, _maxDegreeOfParallelism, - () => _mockServer) - { - ContentBaseAddress = new Uri("http://tempuri.org/packages/") - }; + () => _mockServer); _cursorJsonUri = _catalogToDnxStorage.ResolveUri("cursor.json"); } @@ -72,6 +73,8 @@ public void Constructor_WhenIndexIsNull_Throws() () => new DnxCatalogCollector( index, new TestStorageFactory(), + _nullPreferredPackageSourceStorage, + _contentBaseAddress, Mock.Of(), Mock.Of(), maxDegreeOfParallelism: 1, @@ -93,8 +96,10 @@ public void Constructor_WhenStorageFactoryIsNull_Throws() () => new DnxCatalogCollector( new Uri("https://nuget.test"), storageFactory, - Mock.Of(), - Mock.Of(), + _nullPreferredPackageSourceStorage, + contentBaseAddress: null, + telemetryService: Mock.Of(), + logger: Mock.Of(), maxDegreeOfParallelism: 1, handlerFunc: () => clientHandler, httpClientTimeout: TimeSpan.Zero)); @@ -106,16 +111,16 @@ public void Constructor_WhenStorageFactoryIsNull_Throws() [Fact] public void Constructor_WhenTelemetryServiceIsNull_Throws() { - ITelemetryService telemetryService = null; - using (var clientHandler = new HttpClientHandler()) { var exception = Assert.Throws( () => new DnxCatalogCollector( new Uri("https://nuget.test"), new TestStorageFactory(), - telemetryService, - Mock.Of(), + _nullPreferredPackageSourceStorage, + contentBaseAddress: null, + telemetryService: null, + logger: Mock.Of(), maxDegreeOfParallelism: 1, handlerFunc: () => clientHandler, httpClientTimeout: TimeSpan.Zero)); @@ -135,6 +140,8 @@ public void Constructor_WhenLoggerIsNull_Throws() () => new DnxCatalogCollector( new Uri("https://nuget.test"), new TestStorageFactory(), + _nullPreferredPackageSourceStorage, + null, Mock.Of(), logger, maxDegreeOfParallelism: 1, @@ -156,9 +163,11 @@ public void Constructor_WhenMaxDegreeOfParallelismIsLessThanOne_Throws(int maxDe () => new DnxCatalogCollector( new Uri("https://nuget.test"), new TestStorageFactory(), - Mock.Of(), - Mock.Of(), - maxDegreeOfParallelism, + _nullPreferredPackageSourceStorage, + contentBaseAddress: null, + telemetryService: Mock.Of(), + logger: Mock.Of(), + maxDegreeOfParallelism: maxDegreeOfParallelism, handlerFunc: () => clientHandler, httpClientTimeout: TimeSpan.Zero)); @@ -173,8 +182,10 @@ public void Constructor_WhenHandlerFuncIsNull_InstantiatesClass() new DnxCatalogCollector( new Uri("https://nuget.test"), new TestStorageFactory(), - Mock.Of(), - Mock.Of(), + _nullPreferredPackageSourceStorage, + contentBaseAddress: null, + telemetryService: Mock.Of(), + logger: Mock.Of(), maxDegreeOfParallelism: 1, handlerFunc: null, httpClientTimeout: TimeSpan.Zero); @@ -188,8 +199,10 @@ public void Constructor_WhenHttpClientTimeoutIsNull_InstantiatesClass() new DnxCatalogCollector( new Uri("https://nuget.test"), new TestStorageFactory(), - Mock.Of(), - Mock.Of(), + _nullPreferredPackageSourceStorage, + contentBaseAddress: null, + telemetryService: Mock.Of(), + logger: Mock.Of(), maxDegreeOfParallelism: 1, handlerFunc: () => clientHandler, httpClientTimeout: null); @@ -385,20 +398,20 @@ public async Task Run_WithPackageCreatedThenDeleted_LeavesNoArtifacts() } [Fact] - public async Task Run_WhenPackageIsAlreadySynchronized_SkipsPackage() + public async Task Run_WithNonIAzureStorage_WhenPackageIsAlreadySynchronizedAndHasRequiredProperties_SkipsPackage() { - var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json"); - var nupkgUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.1/listedpackage.1.0.1.nupkg"); - var nuspecUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.1/listedpackage.nuspec"); - var nupkgStream = File.OpenRead("Packages\\ListedPackage.1.0.1.zip"); - var expectedNupkg = GetStreamBytes(nupkgStream); - _catalogToDnxStorage = new SynchronizedMemoryStorage(new[] { new Uri("http://tempuri.org/packages/listedpackage.1.0.1.nupkg"), }); _catalogToDnxStorageFactory = new TestStorageFactory(name => _catalogToDnxStorage.WithName(name)); + var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json"); + var nupkgUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.1/listedpackage.1.0.1.nupkg"); + var nuspecUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.1/listedpackage.nuspec"); + var nupkgStream = File.OpenRead("Packages\\ListedPackage.1.0.1.zip"); + var expectedNupkg = GetStreamBytes(nupkgStream); + await _catalogToDnxStorage.SaveAsync( new Uri("http://tempuri.org/listedpackage/index.json"), new StringStorageContent(GetExpectedIndexJsonContent("1.0.1")), @@ -407,13 +420,12 @@ public async Task Run_WhenPackageIsAlreadySynchronized_SkipsPackage() _target = new DnxCatalogCollector( new Uri("http://tempuri.org/index.json"), _catalogToDnxStorageFactory, - new Mock().Object, + _nullPreferredPackageSourceStorage, + _contentBaseAddress, + Mock.Of(), _logger, _maxDegreeOfParallelism, - () => _mockServer) - { - ContentBaseAddress = new Uri("http://tempuri.org/packages/") - }; + () => _mockServer); var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete(); await _mockServer.AddStorageAsync(catalogStorage); @@ -441,12 +453,118 @@ public async Task Run_WhenPackageIsAlreadySynchronized_SkipsPackage() } [Fact] - public async Task Run_WhenPackageIsAlreadySynchronizedButNotInIndex_ProcessesPackage() + public async Task Run_WithIAzureStorage_WhenPackageIsAlreadySynchronizedAndHasRequiredProperties_SkipsPackage() { + _catalogToDnxStorage = new AzureSynchronizedMemoryStorageStub(new[] + { + new Uri("http://tempuri.org/packages/listedpackage.1.0.0.nupkg") + }, areRequiredPropertiesPresentAsync: true); + _catalogToDnxStorageFactory = new TestStorageFactory(name => _catalogToDnxStorage.WithName(name)); + var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json"); - var nupkgUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.1/listedpackage.1.0.1.nupkg"); - var nuspecUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.1/listedpackage.nuspec"); + var nupkgUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.0/listedpackage.1.0.0.nupkg"); + var nuspecUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.0/listedpackage.nuspec"); + var nupkgStream = File.OpenRead("Packages\\ListedPackage.1.0.0.zip"); + var expectedNupkg = GetStreamBytes(nupkgStream); + + await _catalogToDnxStorage.SaveAsync( + new Uri("http://tempuri.org/listedpackage/index.json"), + new StringStorageContent(GetExpectedIndexJsonContent("1.0.0")), + CancellationToken.None); + + _target = new DnxCatalogCollector( + new Uri("http://tempuri.org/index.json"), + _catalogToDnxStorageFactory, + _nullPreferredPackageSourceStorage, + _contentBaseAddress, + Mock.Of(), + _logger, + _maxDegreeOfParallelism, + () => _mockServer); + + var catalogStorage = Catalogs.CreateTestCatalogWithOnePackage(); + await _mockServer.AddStorageAsync(catalogStorage); + + _mockServer.SetAction( + "/packages/listedpackage.1.0.0.nupkg", + request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(nupkgStream) })); + + var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); + ReadCursor back = MemoryCursor.CreateMax(); + + await _target.Run(front, back, CancellationToken.None); + + Assert.Equal(2, _catalogToDnxStorage.Content.Count); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri)); + Assert.False(_catalogToDnxStorage.Content.ContainsKey(nupkgUri)); + Assert.False(_catalogToDnxStorage.Content.ContainsKey(nuspecUri)); + Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(_cursorJsonUri)); + Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(indexJsonUri, out var indexJson)); + Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nupkgUri)); + Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nuspecUri)); + + Assert.Equal(GetExpectedIndexJsonContent("1.0.0"), Encoding.UTF8.GetString(indexJson)); + } + + [Fact] + public async Task Run_WithFakeIAzureStorage_WhenPackageIsAlreadySynchronizedButDoesNotHaveRequiredProperties_ProcessesPackage() + { + _catalogToDnxStorage = new AzureSynchronizedMemoryStorageStub(new[] + { + new Uri("http://tempuri.org/packages/listedpackage.1.0.0.nupkg") + }, areRequiredPropertiesPresentAsync: false); + _catalogToDnxStorageFactory = new TestStorageFactory(name => _catalogToDnxStorage.WithName(name)); + + var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json"); + var nupkgUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.0/listedpackage.1.0.0.nupkg"); + var nuspecUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.0/listedpackage.nuspec"); + var nupkgStream = File.OpenRead("Packages\\ListedPackage.1.0.0.zip"); + var expectedNupkg = GetStreamBytes(nupkgStream); + + await _catalogToDnxStorage.SaveAsync( + new Uri("http://tempuri.org/listedpackage/index.json"), + new StringStorageContent(GetExpectedIndexJsonContent("1.0.0")), + CancellationToken.None); + + _target = new DnxCatalogCollector( + new Uri("http://tempuri.org/index.json"), + _catalogToDnxStorageFactory, + _nullPreferredPackageSourceStorage, + _contentBaseAddress, + Mock.Of(), + _logger, + _maxDegreeOfParallelism, + () => _mockServer); + + var catalogStorage = Catalogs.CreateTestCatalogWithOnePackage(); + await _mockServer.AddStorageAsync(catalogStorage); + + _mockServer.SetAction( + "/packages/listedpackage.1.0.0.nupkg", + request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(nupkgStream) })); + + var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); + ReadCursor back = MemoryCursor.CreateMax(); + + await _target.Run(front, back, CancellationToken.None); + + Assert.Equal(4, _catalogToDnxStorage.Content.Count); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri)); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(nupkgUri)); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(nuspecUri)); + Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(_cursorJsonUri)); + Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(indexJsonUri, out var indexJson)); + Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(nupkgUri)); + Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(nuspecUri)); + + Assert.Equal(GetExpectedIndexJsonContent("1.0.0"), Encoding.UTF8.GetString(indexJson)); + } + [Fact] + public async Task Run_WhenPackageIsAlreadySynchronizedButNotInIndex_ProcessesPackage() + { _catalogToDnxStorage = new SynchronizedMemoryStorage(new[] { new Uri("http://tempuri.org/packages/listedpackage.1.0.1.nupkg"), @@ -455,16 +573,19 @@ public async Task Run_WhenPackageIsAlreadySynchronizedButNotInIndex_ProcessesPac _mockServer = new MockServerHttpClientHandler(); _mockServer.SetAction("/", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK))); + var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json"); + var nupkgUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.1/listedpackage.1.0.1.nupkg"); + var nuspecUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.1/listedpackage.nuspec"); + _target = new DnxCatalogCollector( new Uri("http://tempuri.org/index.json"), _catalogToDnxStorageFactory, - new Mock().Object, + _nullPreferredPackageSourceStorage, + _contentBaseAddress, + Mock.Of(), new Mock().Object, _maxDegreeOfParallelism, - () => _mockServer) - { - ContentBaseAddress = new Uri("http://tempuri.org/packages/") - }; + () => _mockServer); var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete(); @@ -711,11 +832,11 @@ private static MemoryStream CreateZipStreamWithEntry(string name, string content private class SynchronizedMemoryStorage : MemoryStorage { - private readonly HashSet _synchronizedUris; + protected HashSet SynchronizedUris { get; private set; } public SynchronizedMemoryStorage(IEnumerable synchronizedUris) { - _synchronizedUris = new HashSet(synchronizedUris); + SynchronizedUris = new HashSet(synchronizedUris); } protected SynchronizedMemoryStorage( @@ -725,12 +846,12 @@ public SynchronizedMemoryStorage(IEnumerable synchronizedUris) HashSet synchronizedUris) : base(baseAddress, content, contentBytes) { - _synchronizedUris = synchronizedUris; + SynchronizedUris = synchronizedUris; } public override Task AreSynchronized(Uri firstResourceUri, Uri secondResourceUri) { - return Task.FromResult(_synchronizedUris.Contains(firstResourceUri)); + return Task.FromResult(SynchronizedUris.Contains(firstResourceUri)); } public override Storage WithName(string name) @@ -739,7 +860,56 @@ public override Storage WithName(string name) new Uri(BaseAddress + name), Content, ContentBytes, - _synchronizedUris); + SynchronizedUris); + } + } + + private class AzureSynchronizedMemoryStorageStub : SynchronizedMemoryStorage, IAzureStorage + { + private readonly bool _areRequiredPropertiesPresentAsync; + + internal AzureSynchronizedMemoryStorageStub( + IEnumerable synchronizedUris, + bool areRequiredPropertiesPresentAsync) + : base(synchronizedUris) + { + _areRequiredPropertiesPresentAsync = areRequiredPropertiesPresentAsync; + } + + protected AzureSynchronizedMemoryStorageStub( + Uri baseAddress, + ConcurrentDictionary content, + ConcurrentDictionary contentBytes, + HashSet synchronizedUris, + bool areRequiredPropertiesPresentAsync) + : base(baseAddress, content, contentBytes, synchronizedUris) + { + _areRequiredPropertiesPresentAsync = areRequiredPropertiesPresentAsync; + } + + public override Storage WithName(string name) + { + return new AzureSynchronizedMemoryStorageStub( + new Uri(BaseAddress + name), + Content, + ContentBytes, + SynchronizedUris, + _areRequiredPropertiesPresentAsync); + } + + public Task GetCloudBlockBlobReferenceAsync(Uri blobUri) + { + throw new NotImplementedException(); + } + + public Task GetCloudBlockBlobReferenceAsync(string name) + { + throw new NotImplementedException(); + } + + public Task HasPropertiesAsync(Uri blobUri, string contentType, string cacheControl) + { + return Task.FromResult(_areRequiredPropertiesPresentAsync); } } } diff --git a/tests/NgTests/DnxMakerTests.cs b/tests/NgTests/DnxMakerTests.cs index 6fff9092b..f0dc73792 100644 --- a/tests/NgTests/DnxMakerTests.cs +++ b/tests/NgTests/DnxMakerTests.cs @@ -2,15 +2,18 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Text; using System.Threading; using System.Threading.Tasks; +using Moq; using Newtonsoft.Json.Linq; using NgTests.Infrastructure; using NuGet.Services.Metadata.Catalog.Dnx; using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Persistence; using NuGet.Versioning; using Xunit; @@ -18,6 +21,11 @@ namespace NgTests { public class DnxMakerTests { + private const string _expectedCacheControl = "max-age=120"; + private const string _expectedNuspecContentType = "text/xml"; + private const string _expectedPackageContentType = "application/octet-stream"; + private const string _expectedPackageVersionIndexJsonCacheControl = "no-store"; + private const string _expectedPackageVersionIndexJsonContentType = "application/json"; private const string _packageId = "testid"; private const string _nupkgData = "nupkg data"; private const string _nuspecData = "nuspec data"; @@ -174,8 +182,8 @@ public async Task AddPackageAsync_WhenNupkgStreamIsNull_Throws() () => maker.AddPackageAsync( nupkgStream: null, nuspec: "a", - id: "b", - version: "c", + packageId: "b", + normalizedPackageVersion: "c", cancellationToken: CancellationToken.None)); Assert.Equal("nupkgStream", exception.ParamName); @@ -192,8 +200,8 @@ public async Task AddPackageAsync_WhenNuspecIsNullOrEmpty_Throws(string nuspec) () => maker.AddPackageAsync( Stream.Null, nuspec, - id: "a", - version: "b", + packageId: "a", + normalizedPackageVersion: "b", cancellationToken: CancellationToken.None)); Assert.Equal("nuspec", exception.ParamName); @@ -203,7 +211,7 @@ public async Task AddPackageAsync_WhenNuspecIsNullOrEmpty_Throws(string nuspec) [Theory] [InlineData(null)] [InlineData("")] - public async Task AddPackageAsync_WhenIdIsNullOrEmpty_Throws(string id) + public async Task AddPackageAsync_WhenPackageIdIsNullOrEmpty_Throws(string packageId) { var maker = CreateDnxMaker(); @@ -211,18 +219,18 @@ public async Task AddPackageAsync_WhenIdIsNullOrEmpty_Throws(string id) () => maker.AddPackageAsync( Stream.Null, nuspec: "a", - id: id, - version: "b", + packageId: packageId, + normalizedPackageVersion: "b", cancellationToken: CancellationToken.None)); - Assert.Equal("id", exception.ParamName); + Assert.Equal("packageId", exception.ParamName); Assert.StartsWith("The argument must not be null or empty.", exception.Message); } [Theory] [InlineData(null)] [InlineData("")] - public async Task AddPackageAsync_WhenVersionIsNullOrEmpty_Throws(string version) + public async Task AddPackageAsync_WhenNormalizedPackageVersionIsNullOrEmpty_Throws(string normalizedPackageVersion) { var maker = CreateDnxMaker(); @@ -230,11 +238,11 @@ public async Task AddPackageAsync_WhenVersionIsNullOrEmpty_Throws(string version () => maker.AddPackageAsync( Stream.Null, nuspec: "a", - id: "b", - version: version, + packageId: "b", + normalizedPackageVersion: normalizedPackageVersion, cancellationToken: CancellationToken.None)); - Assert.Equal("version", exception.ParamName); + Assert.Equal("normalizedPackageVersion", exception.ParamName); Assert.StartsWith("The argument must not be null or empty.", exception.Message); } @@ -247,8 +255,8 @@ public async Task AddPackageAsync_WhenCancelled_Throws() () => maker.AddPackageAsync( Stream.Null, nuspec: "a", - id: "b", - version: "c", + packageId: "b", + normalizedPackageVersion: "c", cancellationToken: new CancellationToken(canceled: true))); } @@ -274,13 +282,133 @@ public async Task AddPackageAsync_WithValidVersion_PopulatesStorageWithNupkgAndN Assert.Equal(2, catalogToDnxStorage.Content.Count); Assert.Equal(2, storageForPackage.Content.Count); - Verify(catalogToDnxStorage, expectedNupkg, _nupkgData); - Verify(catalogToDnxStorage, expectedNuspec, _nuspecData); - Verify(storageForPackage, expectedNupkg, _nupkgData); - Verify(storageForPackage, expectedNuspec, _nuspecData); + Verify(catalogToDnxStorage, expectedNupkg, _nupkgData, _expectedCacheControl, _expectedPackageContentType); + Verify(catalogToDnxStorage, expectedNuspec, _nuspecData, _expectedCacheControl, _expectedNuspecContentType); + Verify(storageForPackage, expectedNupkg, _nupkgData, _expectedCacheControl, _expectedPackageContentType); + Verify(storageForPackage, expectedNuspec, _nuspecData, _expectedCacheControl, _expectedNuspecContentType); } } + [Fact] + public async Task AddPackageAsync_WithStorage_WhenSourceStorageIsNull_Throws() + { + var maker = CreateDnxMaker(); + + var exception = await Assert.ThrowsAsync( + () => maker.AddPackageAsync( + sourceStorage: null, + nuspec: "a", + packageId: "b", + normalizedPackageVersion: "c", + cancellationToken: CancellationToken.None)); + + Assert.Equal("sourceStorage", exception.ParamName); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public async Task AddPackageAsync_WithStorage_WhenNuspecIsNullOrEmpty_Throws(string nuspec) + { + var maker = CreateDnxMaker(); + + var exception = await Assert.ThrowsAsync( + () => maker.AddPackageAsync( + Mock.Of(), + nuspec, + packageId: "a", + normalizedPackageVersion: "b", + cancellationToken: CancellationToken.None)); + + Assert.Equal("nuspec", exception.ParamName); + Assert.StartsWith("The argument must not be null or empty.", exception.Message); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public async Task AddPackageAsync_WithStorage_WhenPackageIdIsNullOrEmpty_Throws(string id) + { + var maker = CreateDnxMaker(); + + var exception = await Assert.ThrowsAsync( + () => maker.AddPackageAsync( + Mock.Of(), + nuspec: "a", + packageId: id, + normalizedPackageVersion: "b", + cancellationToken: CancellationToken.None)); + + Assert.Equal("packageId", exception.ParamName); + Assert.StartsWith("The argument must not be null or empty.", exception.Message); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public async Task AddPackageAsync_WithStorage_WhenNormalizedPackageVersionIsNullOrEmpty_Throws(string version) + { + var maker = CreateDnxMaker(); + + var exception = await Assert.ThrowsAsync( + () => maker.AddPackageAsync( + Mock.Of(), + nuspec: "a", + packageId: "b", + normalizedPackageVersion: version, + cancellationToken: CancellationToken.None)); + + Assert.Equal("normalizedPackageVersion", exception.ParamName); + Assert.StartsWith("The argument must not be null or empty.", exception.Message); + } + + [Fact] + public async Task AddPackageAsync_WithStorage_WhenCancelled_Throws() + { + var maker = CreateDnxMaker(); + + await Assert.ThrowsAsync( + () => maker.AddPackageAsync( + Mock.Of(), + nuspec: "a", + packageId: "b", + normalizedPackageVersion: "c", + cancellationToken: new CancellationToken(canceled: true))); + } + + [Theory] + [MemberData(nameof(PackageVersions))] + public async Task AddPackageAsync_WithStorage_WithIStorage_PopulatesStorageWithNupkgAndNuspec(string version) + { + var catalogToDnxStorage = new AzureStorageStub(); + var catalogToDnxStorageFactory = new TestStorageFactory(name => catalogToDnxStorage.WithName(name)); + var maker = new DnxMaker(catalogToDnxStorageFactory); + var normalizedVersion = NuGetVersionUtility.NormalizeVersion(version); + var sourceStorage = new AzureStorageStub(); + + var dnxEntry = await maker.AddPackageAsync( + sourceStorage, + _nuspecData, + _packageId, + normalizedVersion, + CancellationToken.None); + + var expectedNuspecUri = new Uri($"{catalogToDnxStorage.BaseAddress}{_packageId}/{normalizedVersion}/{_packageId}.nuspec"); + var expectedNupkgUri = new Uri($"{catalogToDnxStorage.BaseAddress}{_packageId}/{normalizedVersion}/{_packageId}.{normalizedVersion}.nupkg"); + var expectedSourceUri = new Uri(sourceStorage.BaseAddress, $"{_packageId}.{normalizedVersion}.nupkg"); + var storageForPackage = (MemoryStorage)catalogToDnxStorageFactory.Create(_packageId); + + Assert.Equal(expectedNuspecUri, dnxEntry.Nuspec); + Assert.Equal(expectedNupkgUri, dnxEntry.Nupkg); + Assert.Equal(2, catalogToDnxStorage.Content.Count); + Assert.Equal(2, storageForPackage.Content.Count); + + Verify(catalogToDnxStorage, expectedNupkgUri, expectedSourceUri.AbsoluteUri, _expectedCacheControl, _expectedPackageContentType); + Verify(catalogToDnxStorage, expectedNuspecUri, _nuspecData, _expectedCacheControl, _expectedNuspecContentType); + Verify(storageForPackage, expectedNupkgUri, expectedSourceUri.AbsoluteUri, _expectedCacheControl, _expectedPackageContentType); + Verify(storageForPackage, expectedNuspecUri, _nuspecData, _expectedCacheControl, _expectedNuspecContentType); + } + [Theory] [InlineData(null)] [InlineData("")] @@ -415,8 +543,8 @@ public async Task UpdatePackageVersionIndexAsync_WithValidVersion_CreatesIndex(s Assert.Equal(1, catalogToDnxStorage.Content.Count); Assert.Equal(1, storageForPackage.Content.Count); - Verify(catalogToDnxStorage, indexJsonUri, expectedContent); - Verify(storageForPackage, indexJsonUri, expectedContent); + Verify(catalogToDnxStorage, indexJsonUri, expectedContent, _expectedPackageVersionIndexJsonCacheControl, _expectedPackageVersionIndexJsonContentType); + Verify(storageForPackage, indexJsonUri, expectedContent, _expectedPackageVersionIndexJsonCacheControl, _expectedPackageVersionIndexJsonContentType); Assert.Equal(new[] { normalizedVersion }, versions); } @@ -548,9 +676,17 @@ private static string GetExpectedIndexJsonContent(string version) return $"{{\r\n \"versions\": [\r\n \"{version}\"\r\n ]\r\n}}"; } - private static void Verify(MemoryStorage storage, Uri uri, string expectedContent) + private static void Verify( + MemoryStorage storage, + Uri uri, + string expectedContent, + string expectedCacheControl, + string expectedContentType) { - Assert.True(storage.Content.ContainsKey(uri)); + Assert.True(storage.Content.TryGetValue(uri, out var content)); + Assert.Equal(expectedCacheControl, content.CacheControl); + Assert.Equal(expectedContentType, content.ContentType); + Assert.True(storage.ContentBytes.TryGetValue(uri, out var bytes)); Assert.Equal(Encoding.UTF8.GetBytes(expectedContent), bytes); @@ -567,5 +703,65 @@ private static void Verify(MemoryStorage storage, Uri uri, string expectedConten Assert.InRange(list.LastModifiedUtc.Value, utc.AddMinutes(-1), utc); } } + + private sealed class AzureStorageStub : MemoryStorage, IAzureStorage + { + internal AzureStorageStub() + { + } + + private AzureStorageStub( + Uri baseAddress, + ConcurrentDictionary content, + ConcurrentDictionary contentBytes) + : base(baseAddress, content, contentBytes) + { + } + + public override Storage WithName(string name) + { + return new AzureStorageStub( + new Uri(BaseAddress + name), + Content, + ContentBytes); + } + + public Task GetCloudBlockBlobReferenceAsync(Uri blobUri) + { + throw new NotImplementedException(); + } + + public Task GetCloudBlockBlobReferenceAsync(string name) + { + throw new NotImplementedException(); + } + + public Task HasPropertiesAsync(Uri blobUri, string contentType, string cacheControl) + { + throw new NotImplementedException(); + } + + protected override Task OnCopyAsync( + Uri sourceUri, + IStorage destinationStorage, + Uri destinationUri, + IReadOnlyDictionary destinationProperties, + CancellationToken cancellationToken) + { + var destinationMemoryStorage = (AzureStorageStub)destinationStorage; + + string cacheControl = null; + string contentType = null; + + destinationProperties?.TryGetValue(StorageConstants.CacheControl, out cacheControl); + destinationProperties?.TryGetValue(StorageConstants.ContentType, out contentType); + + destinationMemoryStorage.Content[destinationUri] = new StringStorageContent(sourceUri.AbsoluteUri, contentType, cacheControl); + destinationMemoryStorage.ContentBytes[destinationUri] = Encoding.UTF8.GetBytes(sourceUri.AbsoluteUri); + destinationMemoryStorage.ListMock[destinationUri] = new StorageListItem(destinationUri, DateTime.UtcNow); + + return Task.FromResult(0); + } + } } } \ No newline at end of file diff --git a/tests/NgTests/Infrastructure/MemoryStorage.cs b/tests/NgTests/Infrastructure/MemoryStorage.cs index e29bd8ecc..d986b67b9 100644 --- a/tests/NgTests/Infrastructure/MemoryStorage.cs +++ b/tests/NgTests/Infrastructure/MemoryStorage.cs @@ -12,8 +12,7 @@ namespace NgTests.Infrastructure { - public class MemoryStorage - : Storage + public class MemoryStorage : Storage { public ConcurrentDictionary Content { get; } @@ -50,6 +49,11 @@ public MemoryStorage(Uri baseAddress) } } + public override Task GetOptimisticConcurrencyControlTokenAsync(Uri resourceUri, CancellationToken cancellationToken) + { + return Task.FromResult(OptimisticConcurrencyControlToken.Null); + } + private static StorageListItem CreateStorageListItem(Uri uri) { return new StorageListItem(uri, DateTime.UtcNow); @@ -60,7 +64,17 @@ public virtual Storage WithName(string name) return new MemoryStorage(new Uri(BaseAddress + name), Content, ContentBytes); } - protected override async Task OnSave(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) + protected override Task OnCopyAsync( + Uri sourceUri, + IStorage destinationStorage, + Uri destinationUri, + IReadOnlyDictionary destinationProperties, + CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + + protected override async Task OnSaveAsync(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) { Content[resourceUri] = content; @@ -77,7 +91,7 @@ protected override async Task OnSave(Uri resourceUri, StorageContent content, Ca ListMock[resourceUri] = CreateStorageListItem(resourceUri); } - protected override Task OnLoad(Uri resourceUri, CancellationToken cancellationToken) + protected override Task OnLoadAsync(Uri resourceUri, CancellationToken cancellationToken) { StorageContent content; @@ -86,7 +100,7 @@ protected override Task OnLoad(Uri resourceUri, CancellationToke return Task.FromResult(content); } - protected override Task OnDelete(Uri resourceUri, CancellationToken cancellationToken) + protected override Task OnDeleteAsync(Uri resourceUri, CancellationToken cancellationToken) { Content.TryRemove(resourceUri, out var content); ContentBytes.TryRemove(resourceUri, out var contentBytes); From a6f8796859d3f44f0fadff4890ceed35df93c45b Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Wed, 22 Aug 2018 16:42:23 -0700 Subject: [PATCH 09/15] Functional tests should not be dependent on a specific hash and file size of the test package (#343) --- .../V2SearchFunctionalTests.cs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/BasicSearchTests.FunctionalTests.Core/V2SearchFunctionalTests.cs b/tests/BasicSearchTests.FunctionalTests.Core/V2SearchFunctionalTests.cs index 81053fd33..5455532eb 100644 --- a/tests/BasicSearchTests.FunctionalTests.Core/V2SearchFunctionalTests.cs +++ b/tests/BasicSearchTests.FunctionalTests.Core/V2SearchFunctionalTests.cs @@ -125,9 +125,18 @@ public async Task EnsureTestPackageIsValid() // Assert.True(package.FlattenedDependencies == ""); TODO: Reenable this when catalog2lucene has been fixed, tracked by Gallery issue #4029 - Loic Sharma @ 5/30/2017 Assert.True(package.Dependencies.Count() == 0); Assert.True(package.SupportedFrameworks.Count() == 0); - Assert.True(package.Hash == "5KJqge5+IYZkmba5C/pRVwjqwwaF1YM28xs6AiWMoxfxE/dzFVXJ5QGR7Rx2JmKWPLwz0R3eO+jWjd4lRX1WxA=="); + + try + { + Convert.FromBase64String(package.Hash); + } + catch (Exception e) + { + throw new ArgumentException("Package hash should be a valid Base64 encoded string!", e); + } + Assert.True(package.HashAlgorithm == "SHA512"); - Assert.True(package.PackageFileSize == 3943); + Assert.True(package.PackageFileSize > 0); Assert.True(package.RequireslicenseAcceptance == false); } From 691fd7d0ffed6da4a06fef703f4ba46a3053ea04 Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Fri, 17 Aug 2018 11:47:38 -0700 Subject: [PATCH 10/15] Add MicroBuild dependency and signing of output DLLs (#345) Progress on https://github.com/NuGet/Engineering/issues/1644 --- build.ps1 | 2 +- .../NuGet.ApplicationInsights.Owin.csproj | 11 + .../packages.config | 1 + src/NuGet.Indexing/NuGet.Indexing.csproj | 17 +- src/NuGet.Indexing/packages.config | 1 + .../NuGet.Services.BasicSearch.csproj | 5 + .../packages.config | 1 + .../NuGet.Services.Metadata.csproj | 217 ------------------ src/NuGet.Services.Metadata/Web.config | 100 -------- src/NuGet.Services.Metadata/packages.config | 25 -- 10 files changed, 31 insertions(+), 349 deletions(-) delete mode 100644 src/NuGet.Services.Metadata/NuGet.Services.Metadata.csproj delete mode 100644 src/NuGet.Services.Metadata/Web.config delete mode 100644 src/NuGet.Services.Metadata/packages.config diff --git a/build.ps1 b/build.ps1 index fcf749c50..4c68bb591 100644 --- a/build.ps1 +++ b/build.ps1 @@ -9,7 +9,7 @@ param ( [string]$SemanticVersion = '1.0.0-zlocal', [string]$Branch, [string]$CommitSHA, - [string]$BuildBranch = '37ff6e758c38b3f513af39f881399ce85f4ff20b' + [string]$BuildBranch = 'aebd86ceb96cf777b122991459838b7fc9d5a7bb' ) # For TeamCity - If any issue occurs, this script fails the build. - By default, TeamCity returns an exit code of 0 for all powershell scripts, even if they fail diff --git a/src/NuGet.ApplicationInsights.Owin/NuGet.ApplicationInsights.Owin.csproj b/src/NuGet.ApplicationInsights.Owin/NuGet.ApplicationInsights.Owin.csproj index 174f11091..7627ac366 100644 --- a/src/NuGet.ApplicationInsights.Owin/NuGet.ApplicationInsights.Owin.csproj +++ b/src/NuGet.ApplicationInsights.Owin/NuGet.ApplicationInsights.Owin.csproj @@ -1,5 +1,6 @@  + Debug @@ -80,11 +81,21 @@ $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) + + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + ..\..\build $(BUILD_SOURCESDIRECTORY)\build $(NuGetBuildPath) + + \ No newline at end of file diff --git a/src/NuGet.ApplicationInsights.Owin/packages.config b/src/NuGet.ApplicationInsights.Owin/packages.config index d982174b3..06fdac5c7 100644 --- a/src/NuGet.ApplicationInsights.Owin/packages.config +++ b/src/NuGet.ApplicationInsights.Owin/packages.config @@ -1,5 +1,6 @@  + diff --git a/src/NuGet.Indexing/NuGet.Indexing.csproj b/src/NuGet.Indexing/NuGet.Indexing.csproj index 744c15091..d5584f388 100644 --- a/src/NuGet.Indexing/NuGet.Indexing.csproj +++ b/src/NuGet.Indexing/NuGet.Indexing.csproj @@ -1,5 +1,6 @@  + Debug @@ -364,17 +365,21 @@ - - ..\..\build - $(BUILD_SOURCESDIRECTORY)\build - $(NuGetBuildPath) - - This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) + + + + \ No newline at end of file diff --git a/src/NuGet.Indexing/packages.config b/src/NuGet.Indexing/packages.config index 310bc6fe9..d870a5200 100644 --- a/src/NuGet.Indexing/packages.config +++ b/src/NuGet.Indexing/packages.config @@ -5,6 +5,7 @@ + diff --git a/src/NuGet.Services.BasicSearch/NuGet.Services.BasicSearch.csproj b/src/NuGet.Services.BasicSearch/NuGet.Services.BasicSearch.csproj index 4b9213138..7059bd5e9 100644 --- a/src/NuGet.Services.BasicSearch/NuGet.Services.BasicSearch.csproj +++ b/src/NuGet.Services.BasicSearch/NuGet.Services.BasicSearch.csproj @@ -1,5 +1,6 @@  + @@ -405,6 +406,8 @@ + + ..\..\build @@ -412,4 +415,6 @@ $(NuGetBuildPath) + + \ No newline at end of file diff --git a/src/NuGet.Services.BasicSearch/packages.config b/src/NuGet.Services.BasicSearch/packages.config index 77b2824c1..4e748ceed 100644 --- a/src/NuGet.Services.BasicSearch/packages.config +++ b/src/NuGet.Services.BasicSearch/packages.config @@ -2,6 +2,7 @@ + diff --git a/src/NuGet.Services.Metadata/NuGet.Services.Metadata.csproj b/src/NuGet.Services.Metadata/NuGet.Services.Metadata.csproj deleted file mode 100644 index 375d78450..000000000 --- a/src/NuGet.Services.Metadata/NuGet.Services.Metadata.csproj +++ /dev/null @@ -1,217 +0,0 @@ - - - - - Debug - AnyCPU - - - 2.0 - {BBA2FEAE-6872-4B3D-A4EA-CED639618426} - {349c5851-65df-11da-9384-00065b846f21};{fae04ec0-301f-11d3-bf4b-00c04f79efbc} - Library - Properties - NuGet.Services.Metadata - NuGet.Services.Metadata - v4.5.2 - true - 44333 - enabled - disabled - false - - - - - true - full - false - bin\ - DEBUG;TRACE - prompt - 4 - - - pdbonly - true - bin\ - TRACE - prompt - 4 - - - - ..\..\packages\SharpZipLib.0.86.0\lib\20\ICSharpCode.SharpZipLib.dll - True - - - ..\..\packages\Lucene.Net.3.0.3\lib\NET40\Lucene.Net.dll - True - - - - ..\..\packages\Microsoft.Data.Edm.5.6.4\lib\net40\Microsoft.Data.Edm.dll - True - - - ..\..\packages\Microsoft.Data.OData.5.6.4\lib\net40\Microsoft.Data.OData.dll - True - - - ..\..\packages\Microsoft.Data.Services.Client.5.6.4\lib\net40\Microsoft.Data.Services.Client.dll - True - - - ..\..\packages\Microsoft.Owin.3.0.1\lib\net45\Microsoft.Owin.dll - True - - - ..\..\packages\Microsoft.Owin.Diagnostics.3.0.1\lib\net45\Microsoft.Owin.Diagnostics.dll - True - - - ..\..\packages\Microsoft.Owin.FileSystems.3.0.1\lib\net45\Microsoft.Owin.FileSystems.dll - True - - - ..\..\packages\Microsoft.Owin.Host.SystemWeb.3.0.1\lib\net45\Microsoft.Owin.Host.SystemWeb.dll - True - - - ..\..\packages\Microsoft.Owin.Hosting.3.0.1\lib\net45\Microsoft.Owin.Hosting.dll - True - - - ..\..\packages\Microsoft.Owin.Security.3.0.1\lib\net45\Microsoft.Owin.Security.dll - True - - - ..\..\packages\Microsoft.Owin.Security.ActiveDirectory.3.0.1\lib\net45\Microsoft.Owin.Security.ActiveDirectory.dll - True - - - ..\..\packages\Microsoft.Owin.Security.Jwt.3.0.1\lib\net45\Microsoft.Owin.Security.Jwt.dll - True - - - ..\..\packages\Microsoft.Owin.Security.OAuth.3.0.1\lib\net45\Microsoft.Owin.Security.OAuth.dll - True - - - ..\..\packages\Microsoft.Owin.StaticFiles.3.0.1\lib\net45\Microsoft.Owin.StaticFiles.dll - True - - - ..\..\packages\Microsoft.WindowsAzure.ConfigurationManager.3.1.0\lib\net40\Microsoft.WindowsAzure.Configuration.dll - True - - - True - - - False - - - ..\..\packages\WindowsAzure.Storage.4.3.0\lib\net40\Microsoft.WindowsAzure.Storage.dll - True - - - ..\..\packages\newtonsoft.json.6.0.8\lib\net45\Newtonsoft.Json.dll - True - - - ..\..\packages\Owin.1.0\lib\net40\Owin.dll - True - - - - - ..\..\packages\System.IdentityModel.Tokens.Jwt.4.0.2.206221351\lib\net45\System.IdentityModel.Tokens.Jwt.dll - True - - - - - ..\..\packages\System.Spatial.5.6.4\lib\net40\System.Spatial.dll - True - - - - - - - - - - - - - - - - - - - - - - - - - - - Web.config - - - Web.config - - - - - - - - - - - - - - - - {ddb34145-870f-42c3-9663-a9390cee1e35} - NuGet.Indexing - - - - 10.0 - $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) - - - - - - - - - True - True - 22767 - / - https://localhost:44334/ - False - False - - - False - - - - - - ..\..\build - $(BUILD_SOURCESDIRECTORY)\build - $(NuGetBuildPath) - - - \ No newline at end of file diff --git a/src/NuGet.Services.Metadata/Web.config b/src/NuGet.Services.Metadata/Web.config deleted file mode 100644 index fb7095979..000000000 --- a/src/NuGet.Services.Metadata/Web.config +++ /dev/null @@ -1,100 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/src/NuGet.Services.Metadata/packages.config b/src/NuGet.Services.Metadata/packages.config deleted file mode 100644 index 1363fa575..000000000 --- a/src/NuGet.Services.Metadata/packages.config +++ /dev/null @@ -1,25 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file From b2a7b2e796aec57c0d7955ffec2e0c448472cc5f Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Thu, 23 Aug 2018 16:39:12 -0700 Subject: [PATCH 11/15] Rewrite SafeRoleEnvironment to not use Assembly.LoadWithPartialName (#339) --- ...nvironmentSettingsConfigurationProvider.cs | 6 +- .../DeploymentIdContextInitializer.cs | 4 +- .../SafeRoleEnvironment.cs | 89 +++++++------------ src/NuGet.Services.BasicSearch/Startup.cs | 16 ++-- 4 files changed, 44 insertions(+), 71 deletions(-) diff --git a/src/NuGet.Services.BasicSearch/Configuration/EnvironmentSettingsConfigurationProvider.cs b/src/NuGet.Services.BasicSearch/Configuration/EnvironmentSettingsConfigurationProvider.cs index c8a386d3b..6d73fb84f 100644 --- a/src/NuGet.Services.BasicSearch/Configuration/EnvironmentSettingsConfigurationProvider.cs +++ b/src/NuGet.Services.BasicSearch/Configuration/EnvironmentSettingsConfigurationProvider.cs @@ -24,10 +24,10 @@ private static string GetEnvironmentSettingValue(string key) // Get value from Cloud Services (if it throws, just ignore) try { - if (SafeRoleEnvironment.IsAvailable) + var cloudKey = key.Replace(':', '-'); // no ':' supported in cloud services + if (SafeRoleEnvironment.TryGetConfigurationSettingValue(cloudKey, out var configValue)) { - var cloudKey = key.Replace(':', '-'); // no ':' supported in cloud services - return SafeRoleEnvironment.GetConfigurationSettingValue(cloudKey); + return configValue; } } catch diff --git a/src/NuGet.Services.BasicSearch/DeploymentIdContextInitializer.cs b/src/NuGet.Services.BasicSearch/DeploymentIdContextInitializer.cs index 597e69a20..79cd07105 100644 --- a/src/NuGet.Services.BasicSearch/DeploymentIdContextInitializer.cs +++ b/src/NuGet.Services.BasicSearch/DeploymentIdContextInitializer.cs @@ -12,9 +12,9 @@ public class DeploymentIdTelemetryInitializer : ITelemetryInitializer public void Initialize(ITelemetry telemetry) { - if (SafeRoleEnvironment.IsAvailable) + if (SafeRoleEnvironment.TryGetDeploymentId(out var id)) { - telemetry.Context.Properties[DeploymentId] = SafeRoleEnvironment.GetDeploymentId(); + telemetry.Context.Properties[DeploymentId] = id; } } } diff --git a/src/NuGet.Services.BasicSearch/SafeRoleEnvironment.cs b/src/NuGet.Services.BasicSearch/SafeRoleEnvironment.cs index 58feb9c59..4d147012d 100644 --- a/src/NuGet.Services.BasicSearch/SafeRoleEnvironment.cs +++ b/src/NuGet.Services.BasicSearch/SafeRoleEnvironment.cs @@ -1,93 +1,66 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + using Microsoft.WindowsAzure.ServiceRuntime; using System; using System.IO; -using System.Reflection; namespace NuGet.Services.BasicSearch { /// - /// Safe role environment which allows this application to be run on both Azure Cloud Services and Azure Web Sites. + /// A safe wrapper for . + /// How this service is deployed (e.g. Cloud Service, Web App) determines whether or not is accessible. + /// is loaded from the machine and not from the bin directory, so it may or may not be present on the machine. /// internal static class SafeRoleEnvironment { - private const string _serviceRuntimeAssembly = "Microsoft.WindowsAzure.ServiceRuntime"; - private const string _roleEnvironmentTypeName = "Microsoft.WindowsAzure.ServiceRuntime.RoleEnvironment"; - private const string _isAvailablePropertyName = "IsAvailable"; - - public static bool IsAvailable { get; private set; } - - static SafeRoleEnvironment() - { - // Find out if the code is running in the cloud service context. - Assembly assembly = GetServiceRuntimeAssembly(); - if (assembly != null) - { - Type roleEnvironmentType = assembly.GetType(_roleEnvironmentTypeName, false); - if (roleEnvironmentType != null) - { - PropertyInfo isAvailableProperty = roleEnvironmentType.GetProperty(_isAvailablePropertyName); - - try - { - IsAvailable = isAvailableProperty != null && (bool)isAvailableProperty.GetValue(null, new object[] { }); - } - catch (TargetInvocationException) - { - IsAvailable = false; - } - } - } - } - - /// - /// Delegate the call because we don't want RoleEnvironment appearing in the function scope of the caller because that - /// would trigger the assembly load: the very thing we are attempting to avoid - /// - /// - /// - public static string GetConfigurationSettingValue(string configurationSettingName) + public static bool TryGetConfigurationSettingValue(string configurationSettingName, out string value) { - return RoleEnvironment.GetConfigurationSettingValue(configurationSettingName); + return TryGetField(() => RoleEnvironment.GetConfigurationSettingValue(configurationSettingName), out value); } - public static string GetDeploymentId() + public static bool TryGetDeploymentId(out string id) { - if (IsAvailable) - { - return RoleEnvironment.DeploymentId; - } - - return string.Empty; + return TryGetField(() => RoleEnvironment.DeploymentId, out id); } - public static string GetLocalResourceRootPath(string name) + public static bool TryGetLocalResourceRootPath(string name, out string path) { - return RoleEnvironment.GetLocalResource(name).RootPath; + return TryGetField(() => RoleEnvironment.GetLocalResource(name).RootPath, out path); } - /// - /// Loads and returns the latest available version of the service runtime assembly. - /// - /// Loaded assembly, if any. - private static Assembly GetServiceRuntimeAssembly() + private static bool _assemblyIsAvailable = true; + private static bool TryGetField(Func getValue, out T value) { - Assembly assembly = null; + value = default(T); try { - assembly = Assembly.LoadWithPartialName(_serviceRuntimeAssembly); + if (!_assemblyIsAvailable || !RoleEnvironment.IsAvailable) + { + // If RoleEnvironment isn't available, we can't access it. + return false; + } + + value = getValue(); + return true; } catch (Exception e) { - if (!(e is FileNotFoundException || e is FileLoadException || e is BadImageFormatException)) + if (e is FileNotFoundException || e is FileLoadException || e is BadImageFormatException) { + // If an exception related to loading files is thrown, the assembly is not available. + // Cache the fact that it is not available so we don't repeatedly throw and catch exceptions. + _assemblyIsAvailable = false; + return false; + } + else + { + // If an exception unrelated to loading files is thrown, the assembly must have thrown the exception itself. + // Rethrow the exception so it can be handled by the caller. throw; } } - - return assembly; } } } \ No newline at end of file diff --git a/src/NuGet.Services.BasicSearch/Startup.cs b/src/NuGet.Services.BasicSearch/Startup.cs index 0afd828ad..e9e78d1a3 100644 --- a/src/NuGet.Services.BasicSearch/Startup.cs +++ b/src/NuGet.Services.BasicSearch/Startup.cs @@ -94,21 +94,21 @@ public class Startup // Overwrite the index's Azure Directory cache path if configured ot use an Azure Local Storage resource. if (!string.IsNullOrEmpty(config.AzureDirectoryCacheLocalResourceName)) { - if (!SafeRoleEnvironment.IsAvailable) + if (SafeRoleEnvironment.TryGetLocalResourceRootPath(config.AzureDirectoryCacheLocalResourceName, out var path)) { - _logger.LogWarning( - "Cannot use Azure Local Resource {LocalResourceName} for caching when the RoleEnvironment is not available", - config.AzureDirectoryCacheLocalResourceName); - } - else - { - config.AzureDirectoryCachePath = SafeRoleEnvironment.GetLocalResourceRootPath(config.AzureDirectoryCacheLocalResourceName); + config.AzureDirectoryCachePath = path; _logger.LogInformation( "Set Azure Directory cache path to Azure Local Resource = {LocalResourceName}, Path = {LocalResourcePath}", config.AzureDirectoryCacheLocalResourceName, config.AzureDirectoryCachePath); } + else + { + _logger.LogWarning( + "Cannot use Azure Local Resource {LocalResourceName} for caching when the RoleEnvironment is not available", + config.AzureDirectoryCacheLocalResourceName); + } } // redirect all HTTP requests to HTTPS From 79e85dfcf70e7638e3c8cc1b7ad4c8f876a6f44e Mon Sep 17 00:00:00 2001 From: Damon Tivel Date: Fri, 24 Aug 2018 11:39:40 -0700 Subject: [PATCH 12/15] Test: improve and add registration tests (#346) More progress on https://github.com/NuGet/NuGetGallery/issues/6317. --- tests/CatalogTests/CatalogTests.csproj | 5 +- ...s.cs => RegistrationIndependentPackage.cs} | 35 +- .../Helpers/RegistrationIndependentPage.cs | 32 ++ .../CatalogTests/Helpers/RegistrationIndex.cs | 4 +- .../Helpers/RegistrationPackage.cs | 31 +- ...rationIndexPage.cs => RegistrationPage.cs} | 24 +- .../Registration/RegistrationMakerTests.cs | 423 +++++++++++++----- 7 files changed, 386 insertions(+), 168 deletions(-) rename tests/CatalogTests/Helpers/{RegistrationIndexPackageDetails.cs => RegistrationIndependentPackage.cs} (54%) create mode 100644 tests/CatalogTests/Helpers/RegistrationIndependentPage.cs rename tests/CatalogTests/Helpers/{RegistrationIndexPage.cs => RegistrationPage.cs} (67%) diff --git a/tests/CatalogTests/CatalogTests.csproj b/tests/CatalogTests/CatalogTests.csproj index 470807c27..39906d603 100644 --- a/tests/CatalogTests/CatalogTests.csproj +++ b/tests/CatalogTests/CatalogTests.csproj @@ -194,10 +194,11 @@ + - - + + diff --git a/tests/CatalogTests/Helpers/RegistrationIndexPackageDetails.cs b/tests/CatalogTests/Helpers/RegistrationIndependentPackage.cs similarity index 54% rename from tests/CatalogTests/Helpers/RegistrationIndexPackageDetails.cs rename to tests/CatalogTests/Helpers/RegistrationIndependentPackage.cs index 0a17e37ee..efe89f6a7 100644 --- a/tests/CatalogTests/Helpers/RegistrationIndexPackageDetails.cs +++ b/tests/CatalogTests/Helpers/RegistrationIndependentPackage.cs @@ -2,44 +2,49 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using Newtonsoft.Json; +using Newtonsoft.Json.Linq; using NgTests; namespace CatalogTests.Helpers { - internal sealed class RegistrationIndexPackageDetails + internal sealed class RegistrationIndependentPackage { [JsonProperty(CatalogConstants.IdKeyword)] internal string IdKeyword { get; } [JsonProperty(CatalogConstants.TypeKeyword)] - internal string TypeKeyword { get; } - [JsonProperty(CatalogConstants.CommitId)] - internal string CommitId { get; } - [JsonProperty(CatalogConstants.CommitTimeStamp)] - internal string CommitTimeStamp { get; } + internal string[] TypeKeyword { get; } [JsonProperty(CatalogConstants.CatalogEntry)] - internal RegistrationPackageDetails CatalogEntry { get; } + internal string CatalogEntry { get; } + [JsonProperty(CatalogConstants.Listed)] + internal bool Listed { get; } [JsonProperty(CatalogConstants.PackageContent)] internal string PackageContent { get; } + [JsonProperty(CatalogConstants.Published)] + internal string Published { get; } [JsonProperty(CatalogConstants.Registration)] internal string Registration { get; } + [JsonProperty(CatalogConstants.ContextKeyword)] + internal JObject ContextKeyword { get; } [JsonConstructor] - internal RegistrationIndexPackageDetails( + internal RegistrationIndependentPackage( string idKeyword, - string typeKeyword, - string commitId, - string commitTimeStamp, - RegistrationPackageDetails catalogEntry, + string[] typeKeyword, + string catalogEntry, + bool listed, string packageContent, - string registration) + string published, + string registration, + JObject contextKeyword) { IdKeyword = idKeyword; TypeKeyword = typeKeyword; - CommitId = commitId; - CommitTimeStamp = commitTimeStamp; CatalogEntry = catalogEntry; + Listed = listed; PackageContent = packageContent; + Published = published; Registration = registration; + ContextKeyword = contextKeyword; } } } \ No newline at end of file diff --git a/tests/CatalogTests/Helpers/RegistrationIndependentPage.cs b/tests/CatalogTests/Helpers/RegistrationIndependentPage.cs new file mode 100644 index 000000000..ba0d26d4c --- /dev/null +++ b/tests/CatalogTests/Helpers/RegistrationIndependentPage.cs @@ -0,0 +1,32 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NgTests; + +namespace CatalogTests.Helpers +{ + internal sealed class RegistrationIndependentPage : RegistrationPage + { + [JsonProperty(CatalogConstants.ContextKeyword)] + internal JObject ContextKeyword { get; } + + [JsonConstructor] + internal RegistrationIndependentPage( + string idKeyword, + string typeKeyword, + string commitId, + string commitTimeStamp, + int count, + RegistrationPackage[] items, + string parent, + string lower, + string upper, + JObject contextKeyword) + : base(idKeyword, typeKeyword, commitId, commitTimeStamp, count, items, parent, lower, upper) + { + ContextKeyword = contextKeyword; + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Helpers/RegistrationIndex.cs b/tests/CatalogTests/Helpers/RegistrationIndex.cs index 7d478ce90..57f4a954b 100644 --- a/tests/CatalogTests/Helpers/RegistrationIndex.cs +++ b/tests/CatalogTests/Helpers/RegistrationIndex.cs @@ -20,7 +20,7 @@ internal sealed class RegistrationIndex [JsonProperty(CatalogConstants.Count)] internal int Count { get; } [JsonProperty(CatalogConstants.Items)] - internal RegistrationIndexPage[] Items { get; } + internal RegistrationPage[] Items { get; } [JsonProperty(CatalogConstants.ContextKeyword)] internal JObject ContextKeyword { get; } @@ -31,7 +31,7 @@ internal sealed class RegistrationIndex string commitId, string commitTimeStamp, int count, - RegistrationIndexPage[] items, + RegistrationPage[] items, JObject contextKeyword) { IdKeyword = idKeyword; diff --git a/tests/CatalogTests/Helpers/RegistrationPackage.cs b/tests/CatalogTests/Helpers/RegistrationPackage.cs index d5ba456f8..391629e4b 100644 --- a/tests/CatalogTests/Helpers/RegistrationPackage.cs +++ b/tests/CatalogTests/Helpers/RegistrationPackage.cs @@ -2,7 +2,6 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using Newtonsoft.Json; -using Newtonsoft.Json.Linq; using NgTests; namespace CatalogTests.Helpers @@ -12,39 +11,35 @@ internal sealed class RegistrationPackage [JsonProperty(CatalogConstants.IdKeyword)] internal string IdKeyword { get; } [JsonProperty(CatalogConstants.TypeKeyword)] - internal string[] TypeKeyword { get; } + internal string TypeKeyword { get; } + [JsonProperty(CatalogConstants.CommitId)] + internal string CommitId { get; } + [JsonProperty(CatalogConstants.CommitTimeStamp)] + internal string CommitTimeStamp { get; } [JsonProperty(CatalogConstants.CatalogEntry)] - internal string CatalogEntry { get; } - [JsonProperty(CatalogConstants.Listed)] - internal bool Listed { get; } + internal RegistrationPackageDetails CatalogEntry { get; } [JsonProperty(CatalogConstants.PackageContent)] internal string PackageContent { get; } - [JsonProperty(CatalogConstants.Published)] - internal string Published { get; } [JsonProperty(CatalogConstants.Registration)] internal string Registration { get; } - [JsonProperty(CatalogConstants.ContextKeyword)] - internal JObject ContextKeyword { get; } [JsonConstructor] internal RegistrationPackage( string idKeyword, - string[] typeKeyword, - string catalogEntry, - bool listed, + string typeKeyword, + string commitId, + string commitTimeStamp, + RegistrationPackageDetails catalogEntry, string packageContent, - string published, - string registration, - JObject contextKeyword) + string registration) { IdKeyword = idKeyword; TypeKeyword = typeKeyword; + CommitId = commitId; + CommitTimeStamp = commitTimeStamp; CatalogEntry = catalogEntry; - Listed = listed; PackageContent = packageContent; - Published = published; Registration = registration; - ContextKeyword = contextKeyword; } } } \ No newline at end of file diff --git a/tests/CatalogTests/Helpers/RegistrationIndexPage.cs b/tests/CatalogTests/Helpers/RegistrationPage.cs similarity index 67% rename from tests/CatalogTests/Helpers/RegistrationIndexPage.cs rename to tests/CatalogTests/Helpers/RegistrationPage.cs index f6cca2d61..86c19f3d0 100644 --- a/tests/CatalogTests/Helpers/RegistrationIndexPage.cs +++ b/tests/CatalogTests/Helpers/RegistrationPage.cs @@ -6,35 +6,35 @@ namespace CatalogTests.Helpers { - internal sealed class RegistrationIndexPage + internal class RegistrationPage { [JsonProperty(CatalogConstants.IdKeyword)] - internal string IdKeyword { get; } + public string IdKeyword { get; protected set; } [JsonProperty(CatalogConstants.TypeKeyword)] - internal string TypeKeyword { get; } + public string TypeKeyword { get; protected set; } [JsonProperty(CatalogConstants.CommitId)] - internal string CommitId { get; } + public string CommitId { get; } [JsonProperty(CatalogConstants.CommitTimeStamp)] - internal string CommitTimeStamp { get; } + public string CommitTimeStamp { get; protected set; } [JsonProperty(CatalogConstants.Count)] - internal int Count { get; } + public int Count { get; protected set; } [JsonProperty(CatalogConstants.Items)] - internal RegistrationIndexPackageDetails[] Items { get; } + public RegistrationPackage[] Items { get; protected set; } [JsonProperty(CatalogConstants.Parent)] - internal string Parent { get; } + public string Parent { get; protected set; } [JsonProperty(CatalogConstants.Lower)] - internal string Lower { get; } + public string Lower { get; protected set; } [JsonProperty(CatalogConstants.Upper)] - internal string Upper { get; } + public string Upper { get; protected set; } [JsonConstructor] - internal RegistrationIndexPage( + internal RegistrationPage( string idKeyword, string typeKeyword, string commitId, string commitTimeStamp, int count, - RegistrationIndexPackageDetails[] items, + RegistrationPackage[] items, string parent, string lower, string upper) diff --git a/tests/CatalogTests/Registration/RegistrationMakerTests.cs b/tests/CatalogTests/Registration/RegistrationMakerTests.cs index 250a10c3e..b62a428e2 100644 --- a/tests/CatalogTests/Registration/RegistrationMakerTests.cs +++ b/tests/CatalogTests/Registration/RegistrationMakerTests.cs @@ -6,6 +6,7 @@ using System.IO; using System.Linq; using System.Reflection; +using System.Text; using System.Threading; using System.Threading.Tasks; using CatalogTests.Helpers; @@ -27,8 +28,7 @@ public class RegistrationMakerTests { private const string _cacheControl = "no-store"; private const string _contentType = "application/json"; - private const int _partitionSize = 1; - private const int _packageCountThreshold = 128; + private const int _defaultPackageCountThreshold = 128; private static readonly Uri _contentBaseAddress = new Uri("https://nuget.test/"); private static readonly JsonSerializerSettings _jsonSettings = new JsonSerializerSettings() @@ -49,25 +49,12 @@ public RegistrationMakerTests() public async Task ProcessAsync_WithEmptyStorageAndEmptyNewItems_DoesNotCreateAnything() { var emptyItems = new Dictionary(); - var storage = await ProcessAsync(emptyItems, packageId: null); + var storage = await ProcessAsync(emptyItems, packageId: null, partitionSize: 1); Assert.Empty(storage.Content); Assert.Empty(storage.ContentBytes); } - [Fact] - public async Task ProcessAsync_WithEmptyStorageAndOneNewItem_CreatesIndexAndLeaf() - { - var packageDetails = new CatalogPackageDetails(); - var newItem = CreateNewItem(packageDetails); - var storage = await ProcessAsync(newItem, packageDetails.Id); - - Verify( - storage, - expectedStorageContentCount: 2, - expectedPage: new ExpectedPage(packageDetails)); - } - [Theory] [InlineData(1)] [InlineData(2)] @@ -89,15 +76,12 @@ public async Task ProcessAsync_WithCustomPartitionSize_FillsPages(int partitionS var expectedPages = Repaginate(pages, partitionSize); - Verify( - storage, - expectedStorageContentCount: pages.Count + 1 /*index*/, - expectedPages: expectedPages); + Verify(storage, expectedPages, partitionSize); } } [Fact] - public async Task ProcessAsync_WithVariablePartitionSize_RepaginatesOlderPages() + public async Task ProcessAsync_WithVariablePartitionSize_RepaginatesExistingPages() { var partitionSize = 2; var packageVersionCount = partitionSize * 2; @@ -117,10 +101,7 @@ public async Task ProcessAsync_WithVariablePartitionSize_RepaginatesOlderPages() expectedPages = Repaginate(pages, partitionSize); - Verify( - storage, - expectedStorageContentCount: pages.Count + 1 /*index*/, - expectedPages: expectedPages); + Verify(storage, expectedPages, partitionSize); } partitionSize = 3; @@ -134,10 +115,66 @@ public async Task ProcessAsync_WithVariablePartitionSize_RepaginatesOlderPages() expectedPages = Repaginate(pages, partitionSize); - Verify( - storage, - expectedStorageContentCount: pages.Count + 1 /*index*/, - expectedPages: expectedPages); + Verify(storage, expectedPages, partitionSize); + } + + [Fact] + public async Task ProcessAsync_WhenNewPackageVersionWouldChangeExistingPage_RepaginatesExistingPage() + { + const int partitionSize = 2; + var pages = new List(); + CatalogPackageDetails packageDetails; + IReadOnlyDictionary newItem; + MemoryStorage storage; + IReadOnlyList expectedPages; + + for (var i = 0; i < 2; ++i) + { + packageDetails = new CatalogPackageDetails(id: "a", version: $"1.0.{i * 2}"); + newItem = CreateNewItem(packageDetails); + storage = await ProcessAsync(newItem, packageDetails.Id, partitionSize); + + pages.Add(new ExpectedPage(packageDetails)); + + expectedPages = Repaginate(pages, partitionSize); + + Verify(storage, expectedPages, partitionSize); + } + + packageDetails = new CatalogPackageDetails(id: "a", version: "1.0.1"); + newItem = CreateNewItem(packageDetails); + storage = await ProcessAsync(newItem, packageDetails.Id, partitionSize); + + pages.Insert(index: 1, item: new ExpectedPage(packageDetails)); + + expectedPages = Repaginate(pages, partitionSize); + + Verify(storage, expectedPages, partitionSize); + } + + [Fact] + public async Task ProcessAsync_WithCustomPackageCountThreshold_TransitionsToPageWhenThresholdHit() + { + const int partitionSize = 2; + const int packageCountThreshold = 2; + var pages = new List(); + CatalogPackageDetails packageDetails; + IReadOnlyDictionary newItem; + MemoryStorage storage; + IReadOnlyList expectedPages; + + for (var i = 0; i < 2; ++i) + { + packageDetails = new CatalogPackageDetails(id: "a", version: $"1.0.{i}"); + newItem = CreateNewItem(packageDetails); + storage = await ProcessAsync(newItem, packageDetails.Id, partitionSize, packageCountThreshold); + + pages.Add(new ExpectedPage(packageDetails)); + + expectedPages = Repaginate(pages, partitionSize); + + Verify(storage, expectedPages, partitionSize, packageCountThreshold); + } } private static IReadOnlyDictionary CreateNewItem(CatalogPackageDetails packageDetails) @@ -161,7 +198,8 @@ public async Task ProcessAsync_WithVariablePartitionSize_RepaginatesOlderPages() private async Task ProcessAsync( IReadOnlyDictionary newItems, string packageId, - int? partitionSize = null) + int partitionSize, + int packageCountThreshold = _defaultPackageCountThreshold) { var registrationKey = new RegistrationKey(packageId?.ToLowerInvariant() ?? string.Empty); @@ -170,8 +208,8 @@ public async Task ProcessAsync_WithVariablePartitionSize_RepaginatesOlderPages() newItems, _storageFactory, _contentBaseAddress, - partitionSize ?? _partitionSize, - _packageCountThreshold, + partitionSize, + packageCountThreshold, _telemetryService.Object, CancellationToken.None); @@ -180,32 +218,41 @@ public async Task ProcessAsync_WithVariablePartitionSize_RepaginatesOlderPages() private void Verify( MemoryStorage registrationStorage, - int expectedStorageContentCount, - ExpectedPage expectedPage) + IReadOnlyList expectedPages, + int partitionSize, + int packageCountThreshold = _defaultPackageCountThreshold) { - Verify(registrationStorage, expectedStorageContentCount, new[] { expectedPage }); - } + var expectedStorageContentCount = expectedPages.SelectMany(page => page.Details).Count(); + + expectedStorageContentCount += expectedStorageContentCount / packageCountThreshold; + + ++expectedStorageContentCount; // index - private void Verify( - MemoryStorage registrationStorage, - int expectedStorageContentCount, - IReadOnlyList expectedPages) - { Assert.Equal(expectedStorageContentCount, registrationStorage.Content.Count); var firstPage = expectedPages.First(); var packageId = firstPage.Details.First().Id.ToLowerInvariant(); - var indexUri = new Uri(GetRegistrationPackageIndexUri(packageId)); + var indexUri = GetRegistrationPackageIndexUri(packageId); var index = GetStorageContent(registrationStorage, indexUri); - VerifyIndex(index, indexUri, packageId, expectedPages); + VerifyRegistrationIndex( + registrationStorage, + index, + indexUri, + packageId, + expectedPages, + partitionSize, + packageCountThreshold); } - private void VerifyIndex( + private void VerifyRegistrationIndex( + MemoryStorage registrationStorage, RegistrationIndex index, Uri indexUri, string packageId, - IReadOnlyList expectedPages) + IReadOnlyList expectedPages, + int partitionSize, + int packageCountThreshold) { Assert.Equal(indexUri.AbsoluteUri, index.IdKeyword); Assert.Equal( @@ -220,15 +267,41 @@ public async Task ProcessAsync_WithVariablePartitionSize_RepaginatesOlderPages() for (var i = 0; i < index.Count; ++i) { var page = index.Items[i]; - var expectedPageVersionRange = expectedPages[i]; - - VerifyIndexItems( - page, - indexUri, - packageId, - expectedPageVersionRange, - index.CommitId, - index.CommitTimeStamp); + var expectedPage = expectedPages[i]; + var expectFullPage = i < index.Count - 1; + + if (expectFullPage) + { + Assert.Equal(partitionSize, page.Count); + } + else + { + Assert.InRange(page.Count, low: 1, high: partitionSize); + } + + if (page.Count == packageCountThreshold) + { + VerifyRegistrationPageReference( + registrationStorage, + page, + packageId, + expectedPage, + index.CommitId, + index.CommitTimeStamp); + } + else + { + var pageUri = GetRegistrationPageUri(packageId, expectedPage); + + VerifyRegistrationPage( + registrationStorage, + page, + pageUri, + packageId, + expectedPage, + index.CommitId, + index.CommitTimeStamp); + } } var expectedContext = new JObject( @@ -282,88 +355,137 @@ public async Task ProcessAsync_WithVariablePartitionSize_RepaginatesOlderPages() Assert.Equal(expectedContext.ToString(), index.ContextKeyword.ToString()); } - private void VerifyIndexItems( - RegistrationIndexPage items, - Uri indexUri, + private void VerifyRegistrationPage( + MemoryStorage registrationStorage, + RegistrationPage page, + Uri pageUri, string packageId, ExpectedPage expectedPage, string commitId, string commitTimeStamp) { - Assert.Equal( - GetRegistrationPackageIndexUri(packageId) + $"#page/{expectedPage.LowerVersion}/{expectedPage.UpperVersion}", - items.IdKeyword); - Assert.Equal(CatalogConstants.CatalogCatalogPage, items.TypeKeyword); - Assert.Equal(commitId, items.CommitId); - Assert.Equal(commitTimeStamp, items.CommitTimeStamp); - Assert.Equal(items.Count, items.Items.Length); - Assert.Equal(GetRegistrationPackageIndexUri(packageId), items.Parent); - Assert.Equal(expectedPage.LowerVersion, items.Lower); - Assert.Equal(expectedPage.UpperVersion, items.Upper); - - for (var i = 0; i < items.Count; ++i) + var packageIndexUri = GetRegistrationPackageIndexUri(packageId); + + Assert.Equal(pageUri.AbsoluteUri, page.IdKeyword); + Assert.Equal(CatalogConstants.CatalogCatalogPage, page.TypeKeyword); + Assert.Equal(commitId, page.CommitId); + Assert.Equal(commitTimeStamp, page.CommitTimeStamp); + Assert.Equal(expectedPage.LowerVersion, page.Lower); + Assert.Equal(expectedPage.UpperVersion, page.Upper); + Assert.Equal(page.Count, page.Items.Length); + Assert.Equal(packageIndexUri.AbsoluteUri, page.Parent); + + for (var i = 0; i < page.Count; ++i) { - var item = items.Items[i]; + var item = page.Items[i]; var packageDetails = expectedPage.Details[i]; - VerifyIndexItem(item, packageDetails, commitId, commitTimeStamp); + VerifyRegistrationPackage(registrationStorage, item, packageDetails, commitId, commitTimeStamp); } } - private void VerifyIndexItem( - RegistrationIndexPackageDetails item, + private void VerifyRegistrationPageReference( + MemoryStorage registrationStorage, + RegistrationPage page, + string packageId, + ExpectedPage expectedPage, + string commitId, + string commitTimeStamp) + { + var pageUri = GetRegistrationPageReferenceUri(packageId, expectedPage); + + Assert.Equal(pageUri.AbsoluteUri, page.IdKeyword); + Assert.Equal(CatalogConstants.CatalogCatalogPage, page.TypeKeyword); + Assert.Equal(commitId, page.CommitId); + Assert.Equal(commitTimeStamp, page.CommitTimeStamp); + Assert.Equal(expectedPage.LowerVersion, page.Lower); + Assert.Equal(expectedPage.UpperVersion, page.Upper); + + Assert.Null(page.Items); + Assert.Null(page.Parent); + + var independentPage = GetStorageContent(registrationStorage, pageUri); + + VerifyRegistrationPage( + registrationStorage, + independentPage, + pageUri, + packageId, + expectedPage, + commitId, + commitTimeStamp); + + JObject expectedContext = GetExpectedIndexOrPageContext(); + + Assert.Equal(expectedContext.ToString(), independentPage.ContextKeyword.ToString()); + } + + private void VerifyRegistrationPackage( + MemoryStorage registrationStorage, + RegistrationPackage package, CatalogPackageDetails packageDetails, string commitId, string commitTimeStamp) { var packageId = packageDetails.Id.ToLowerInvariant(); var packageVersion = packageDetails.Version.ToLowerInvariant(); - - Assert.Equal(GetRegistrationPackageVersionUri(packageId, packageVersion), item.IdKeyword); - Assert.Equal(CatalogConstants.Package, item.TypeKeyword); - Assert.Equal(commitId, item.CommitId); - Assert.Equal(commitTimeStamp, item.CommitTimeStamp); - Assert.Equal(packageDetails.IdKeyword, item.CatalogEntry.IdKeyword); - Assert.Equal(CatalogConstants.PackageDetails, item.CatalogEntry.TypeKeyword); - Assert.Equal(packageDetails.Authors, item.CatalogEntry.Authors); - Assert.Equal(packageDetails.Description, item.CatalogEntry.Description); - Assert.Empty(item.CatalogEntry.IconUrl); - Assert.Equal(packageDetails.Id, item.CatalogEntry.Id); - Assert.Empty(item.CatalogEntry.Language); - Assert.Empty(item.CatalogEntry.LicenseUrl); - Assert.Equal(packageDetails.Listed, item.CatalogEntry.Listed); - Assert.Empty(item.CatalogEntry.MinClientVersion); - Assert.Equal(GetPackageContentUri(packageId, packageVersion), item.CatalogEntry.PackageContent); - Assert.Empty(item.CatalogEntry.ProjectUrl); - Assert.Equal(GetRegistrationDateTime(packageDetails.Published), item.CatalogEntry.Published); - Assert.Equal(packageDetails.RequireLicenseAcceptance, item.CatalogEntry.RequireLicenseAcceptance); - Assert.Empty(item.CatalogEntry.Summary); - Assert.Equal(new[] { string.Empty }, item.CatalogEntry.Tags); - Assert.Empty(item.CatalogEntry.Title); - Assert.Equal(packageDetails.Version, item.CatalogEntry.Version); - - var leafUri = new Uri(GetRegistrationPackageVersionUri(packageId, packageVersion)); - var registrationStorage = (MemoryStorage)_storageFactory.Create(packageId); - var leaf = GetStorageContent(registrationStorage, leafUri); - - VerifyLeaf(leaf, leafUri, packageDetails, packageId, packageVersion); + var packageVersionUri = GetRegistrationPackageVersionUri(packageId, packageVersion); + var packageContentUri = GetPackageContentUri(packageId, packageVersion); + + Assert.Equal(packageVersionUri.AbsoluteUri, package.IdKeyword); + Assert.Equal(CatalogConstants.Package, package.TypeKeyword); + Assert.Equal(commitId, package.CommitId); + Assert.Equal(commitTimeStamp, package.CommitTimeStamp); + Assert.Equal(packageDetails.IdKeyword, package.CatalogEntry.IdKeyword); + Assert.Equal(CatalogConstants.PackageDetails, package.CatalogEntry.TypeKeyword); + Assert.Equal(packageDetails.Authors, package.CatalogEntry.Authors); + Assert.Equal(packageDetails.Description, package.CatalogEntry.Description); + Assert.Empty(package.CatalogEntry.IconUrl); + Assert.Equal(packageDetails.Id, package.CatalogEntry.Id); + Assert.Empty(package.CatalogEntry.Language); + Assert.Empty(package.CatalogEntry.LicenseUrl); + Assert.Equal(packageDetails.Listed, package.CatalogEntry.Listed); + Assert.Empty(package.CatalogEntry.MinClientVersion); + Assert.Equal(packageContentUri.AbsoluteUri, package.CatalogEntry.PackageContent); + Assert.Empty(package.CatalogEntry.ProjectUrl); + Assert.Equal(GetRegistrationDateTime(packageDetails.Published), package.CatalogEntry.Published); + Assert.Equal(packageDetails.RequireLicenseAcceptance, package.CatalogEntry.RequireLicenseAcceptance); + Assert.Empty(package.CatalogEntry.Summary); + Assert.Equal(new[] { string.Empty }, package.CatalogEntry.Tags); + Assert.Empty(package.CatalogEntry.Title); + Assert.Equal(packageDetails.Version, package.CatalogEntry.Version); + + var independentPackageUri = GetRegistrationPackageVersionUri(packageId, packageVersion); + var independentPackage = GetStorageContent( + registrationStorage, + independentPackageUri); + + VerifyRegistrationIndependentPackage( + independentPackage, + independentPackageUri, + packageDetails, + packageId, + packageVersion); } - private void VerifyLeaf( - RegistrationPackage leaf, - Uri expectedIdKeyword, + private void VerifyRegistrationIndependentPackage( + RegistrationIndependentPackage package, + Uri packageUri, CatalogPackageDetails packageDetails, string packageId, string packageVersion) { - Assert.Equal(expectedIdKeyword.AbsoluteUri, leaf.IdKeyword); + var packageContentUri = GetPackageContentUri(packageId, packageVersion); + var packageIndexUri = GetRegistrationPackageIndexUri(packageId); + + Assert.Equal(packageUri.AbsoluteUri, package.IdKeyword); Assert.Equal( new[] { CatalogConstants.Package, CatalogConstants.NuGetCatalogSchemaPermalinkUri }, - leaf.TypeKeyword); - Assert.Equal(packageDetails.Listed, leaf.Listed); - Assert.Equal(GetPackageContentUri(packageId, packageVersion), leaf.PackageContent); - Assert.Equal(GetRegistrationDateTime(packageDetails.Published), leaf.Published); - Assert.Equal(GetRegistrationPackageIndexUri(packageId), leaf.Registration); + package.TypeKeyword); + Assert.Equal(packageDetails.Listed, package.Listed); + Assert.Equal(packageContentUri.AbsoluteUri, package.PackageContent); + Assert.Equal(GetRegistrationDateTime(packageDetails.Published), package.Published); + Assert.Equal(packageIndexUri.AbsoluteUri, package.Registration); var expectedContext = new JObject( new JProperty(CatalogConstants.VocabKeyword, CatalogConstants.NuGetSchemaUri), @@ -381,7 +503,58 @@ public async Task ProcessAsync_WithVariablePartitionSize_RepaginatesOlderPages() new JObject( new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime)))); - Assert.Equal(expectedContext.ToString(), leaf.ContextKeyword.ToString()); + Assert.Equal(expectedContext.ToString(), package.ContextKeyword.ToString()); + } + + private static JObject GetExpectedIndexOrPageContext() + { + return new JObject( + new JProperty(CatalogConstants.VocabKeyword, CatalogConstants.NuGetSchemaUri), + new JProperty(CatalogConstants.Catalog, CatalogConstants.NuGetCatalogSchemaUri), + new JProperty(CatalogConstants.Xsd, CatalogConstants.XmlSchemaUri), + new JProperty(CatalogConstants.Items, + new JObject( + new JProperty(CatalogConstants.IdKeyword, CatalogConstants.CatalogItem), + new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword))), + new JProperty(CatalogConstants.CommitTimeStamp, + new JObject( + new JProperty(CatalogConstants.IdKeyword, CatalogConstants.CatalogCommitTimeStamp), + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime))), + new JProperty(CatalogConstants.CommitId, + new JObject( + new JProperty(CatalogConstants.IdKeyword, CatalogConstants.CatalogCommitId))), + new JProperty(CatalogConstants.Count, + new JObject( + new JProperty(CatalogConstants.IdKeyword, CatalogConstants.CatalogCount))), + new JProperty(CatalogConstants.Parent, + new JObject( + new JProperty(CatalogConstants.IdKeyword, CatalogConstants.CatalogParent), + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.IdKeyword))), + new JProperty(CatalogConstants.Tags, + new JObject( + new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword), + new JProperty(CatalogConstants.IdKeyword, CatalogConstants.Tag))), + new JProperty(CatalogConstants.PackageTargetFrameworks, + new JObject( + new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword), + new JProperty(CatalogConstants.IdKeyword, CatalogConstants.PackageTargetFramework))), + new JProperty(CatalogConstants.DependencyGroups, + new JObject( + new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword), + new JProperty(CatalogConstants.IdKeyword, CatalogConstants.DependencyGroup))), + new JProperty(CatalogConstants.Dependencies, + new JObject( + new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword), + new JProperty(CatalogConstants.IdKeyword, CatalogConstants.Dependency))), + new JProperty(CatalogConstants.PackageContent, + new JObject( + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.IdKeyword))), + new JProperty(CatalogConstants.Published, + new JObject( + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime))), + new JProperty(CatalogConstants.Registration, + new JObject( + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.IdKeyword)))); } private static string GetRegistrationDateTime(string catalogDateTime) @@ -401,7 +574,7 @@ private static T GetStorageContent(MemoryStorage registrationStorage, Uri con Assert.Equal(_cacheControl, jTokenStorageContent.CacheControl); Assert.Equal(_contentType, jTokenStorageContent.ContentType); - var properties = typeof(T).GetProperties(BindingFlags.NonPublic | BindingFlags.Instance); + var properties = typeof(T).GetProperties(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance); // Verify that no new properties were added unexpectedly. Assert.Equal(properties.Length, ((JObject)jTokenStorageContent.Content).Count); @@ -414,19 +587,31 @@ private static T GetStorageContent(MemoryStorage registrationStorage, Uri con } } - private static string GetPackageContentUri(string packageId, string packageVersion) + private static Uri GetPackageContentUri(string packageId, string packageVersion) + { + return new Uri($"{_contentBaseAddress.AbsoluteUri}packages/{packageId}.{packageVersion}.nupkg"); + } + + private Uri GetRegistrationPageUri(string packageId, ExpectedPage expectedPage) + { + return new Uri(GetRegistrationPackageIndexUri(packageId), + $"#page/{expectedPage.LowerVersion}/{expectedPage.UpperVersion}"); + } + + private Uri GetRegistrationPageReferenceUri(string packageId, ExpectedPage expectedPage) { - return $"{_contentBaseAddress.AbsoluteUri}packages/{packageId}.{packageVersion}.nupkg"; + return new Uri($"{_storageFactory.BaseAddress.AbsoluteUri}{packageId}/page" + + $"/{expectedPage.LowerVersion}/{expectedPage.UpperVersion}.json"); } - private string GetRegistrationPackageIndexUri(string packageId) + private Uri GetRegistrationPackageIndexUri(string packageId) { - return $"{_storageFactory.BaseAddress.AbsoluteUri}{packageId}/index.json"; + return new Uri($"{_storageFactory.BaseAddress.AbsoluteUri}{packageId}/index.json"); } - private string GetRegistrationPackageVersionUri(string packageId, string packageVersion) + private Uri GetRegistrationPackageVersionUri(string packageId, string packageVersion) { - return $"{_storageFactory.BaseAddress.AbsoluteUri}{packageId}/{packageVersion}.json"; + return new Uri($"{_storageFactory.BaseAddress.AbsoluteUri}{packageId}/{packageVersion}.json"); } private static IReadOnlyList Repaginate( From 1c5b6e71f9e9561f4bdaf9e49c5dfa59b52a236c Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Wed, 29 Aug 2018 08:29:28 -0700 Subject: [PATCH 13/15] Use ServerCommon commit that is consistent with other repositories (#347) Progress on https://github.com/NuGet/Engineering/issues/1644 --- build.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.ps1 b/build.ps1 index 4c68bb591..fe4c20e83 100644 --- a/build.ps1 +++ b/build.ps1 @@ -9,7 +9,7 @@ param ( [string]$SemanticVersion = '1.0.0-zlocal', [string]$Branch, [string]$CommitSHA, - [string]$BuildBranch = 'aebd86ceb96cf777b122991459838b7fc9d5a7bb' + [string]$BuildBranch = '306fec22edac68336d7e32124d51248734c3a395' ) # For TeamCity - If any issue occurs, this script fails the build. - By default, TeamCity returns an exit code of 0 for all powershell scripts, even if they fail From d8e656c29467d323d96ea259fb504219c40ea173 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Sharma?= Date: Wed, 5 Sep 2018 10:47:54 -0700 Subject: [PATCH 14/15] [Monitoring] Ensure packages are signed (#348) This adds a validation to ensure indexed packages are signed by verifying that the package contains a [package signature file](https://github.com/NuGet/Home/wiki/Package-Signatures-Technical-Details#-the-package-signature-file). This validation is disabled for now. To enable this validation, the flag `-expectsSignature true` must be added to ng.exe's command line arguments. This validation will be enabled once all packages are repository signed. Part of https://github.com/NuGet/Engineering/issues/1627 --- src/Ng/Arguments.cs | 5 + src/Ng/Jobs/MonitoringProcessorJob.cs | 3 +- .../Model/PackageRegistrationLeafMetadata.cs | 1 + ...ervices.Metadata.Catalog.Monitoring.csproj | 3 + .../Test/Catalog/CatalogAggregateValidator.cs | 39 +++ .../Catalog/PackageHasSignatureValidator.cs | 130 ++++++++ .../MetadataFieldInconsistencyException.cs | 5 - .../MissingPackageSignatureFileException.cs | 25 ++ .../TimestampComparisonException.cs | 1 - .../Test/Exceptions/ValidationException.cs | 9 +- .../Test/PackageValidatorFactory.cs | 12 +- .../Validation/Test/ValidationContext.cs | 9 +- tests/NgTests/NgTests.csproj | 2 + .../CatalogAggregateValidatorFacts.cs | 49 +++ .../PackageHasSignatureValidatorFacts.cs | 302 ++++++++++++++++++ .../Validators/ValidationContextTests.cs | 4 +- 16 files changed, 577 insertions(+), 22 deletions(-) create mode 100644 src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Catalog/CatalogAggregateValidator.cs create mode 100644 src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Catalog/PackageHasSignatureValidator.cs create mode 100644 src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/MissingPackageSignatureFileException.cs create mode 100644 tests/NgTests/Validators/CatalogAggregateValidatorFacts.cs create mode 100644 tests/NgTests/Validators/PackageHasSignatureValidatorFacts.cs diff --git a/src/Ng/Arguments.cs b/src/Ng/Arguments.cs index 6129049c2..3f3bcff41 100644 --- a/src/Ng/Arguments.cs +++ b/src/Ng/Arguments.cs @@ -146,6 +146,11 @@ public static class Arguments /// If the queue contains more messages than this, the job will not requeue any invalid packages. /// public const string MaxRequeueQueueSize = "maxRequeueQueueSize"; + + /// + /// If true, packages are expected to have at least a repository signature. + /// + public const string RequireSignature = "requireSignature"; #endregion #region KeyVault diff --git a/src/Ng/Jobs/MonitoringProcessorJob.cs b/src/Ng/Jobs/MonitoringProcessorJob.cs index 71733f77f..e726978ae 100644 --- a/src/Ng/Jobs/MonitoringProcessorJob.cs +++ b/src/Ng/Jobs/MonitoringProcessorJob.cs @@ -43,6 +43,7 @@ protected override void Init(IDictionary arguments, Cancellation var gallery = arguments.GetOrThrow(Arguments.Gallery); var index = arguments.GetOrThrow(Arguments.Index); var source = arguments.GetOrThrow(Arguments.Source); + var requireSignature = arguments.GetOrDefault(Arguments.RequireSignature, false); var verbose = arguments.GetOrDefault(Arguments.Verbose, false); CommandHelpers.AssertAzureStorage(arguments); @@ -58,7 +59,7 @@ protected override void Init(IDictionary arguments, Cancellation gallery, index, monitoringStorageFactory, auditingStorageFactory, string.Join(", ", endpointInputs.Select(e => e.Name))); _packageValidator = new PackageValidatorFactory(LoggerFactory) - .Create(gallery, index, auditingStorageFactory, endpointInputs, messageHandlerFactory, verbose); + .Create(gallery, index, auditingStorageFactory, endpointInputs, messageHandlerFactory, requireSignature, verbose); _queue = CommandHelpers.CreateStorageQueue(arguments, PackageValidatorContext.Version); diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Model/PackageRegistrationLeafMetadata.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Model/PackageRegistrationLeafMetadata.cs index 1506c5a52..bef6d24bf 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Model/PackageRegistrationLeafMetadata.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Model/PackageRegistrationLeafMetadata.cs @@ -8,6 +8,7 @@ namespace NuGet.Services.Metadata.Catalog.Monitoring { /// /// The metadata for a particular package in its registration leaf. + /// See: https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#registration-leaf /// public class PackageRegistrationLeafMetadata { diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj b/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj index 5d65dea40..309601d72 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj @@ -162,12 +162,14 @@ + + @@ -177,6 +179,7 @@ + diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Catalog/CatalogAggregateValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Catalog/CatalogAggregateValidator.cs new file mode 100644 index 000000000..85e50d091 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Catalog/CatalogAggregateValidator.cs @@ -0,0 +1,39 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog.Monitoring.Validation.Test.Catalog +{ + /// + /// Runs all catalog validations and aggregates their results. + /// + public sealed class CatalogAggregateValidator : IAggregateValidator + { + private List _validators; + + public CatalogAggregateValidator(ValidatorFactory factory, bool requireSignature) + { + factory = factory ?? throw new ArgumentNullException(nameof(factory)); + + _validators = new List(); + + if (requireSignature) + { + _validators.Add(factory.Create(typeof(PackageHasSignatureValidator))); + } + } + + public string Name => GetType().FullName; + + public async Task ValidateAsync(ValidationContext context) + { + return new AggregateValidationResult( + this, + await Task.WhenAll(_validators.Select(v => v.ValidateAsync(context)))); + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Catalog/PackageHasSignatureValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Catalog/PackageHasSignatureValidator.cs new file mode 100644 index 000000000..947c80ec2 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Catalog/PackageHasSignatureValidator.cs @@ -0,0 +1,130 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json.Linq; +using NuGet.Protocol; +using NuGet.Protocol.Core.Types; +using NuGet.Services.Metadata.Catalog.Monitoring.Validation.Test.Exceptions; + +namespace NuGet.Services.Metadata.Catalog.Monitoring.Validation.Test.Catalog +{ + /// + /// Validates that the package is signed by verifying the presence of the "package signature file" + /// in the nupkg. See: https://github.com/NuGet/Home/wiki/Package-Signatures-Technical-Details#-the-package-signature-file + /// + public sealed class PackageHasSignatureValidator : Validator + { + private const string NupkgSignatureFile = ".signature.p7s"; + + private readonly ILogger _logger; + + public PackageHasSignatureValidator( + IDictionary feedToSource, + ILogger logger) + : base(feedToSource, logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + protected override async Task ShouldRunAsync(ValidationContext context) + { + if (!ShouldRunValidator(context)) + { + return false; + } + + return await base.ShouldRunAsync(context); + } + + protected override async Task RunInternalAsync(ValidationContext context) + { + await RunValidatorAsync(context); + } + + public bool ShouldRunValidator(ValidationContext context) + { + var latest = context.Entries + .OrderByDescending(e => e.CommitTimeStamp) + .FirstOrDefault(); + + if (latest == null) + { + _logger.LogInformation( + "Skipping package {PackageId} {PackageVersion} as it had no catalog entries", + context.Package.Id, + context.Package.Version); + + return false; + } + + // We don't need to validate the package if the latest entry indicates deletion. + if (latest.IsDelete()) + { + _logger.LogInformation( + "Skipping package {PackageId} {PackageVersion} as its latest catalog entry is a delete", + context.Package.Id, + context.Package.Version); + + return false; + } + + return true; + } + + public async Task RunValidatorAsync(ValidationContext context) + { + var latest = context.Entries + .OrderByDescending(e => e.CommitTimeStamp) + .FirstOrDefault(); + + _logger.LogInformation( + "Validating that catalog entry {CatalogEntry} for package {PackageId} {PackageVersion} has a package signature file...", + latest.Uri, + context.Package.Id, + context.Package.Version); + + var leaf = await context.Client.GetJObjectAsync(latest.Uri, context.CancellationToken); + + if (!LeafHasSignatureFile(leaf)) + { + _logger.LogWarning( + "Catalog entry {CatalogEntry} for package {PackageId} {PackageVersion} is missing a package signature file", + latest.Uri, + context.Package.Id, + context.Package.Version); + + throw new MissingPackageSignatureFileException( + latest.Uri, + $"Catalog entry {latest.Uri} for package {context.Package.Id} {context.Package.Version} is missing a package signature file"); + } + + _logger.LogInformation( + "Validated that catalog entry {CatalogEntry} for package {PackageId} {PackageVersion} has a package signature", + latest.Uri, + context.Package.Id, + context.Package.Version); + } + + private bool LeafHasSignatureFile(JObject leaf) + { + var packageEntries = leaf["packageEntries"]; + + if (packageEntries == null) + { + throw new InvalidOperationException("Catalog leaf is missing the 'packageEntries' property"); + } + + if (!(packageEntries is JArray files)) + { + throw new InvalidOperationException("Catalog leaf's 'packageEntries' property is malformed"); + } + + return files.Any(file => (string)file["fullName"] == NupkgSignatureFile); + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/MetadataFieldInconsistencyException.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/MetadataFieldInconsistencyException.cs index 17425311a..9de229f23 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/MetadataFieldInconsistencyException.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/MetadataFieldInconsistencyException.cs @@ -2,11 +2,6 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.Serialization; -using System.Text; -using System.Threading.Tasks; namespace NuGet.Services.Metadata.Catalog.Monitoring { diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/MissingPackageSignatureFileException.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/MissingPackageSignatureFileException.cs new file mode 100644 index 000000000..15eacb265 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/MissingPackageSignatureFileException.cs @@ -0,0 +1,25 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.Metadata.Catalog.Monitoring.Validation.Test.Exceptions +{ + /// + /// The exception thrown when a catalog entry is missing a package signature file. + /// This indicates that the package is not signed. + /// See: https://github.com/NuGet/Home/wiki/Package-Signatures-Technical-Details#-the-package-signature-file + /// + public sealed class MissingPackageSignatureFileException : ValidationException + { + public MissingPackageSignatureFileException(Uri catalogEntry, string message) + : base(message) + { + CatalogEntry = catalogEntry; + + Data.Add(nameof(CatalogEntry), catalogEntry); + } + + public Uri CatalogEntry { get; } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/TimestampComparisonException.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/TimestampComparisonException.cs index 87a51103f..2fac129dc 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/TimestampComparisonException.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/TimestampComparisonException.cs @@ -1,7 +1,6 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using System; using Newtonsoft.Json; namespace NuGet.Services.Metadata.Catalog.Monitoring diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/ValidationException.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/ValidationException.cs index 35341f262..52ed7f60b 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/ValidationException.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/ValidationException.cs @@ -1,8 +1,7 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; namespace NuGet.Services.Metadata.Catalog.Monitoring { diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidatorFactory.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidatorFactory.cs index f6d9781f0..f102859ee 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidatorFactory.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidatorFactory.cs @@ -6,7 +6,7 @@ using System.Linq; using System.Net.Http; using Microsoft.Extensions.Logging; -using NuGet.Protocol; +using NuGet.Services.Metadata.Catalog.Monitoring.Validation.Test.Catalog; using NuGet.Services.Metadata.Catalog.Persistence; namespace NuGet.Services.Metadata.Catalog.Monitoring @@ -28,14 +28,18 @@ public PackageValidatorFactory(ILoggerFactory loggerFactory) StorageFactory auditingStorageFactory, IEnumerable endpointInputs, Func messageHandlerFactory, + bool requireSignature = false, bool verbose = false) { var validatorFactory = new ValidatorFactoryFactory(_loggerFactory).Create(galleryUrl, indexUrl); - var endpointFactory = new EndpointFactory(validatorFactory, messageHandlerFactory, _loggerFactory); - var endpoints = endpointInputs.Select(e => endpointFactory.Create(e)); - return new PackageValidator(endpoints, auditingStorageFactory, _loggerFactory.CreateLogger()); + var validators = new List(); + + validators.AddRange(endpointInputs.Select(e => endpointFactory.Create(e))); + validators.Add(new CatalogAggregateValidator(validatorFactory, requireSignature)); + + return new PackageValidator(validators, auditingStorageFactory, _loggerFactory.CreateLogger()); } } } \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/ValidationContext.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/ValidationContext.cs index c9342164f..15508a40d 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/ValidationContext.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/ValidationContext.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; using NuGet.Packaging.Core; @@ -28,12 +29,12 @@ public class ValidationContext /// /// The s for the package that were collected. /// - public IEnumerable Entries { get; } + public IReadOnlyList Entries { get; } /// /// The s, if any are associated with the . /// - public IEnumerable DeletionAuditEntries { get; } + public IReadOnlyList DeletionAuditEntries { get; } /// /// The to use when needed. @@ -64,8 +65,8 @@ public ValidationContext() : this() { Package = package; - Entries = entries; - DeletionAuditEntries = deletionAuditEntries; + Entries = entries?.ToList() ?? null; + DeletionAuditEntries = deletionAuditEntries?.ToList() ?? null; Client = client; CancellationToken = token; } diff --git a/tests/NgTests/NgTests.csproj b/tests/NgTests/NgTests.csproj index d920cbcb9..3940ea81f 100644 --- a/tests/NgTests/NgTests.csproj +++ b/tests/NgTests/NgTests.csproj @@ -308,8 +308,10 @@ TestCatalogEntries.resx + + diff --git a/tests/NgTests/Validators/CatalogAggregateValidatorFacts.cs b/tests/NgTests/Validators/CatalogAggregateValidatorFacts.cs new file mode 100644 index 000000000..6b74fcea7 --- /dev/null +++ b/tests/NgTests/Validators/CatalogAggregateValidatorFacts.cs @@ -0,0 +1,49 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Moq; +using NuGet.Protocol; +using NuGet.Protocol.Core.Types; +using NuGet.Services.Metadata.Catalog.Monitoring; +using NuGet.Services.Metadata.Catalog.Monitoring.Validation.Test.Catalog; +using Xunit; + +namespace NgTests.Validators +{ + public class CatalogAggregateValidatorFacts + { + [Fact] + public async Task DoesntValidateSignatureIfDisabled() + { + var mock = Mock.Of>(); + var loggerFactory = Mock.Of(); + var factory = new ValidatorFactory(mock, loggerFactory); + + var target = new CatalogAggregateValidator(factory, requireSignature: false); + + var result = await target.ValidateAsync(new ValidationContext()); + + Assert.Equal(0, result.ValidationResults.Count()); + } + + [Fact] + public async Task ValidatesSignature() + { + var feedToSource = new Mock>(); + var loggerFactory = Mock.Of(); + var factory = new ValidatorFactory(feedToSource.Object, loggerFactory); + + feedToSource.Setup(x => x[It.IsAny()]).Returns(new Mock().Object); + + var target = new CatalogAggregateValidator(factory, requireSignature: true); + + var result = await target.ValidateAsync(new ValidationContext()); + + Assert.Equal(1, result.ValidationResults.Count()); + } + } +} diff --git a/tests/NgTests/Validators/PackageHasSignatureValidatorFacts.cs b/tests/NgTests/Validators/PackageHasSignatureValidatorFacts.cs new file mode 100644 index 000000000..be2f0bc9b --- /dev/null +++ b/tests/NgTests/Validators/PackageHasSignatureValidatorFacts.cs @@ -0,0 +1,302 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Moq; +using Newtonsoft.Json; +using Newtonsoft.Json.Serialization; +using NgTests.Infrastructure; +using NuGet.Packaging.Core; +using NuGet.Protocol; +using NuGet.Protocol.Core.Types; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Monitoring; +using NuGet.Services.Metadata.Catalog.Monitoring.Validation.Test.Catalog; +using NuGet.Services.Metadata.Catalog.Monitoring.Validation.Test.Exceptions; +using NuGet.Versioning; +using Xunit; + +namespace NgTests +{ + public class PackageHasSignatureValidatorFacts + { + public class ShouldRunValidator : FactsBase + { + [Fact] + public void SkipsIfNoEntries() + { + var context = CreateValidationContext(catalogEntries: new CatalogIndexEntry[0]); + + Assert.False(_target.ShouldRunValidator(context)); + } + + [Fact] + public void SkipsIfLatestEntryIsDelete() + { + var context = CreateValidationContext( + catalogEntries: new[] + { + new CatalogIndexEntry( + uri: null, + type: DetailsCatalogEntry, + commitId: string.Empty, + commitTs: DateTime.MinValue, + id: PackageId, + version: PackageNuGetVersion), + new CatalogIndexEntry( + uri: null, + type: DeleteCatalogEntry, + commitId: string.Empty, + commitTs: DateTime.MinValue.AddDays(1), + id: PackageId, + version: PackageNuGetVersion), + }); + + Assert.False(_target.ShouldRunValidator(context)); + } + + [Fact] + public void RunsIfLatestEntryIsntDelete() + { + var context = CreateValidationContext( + catalogEntries: new[] + { + new CatalogIndexEntry( + uri: null, + type: DeleteCatalogEntry, + commitId: string.Empty, + commitTs: DateTime.MinValue, + id: PackageId, + version: PackageNuGetVersion), + new CatalogIndexEntry( + uri: null, + type: DetailsCatalogEntry, + commitId: string.Empty, + commitTs: DateTime.MinValue.AddDays(1), + id: PackageId, + version: PackageNuGetVersion), + }); + + Assert.True(_target.ShouldRunValidator(context)); + } + } + + public class RunValidatorAsync : FactsBase + { + [Fact] + public async Task ReturnsGracefullyIfLatestLeafHasSignatureFile() + { + // Arrange + var context = CreateValidationContext( + catalogEntries: new[] + { + new CatalogIndexEntry( + uri: new Uri("http://localhost/a.json"), + type: DetailsCatalogEntry, + commitId: string.Empty, + commitTs: DateTime.MinValue, + id: PackageId, + version: PackageNuGetVersion), + new CatalogIndexEntry( + uri: new Uri("http://localhost/b.json"), + type: DetailsCatalogEntry, + commitId: string.Empty, + commitTs: DateTime.MinValue.AddDays(1), + id: PackageId, + version: PackageNuGetVersion), + }); + + AddCatalogLeaf("/a.json", new CatalogLeaf + { + PackageEntries = new[] + { + new PackageEntry { FullName = "hello.txt" } + } + }); + + AddCatalogLeaf("/b.json", new CatalogLeaf + { + PackageEntries = new[] + { + new PackageEntry { FullName = "hello.txt" }, + new PackageEntry { FullName = ".signature.p7s" } + } + }); + + // Act & Assert + await _target.RunValidatorAsync(context); + } + + [Fact] + public async Task ThrowsIfLatestLeafIsMissingASignatureFile() + { + // Arrange + var malformedUri = new Uri("http://localhost/b.json"); + + var context = CreateValidationContext( + catalogEntries: new[] + { + new CatalogIndexEntry( + uri: new Uri("http://localhost/a.json"), + type: DetailsCatalogEntry, + commitId: string.Empty, + commitTs: DateTime.MinValue, + id: PackageId, + version: PackageNuGetVersion), + new CatalogIndexEntry( + uri: malformedUri, + type: DetailsCatalogEntry, + commitId: string.Empty, + commitTs: DateTime.MinValue.AddDays(1), + id: PackageId, + version: PackageNuGetVersion), + }); + + AddCatalogLeaf("/a.json", new CatalogLeaf + { + PackageEntries = new[] + { + new PackageEntry { FullName = ".signature.p7s" } + } + }); + + AddCatalogLeaf("/b.json", new CatalogLeaf + { + PackageEntries = new[] + { + new PackageEntry { FullName = "hello.txt" } + } + }); + + // Act & Assert + var e = await Assert.ThrowsAsync(() => _target.RunValidatorAsync(context)); + + Assert.Same(malformedUri, e.CatalogEntry); + } + + [Fact] + public async Task ThrowsIfLeafPackageEntriesIsMissing() + { + // Arrange + var malformedUri = new Uri("http://localhost/a.json"); + + var context = CreateValidationContext( + catalogEntries: new[] + { + new CatalogIndexEntry( + uri: malformedUri, + type: DetailsCatalogEntry, + commitId: string.Empty, + commitTs: DateTime.MinValue, + id: PackageId, + version: PackageNuGetVersion), + }); + + AddCatalogLeaf("/a.json", "{ 'this': 'is missing the packageEntries field' }"); + + // Act & Assert + var e = await Assert.ThrowsAsync(() => _target.RunValidatorAsync(context)); + + Assert.Equal("Catalog leaf is missing the 'packageEntries' property", e.Message); + } + + [Fact] + public async Task ThrowsIfLeafPackageEntriesIsMalformed() + { + // Arrange + var malformedUri = new Uri("http://localhost/a.json"); + + var context = CreateValidationContext( + catalogEntries: new[] + { + new CatalogIndexEntry( + uri: malformedUri, + type: DetailsCatalogEntry, + commitId: string.Empty, + commitTs: DateTime.MinValue, + id: PackageId, + version: PackageNuGetVersion), + }); + + AddCatalogLeaf("/a.json", "{ 'packageEntries': 'malformed'}"); + + // Act & Assert + var e = await Assert.ThrowsAsync(() => _target.RunValidatorAsync(context)); + + Assert.Equal("Catalog leaf's 'packageEntries' property is malformed", e.Message); + } + } + + public class FactsBase + { + public const string DeleteCatalogEntry = "nuget:PackageDelete"; + public const string DetailsCatalogEntry = "nuget:PackageDetails"; + + public const string PackageId = "TestPackage"; + public const string PackageVersion = "1.0.0"; + + public static readonly NuGetVersion PackageNuGetVersion = NuGetVersion.Parse(PackageVersion); + + private readonly MockServerHttpClientHandler _mockServer; + protected readonly PackageHasSignatureValidator _target; + + public FactsBase() + { + var feedToSource = new Mock>(); + var logger = Mock.Of>(); + + feedToSource.Setup(x => x[It.IsAny()]).Returns(new Mock().Object); + + _mockServer = new MockServerHttpClientHandler(); + _target = new PackageHasSignatureValidator(feedToSource.Object, logger); + } + + protected ValidationContext CreateValidationContext(IEnumerable catalogEntries = null) + { + catalogEntries = catalogEntries ?? new CatalogIndexEntry[0]; + + var httpClient = new CollectorHttpClient(_mockServer); + + return new ValidationContext( + new PackageIdentity(PackageId, PackageNuGetVersion), + catalogEntries, + new DeletionAuditEntry[0], + httpClient, + CancellationToken.None); + } + + protected void AddCatalogLeaf(string path, CatalogLeaf leaf) + { + var jsonSettings = new JsonSerializerSettings + { + ContractResolver = new CamelCasePropertyNamesContractResolver() + }; + + AddCatalogLeaf(path, JsonConvert.SerializeObject(leaf, jsonSettings)); + } + + protected void AddCatalogLeaf(string path, string leafContent) + { + _mockServer.SetAction(path, request => + { + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(leafContent) + }); + }); + } + + public class CatalogLeaf + { + public IEnumerable PackageEntries { get; set; } + } + } + } +} diff --git a/tests/NgTests/Validators/ValidationContextTests.cs b/tests/NgTests/Validators/ValidationContextTests.cs index 1a49525a1..63492297c 100644 --- a/tests/NgTests/Validators/ValidationContextTests.cs +++ b/tests/NgTests/Validators/ValidationContextTests.cs @@ -47,8 +47,8 @@ public void Constructor_InitializesProperties() cancellationToken); Assert.Same(package, context.Package); - Assert.Same(catalogIndexEntries, context.Entries); - Assert.Same(deletionAuditEntries, context.DeletionAuditEntries); + Assert.Equal(catalogIndexEntries.Count(), context.Entries.Count); + Assert.Equal(deletionAuditEntries.Count(), context.DeletionAuditEntries.Count); Assert.Same(client, context.Client); Assert.Equal(cancellationToken, context.CancellationToken); } From c6e1469c04020835a17535659a4a80aa4f5c1ecd Mon Sep 17 00:00:00 2001 From: Damon Tivel Date: Mon, 10 Sep 2018 15:13:44 -0700 Subject: [PATCH 15/15] V3: make stylistic consistency and cleanup changes (#349) Progress on https://github.com/NuGet/NuGetGallery/issues/6411. --- src/Catalog/CollectorBase.cs | 27 +++++---- src/Catalog/CommitCollector.cs | 30 ++++++---- src/Catalog/Dnx/DnxCatalogCollector.cs | 2 +- src/Catalog/PackageCatalogItem.cs | 10 ++-- .../Registration/RegistrationCollector.cs | 21 +++---- src/Catalog/Registration/RegistrationKey.cs | 2 +- src/Catalog/SortingCollector.cs | 10 +--- src/Catalog/SortingGraphCollector.cs | 5 -- src/Ng/Jobs/Catalog2DnxJob.cs | 5 +- src/Ng/Jobs/Catalog2LuceneJob.cs | 6 +- src/Ng/Jobs/Catalog2MonitoringJob.cs | 4 +- src/Ng/Jobs/Catalog2RegistrationJob.cs | 15 ++--- src/Ng/Jobs/CheckLuceneJob.cs | 2 +- src/Ng/Jobs/ClearLuceneJob.cs | 2 +- src/Ng/Jobs/CopyLuceneJob.cs | 2 +- src/Ng/Jobs/Db2LuceneJob.cs | 2 +- src/Ng/Jobs/Feed2CatalogJob.cs | 2 +- src/Ng/Jobs/LightningJob.cs | 8 +-- src/Ng/Jobs/LoopingNgJob.cs | 4 +- src/Ng/Jobs/Monitoring2MonitoringJob.cs | 2 +- src/Ng/Jobs/MonitoringProcessorJob.cs | 2 +- src/Ng/Jobs/NgJob.cs | 10 ++-- src/Ng/Jobs/Package2CatalogJob.cs | 2 +- src/Ng/Program.cs | 2 +- src/Ng/SearchIndexFromCatalogCollector.cs | 11 ++-- src/V3PerPackage/EnqueueCollector.cs | 6 +- src/V3PerPackage/EnqueueCommand.cs | 2 +- src/V3PerPackage/PerBatchProcessor.cs | 14 ++--- .../SortingGraphCollectorTests.cs | 4 +- tests/NgTests/DnxCatalogCollectorTests.cs | 60 +++++++++---------- tests/NgTests/Feed2CatalogTests.cs | 10 ++-- tests/NgTests/Package2CatalogJobTests.cs | 2 +- tests/NgTests/RegistrationCollectorTests.cs | 25 ++++---- .../SearchIndexFromCatalogCollectorTests.cs | 4 +- .../NgTests/SortingIdVersionCollectorTests.cs | 6 +- tests/NgTests/TestableFeed2CatalogJob.cs | 4 +- 36 files changed, 153 insertions(+), 172 deletions(-) diff --git a/src/Catalog/CollectorBase.cs b/src/Catalog/CollectorBase.cs index 9d5e7dcf0..040273f14 100644 --- a/src/Catalog/CollectorBase.cs +++ b/src/Catalog/CollectorBase.cs @@ -27,21 +27,19 @@ public abstract class CollectorBase Index = index ?? throw new ArgumentNullException(nameof(index)); } - public Uri Index { get; private set; } + public Uri Index { get; } - public int RequestCount { get; private set; } - - public async Task Run(CancellationToken cancellationToken) + public async Task RunAsync(CancellationToken cancellationToken) { - return await Run(MemoryCursor.CreateMin(), MemoryCursor.CreateMax(), cancellationToken); + return await RunAsync(MemoryCursor.CreateMin(), MemoryCursor.CreateMax(), cancellationToken); } - public async Task Run(DateTime front, DateTime back, CancellationToken cancellationToken) + public async Task RunAsync(DateTime front, DateTime back, CancellationToken cancellationToken) { - return await Run(new MemoryCursor(front), new MemoryCursor(back), cancellationToken); + return await RunAsync(new MemoryCursor(front), new MemoryCursor(back), cancellationToken); } - public async Task Run(ReadWriteCursor front, ReadCursor back, CancellationToken cancellationToken) + public async Task RunAsync(ReadWriteCursor front, ReadCursor back, CancellationToken cancellationToken) { await Task.WhenAll(front.Load(cancellationToken), back.Load(cancellationToken)); @@ -63,13 +61,16 @@ public async Task Run(ReadWriteCursor front, ReadCursor back, Cancellation client.Timeout = _httpClientTimeout.Value; } - result = await Fetch(client, front, back, cancellationToken); - RequestCount = client.RequestCount; + result = await FetchAsync(client, front, back, cancellationToken); } - + return result; } - protected abstract Task Fetch(CollectorHttpClient client, ReadWriteCursor front, ReadCursor back, CancellationToken cancellationToken); + protected abstract Task FetchAsync( + CollectorHttpClient client, + ReadWriteCursor front, + ReadCursor back, + CancellationToken cancellationToken); } -} +} \ No newline at end of file diff --git a/src/Catalog/CommitCollector.cs b/src/Catalog/CommitCollector.cs index 250cc77e4..f0bdd32ed 100644 --- a/src/Catalog/CommitCollector.cs +++ b/src/Catalog/CommitCollector.cs @@ -23,7 +23,11 @@ public abstract class CommitCollector : CollectorBase { } - protected override async Task Fetch(CollectorHttpClient client, ReadWriteCursor front, ReadCursor back, CancellationToken cancellationToken) + protected override async Task FetchAsync( + CollectorHttpClient client, + ReadWriteCursor front, + ReadCursor back, + CancellationToken cancellationToken) { JObject root; @@ -48,7 +52,7 @@ protected override async Task Fetch(CollectorHttpClient client, ReadWriteC JToken context = null; page.TryGetValue("@context", out context); - var batches = await CreateBatches(page["items"] + var batches = await CreateBatchesAsync(page["items"] .Select(item => new CatalogItem(item)) .Where(item => item.CommitTimeStamp > front.Value && item.CommitTimeStamp <= back.Value)); @@ -76,7 +80,7 @@ protected override async Task Fetch(CollectorHttpClient client, ReadWriteC { TelemetryConstants.BatchItemCount, batch.Items.Count.ToString() } })) { - acceptNextBatch = await OnProcessBatch( + acceptNextBatch = await OnProcessBatchAsync( client, batch.Items.Select(item => item.Value), context, @@ -112,7 +116,7 @@ protected override async Task Fetch(CollectorHttpClient client, ReadWriteC return acceptNextBatch; } - protected virtual Task> CreateBatches(IEnumerable catalogItems) + protected virtual Task> CreateBatchesAsync(IEnumerable catalogItems) { var batches = catalogItems .GroupBy(item => item.CommitTimeStamp) @@ -122,7 +126,13 @@ protected virtual Task> CreateBatches(IEnumerable< return Task.FromResult(batches); } - protected abstract Task OnProcessBatch(CollectorHttpClient client, IEnumerable items, JToken context, DateTime commitTimeStamp, bool isLastBatch, CancellationToken cancellationToken); + protected abstract Task OnProcessBatchAsync( + CollectorHttpClient client, + IEnumerable items, + JToken context, + DateTime commitTimeStamp, + bool isLastBatch, + CancellationToken cancellationToken); protected class CatalogItemBatch : IComparable { @@ -133,8 +143,8 @@ public CatalogItemBatch(DateTime commitTimeStamp, IEnumerable items Items.Sort(); } - public DateTime CommitTimeStamp { get; private set; } - public List Items { get; private set; } + public DateTime CommitTimeStamp { get; } + public List Items { get; } public int CompareTo(object obj) { @@ -151,9 +161,9 @@ public CatalogItem(JToken jtoken) Value = jtoken; } - public DateTime CommitTimeStamp { get; private set; } - public Uri Uri { get; private set; } - public JToken Value { get; private set; } + public DateTime CommitTimeStamp { get; } + public Uri Uri { get; } + public JToken Value { get; } public int CompareTo(object obj) { diff --git a/src/Catalog/Dnx/DnxCatalogCollector.cs b/src/Catalog/Dnx/DnxCatalogCollector.cs index 7c28e974a..3af28b824 100644 --- a/src/Catalog/Dnx/DnxCatalogCollector.cs +++ b/src/Catalog/Dnx/DnxCatalogCollector.cs @@ -55,7 +55,7 @@ public class DnxCatalogCollector : CommitCollector _maxDegreeOfParallelism = maxDegreeOfParallelism; } - protected override async Task OnProcessBatch( + protected override async Task OnProcessBatchAsync( CollectorHttpClient client, IEnumerable items, JToken context, diff --git a/src/Catalog/PackageCatalogItem.cs b/src/Catalog/PackageCatalogItem.cs index 489167635..2385c37a1 100644 --- a/src/Catalog/PackageCatalogItem.cs +++ b/src/Catalog/PackageCatalogItem.cs @@ -46,9 +46,9 @@ public override IGraph CreateContentGraph(CatalogContext context) graph.Assert(resource.Subject, publishedPredicate, graph.CreateLiteralNode(published.ToString("O"), Schema.DataTypes.DateTime)); // listed - INode listedPredicated = graph.CreateUriNode(Schema.Predicates.Listed); + INode listedPredicate = graph.CreateUriNode(Schema.Predicates.Listed); Boolean listed = GetListed(published); - graph.Assert(resource.Subject, listedPredicated, graph.CreateLiteralNode(listed.ToString(), Schema.DataTypes.Boolean)); + graph.Assert(resource.Subject, listedPredicate, graph.CreateLiteralNode(listed.ToString(), Schema.DataTypes.Boolean)); // created INode createdPredicate = graph.CreateUriNode(Schema.Predicates.Created); @@ -161,11 +161,11 @@ public override Uri GetItemType() public override IGraph CreatePageContent(CatalogContext context) { Uri resourceUri = new Uri(GetBaseAddress() + GetRelativeAddress()); - + Graph graph = new Graph(); INode subject = graph.CreateUriNode(resourceUri); - + INode idPredicate = graph.CreateUriNode(Schema.Predicates.Id); INode versionPredicate = graph.CreateUriNode(Schema.Predicates.Version); @@ -187,4 +187,4 @@ protected override string GetItemIdentity() return (_id + "." + _normalizedVersion).ToLowerInvariant(); } } -} +} \ No newline at end of file diff --git a/src/Catalog/Registration/RegistrationCollector.cs b/src/Catalog/Registration/RegistrationCollector.cs index bf1ac3ad7..c2e741f49 100644 --- a/src/Catalog/Registration/RegistrationCollector.cs +++ b/src/Catalog/Registration/RegistrationCollector.cs @@ -27,25 +27,20 @@ public class RegistrationCollector : SortingGraphCollector Uri index, StorageFactory legacyStorageFactory, StorageFactory semVer2StorageFactory, + Uri contentBaseAddress, ITelemetryService telemetryService, Func handlerFunc = null) : base(index, new Uri[] { Schema.DataTypes.PackageDetails, Schema.DataTypes.PackageDelete }, telemetryService, handlerFunc) { - if (legacyStorageFactory == null) - { - throw new ArgumentNullException(nameof(legacyStorageFactory)); - } - - _legacyStorageFactory = legacyStorageFactory; + _legacyStorageFactory = legacyStorageFactory ?? throw new ArgumentNullException(nameof(legacyStorageFactory)); _semVer2StorageFactory = semVer2StorageFactory; _shouldIncludeSemVer2 = GetShouldIncludeRegistrationPackage(_semVer2StorageFactory); - - ContentBaseAddress = new Uri("http://tempuri.org"); + ContentBaseAddress = contentBaseAddress; } - public Uri ContentBaseAddress { get; set; } + public Uri ContentBaseAddress { get; } - protected override Task> CreateBatches(IEnumerable catalogItems) + protected override Task> CreateBatchesAsync(IEnumerable catalogItems) { // Grouping batches by commit is slow if it contains // the same package registration id over and over again. @@ -133,10 +128,8 @@ public static ShouldIncludeRegistrationPackage GetShouldIncludeRegistrationPacka { return (k, u, g) => true; } - else - { - return (k, u, g) => !NuGetVersionUtility.IsGraphSemVer2(k.Version, u, g); - } + + return (k, u, g) => !NuGetVersionUtility.IsGraphSemVer2(k.Version, u, g); } } } \ No newline at end of file diff --git a/src/Catalog/Registration/RegistrationKey.cs b/src/Catalog/Registration/RegistrationKey.cs index e0d5e983c..4bd615042 100644 --- a/src/Catalog/Registration/RegistrationKey.cs +++ b/src/Catalog/Registration/RegistrationKey.cs @@ -11,7 +11,7 @@ public class RegistrationKey { public RegistrationKey(string id) { - Id = id ?? throw new ArgumentNullException(nameof(id)); ; + Id = id ?? throw new ArgumentNullException(nameof(id)); } public string Id { get; } diff --git a/src/Catalog/SortingCollector.cs b/src/Catalog/SortingCollector.cs index e378f06bd..7524c18b4 100644 --- a/src/Catalog/SortingCollector.cs +++ b/src/Catalog/SortingCollector.cs @@ -16,12 +16,9 @@ public abstract class SortingCollector : CommitCollector where T : IEquatable public SortingCollector(Uri index, ITelemetryService telemetryService, Func handlerFunc = null) : base(index, telemetryService, handlerFunc) { - Concurrent = true; } - public bool Concurrent { get; set; } - - protected override async Task OnProcessBatch( + protected override async Task OnProcessBatchAsync( CollectorHttpClient client, IEnumerable items, JToken context, @@ -52,11 +49,6 @@ public SortingCollector(Uri index, ITelemetryService telemetryService, Func arguments, Cancellation TelemetryService.GlobalDimensions[TelemetryConstants.Destination] = _destination.AbsoluteUri; } - protected override async Task RunInternal(CancellationToken cancellationToken) + protected override async Task RunInternalAsync(CancellationToken cancellationToken) { using (Logger.BeginScope($"Logging for {{{TelemetryConstants.Destination}}}", _destination.AbsoluteUri)) using (TelemetryService.TrackDuration(TelemetryConstants.JobLoopSeconds)) @@ -107,7 +106,7 @@ protected override async Task RunInternal(CancellationToken cancellationToken) bool run; do { - run = await _collector.Run(_front, _back, cancellationToken); + run = await _collector.RunAsync(_front, _back, cancellationToken); } while (run); } diff --git a/src/Ng/Jobs/Catalog2LuceneJob.cs b/src/Ng/Jobs/Catalog2LuceneJob.cs index cd813f9f6..341498d53 100644 --- a/src/Ng/Jobs/Catalog2LuceneJob.cs +++ b/src/Ng/Jobs/Catalog2LuceneJob.cs @@ -87,7 +87,7 @@ protected override void Init(IDictionary arguments, Cancellation TelemetryService.GlobalDimensions[TelemetryConstants.Destination] = _destination; } - protected override async Task RunInternal(CancellationToken cancellationToken) + protected override async Task RunInternalAsync(CancellationToken cancellationToken) { using (Logger.BeginScope($"Logging for {{{TelemetryConstants.Destination}}}", _destination)) using (TelemetryService.TrackDuration(TelemetryConstants.JobLoopSeconds)) @@ -110,7 +110,7 @@ protected override async Task RunInternal(CancellationToken cancellationToken) bool run; do { - run = await collector.Run(front, back, cancellationToken); + run = await collector.RunAsync(front, back, cancellationToken); collector.EnsureCommitted(); // commit after each catalog page } @@ -148,4 +148,4 @@ public static IndexWriter CreateIndexWriter(Lucene.Net.Store.Directory directory return indexWriter; } } -} +} \ No newline at end of file diff --git a/src/Ng/Jobs/Catalog2MonitoringJob.cs b/src/Ng/Jobs/Catalog2MonitoringJob.cs index 1d6c718c7..e7185faff 100644 --- a/src/Ng/Jobs/Catalog2MonitoringJob.cs +++ b/src/Ng/Jobs/Catalog2MonitoringJob.cs @@ -65,12 +65,12 @@ protected override void Init(IDictionary arguments, Cancellation _back = context.Back; } - protected override async Task RunInternal(CancellationToken cancellationToken) + protected override async Task RunInternalAsync(CancellationToken cancellationToken) { bool run; do { - run = await _collector.Run(_front, _back, cancellationToken); + run = await _collector.RunAsync(_front, _back, cancellationToken); } while (run); } diff --git a/src/Ng/Jobs/Catalog2RegistrationJob.cs b/src/Ng/Jobs/Catalog2RegistrationJob.cs index 0c3820652..65e9e47ae 100644 --- a/src/Ng/Jobs/Catalog2RegistrationJob.cs +++ b/src/Ng/Jobs/Catalog2RegistrationJob.cs @@ -19,7 +19,8 @@ public class Catalog2RegistrationJob : LoopingNgJob private ReadCursor _back; private Uri _destination; - public Catalog2RegistrationJob(ITelemetryService telemetryService, ILoggerFactory loggerFactory) : base(telemetryService, loggerFactory) + public Catalog2RegistrationJob(ITelemetryService telemetryService, ILoggerFactory loggerFactory) + : base(telemetryService, loggerFactory) { } @@ -101,13 +102,9 @@ protected override void Init(IDictionary arguments, Cancellation new Uri(source), storageFactories.LegacyStorageFactory, storageFactories.SemVer2StorageFactory, + contentBaseAddress == null ? null : new Uri(contentBaseAddress), TelemetryService, - CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose)) - { - ContentBaseAddress = contentBaseAddress == null - ? null - : new Uri(contentBaseAddress) - }; + CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose)); var cursorStorage = storageFactories.LegacyStorageFactory.Create(); _front = new DurableCursor(cursorStorage.ResolveUri("cursor.json"), cursorStorage, MemoryCursor.MinValue); @@ -117,7 +114,7 @@ protected override void Init(IDictionary arguments, Cancellation TelemetryService.GlobalDimensions[TelemetryConstants.Destination] = _destination?.AbsoluteUri; } - protected override async Task RunInternal(CancellationToken cancellationToken) + protected override async Task RunInternalAsync(CancellationToken cancellationToken) { using (Logger.BeginScope($"Logging for {{{TelemetryConstants.Destination}}}", _destination?.AbsoluteUri)) using (TelemetryService.TrackDuration(TelemetryConstants.JobLoopSeconds)) @@ -125,7 +122,7 @@ protected override async Task RunInternal(CancellationToken cancellationToken) bool run; do { - run = await _collector.Run(_front, _back, cancellationToken); + run = await _collector.RunAsync(_front, _back, cancellationToken); } while (run); } diff --git a/src/Ng/Jobs/CheckLuceneJob.cs b/src/Ng/Jobs/CheckLuceneJob.cs index 4aa4647d8..fad7e7b34 100644 --- a/src/Ng/Jobs/CheckLuceneJob.cs +++ b/src/Ng/Jobs/CheckLuceneJob.cs @@ -34,7 +34,7 @@ protected override void Init(IDictionary arguments, Cancellation _directory = CommandHelpers.GetLuceneDirectory(arguments); } - protected override Task RunInternal(CancellationToken cancellationToken) + protected override Task RunInternalAsync(CancellationToken cancellationToken) { using (var reader = IndexReader.Open(_directory, true)) { diff --git a/src/Ng/Jobs/ClearLuceneJob.cs b/src/Ng/Jobs/ClearLuceneJob.cs index 27c81709b..5fbbb904a 100644 --- a/src/Ng/Jobs/ClearLuceneJob.cs +++ b/src/Ng/Jobs/ClearLuceneJob.cs @@ -36,7 +36,7 @@ protected override void Init(IDictionary arguments, Cancellation _directory = CommandHelpers.GetLuceneDirectory(arguments); } - protected override Task RunInternal(CancellationToken cancellationToken) + protected override Task RunInternalAsync(CancellationToken cancellationToken) { if (IndexReader.IndexExists(_directory)) { diff --git a/src/Ng/Jobs/CopyLuceneJob.cs b/src/Ng/Jobs/CopyLuceneJob.cs index abd78666f..654fea358 100644 --- a/src/Ng/Jobs/CopyLuceneJob.cs +++ b/src/Ng/Jobs/CopyLuceneJob.cs @@ -48,7 +48,7 @@ protected override void Init(IDictionary arguments, Cancellation _destDirectory = CommandHelpers.GetCopyDestLuceneDirectory(arguments); } - protected override Task RunInternal(CancellationToken cancellationToken) + protected override Task RunInternalAsync(CancellationToken cancellationToken) { Lucene.Net.Store.Directory.Copy(_srcDirectory, _destDirectory, true); diff --git a/src/Ng/Jobs/Db2LuceneJob.cs b/src/Ng/Jobs/Db2LuceneJob.cs index 1a6b688d6..8e4f896d3 100644 --- a/src/Ng/Jobs/Db2LuceneJob.cs +++ b/src/Ng/Jobs/Db2LuceneJob.cs @@ -41,7 +41,7 @@ protected override void Init(IDictionary arguments, Cancellation _catalogIndexUrl = new Uri(_source); } - protected override Task RunInternal(CancellationToken cancellationToken) + protected override Task RunInternalAsync(CancellationToken cancellationToken) { Sql2Lucene.Export(_connectionString, _catalogIndexUrl, _path, LoggerFactory); diff --git a/src/Ng/Jobs/Feed2CatalogJob.cs b/src/Ng/Jobs/Feed2CatalogJob.cs index a8e248f41..d76a72be4 100644 --- a/src/Ng/Jobs/Feed2CatalogJob.cs +++ b/src/Ng/Jobs/Feed2CatalogJob.cs @@ -109,7 +109,7 @@ protected override void Init(IDictionary arguments, Cancellation Timeout = TimeSpan.FromSeconds(300); } - protected override async Task RunInternal(CancellationToken cancellationToken) + protected override async Task RunInternalAsync(CancellationToken cancellationToken) { using (Logger.BeginScope($"Logging for {{{TelemetryConstants.Destination}}}", Destination.AbsoluteUri)) using (TelemetryService.TrackDuration(TelemetryConstants.JobLoopSeconds)) diff --git a/src/Ng/Jobs/LightningJob.cs b/src/Ng/Jobs/LightningJob.cs index 1d9ec9f8e..e47b83f0e 100644 --- a/src/Ng/Jobs/LightningJob.cs +++ b/src/Ng/Jobs/LightningJob.cs @@ -204,7 +204,7 @@ private void InitStrike(IDictionary arguments) _cursorFile = arguments.GetOrThrow(Arguments.CursorFile); } - protected override async Task RunInternal(CancellationToken cancellationToken) + protected override async Task RunInternalAsync(CancellationToken cancellationToken) { switch (_command.ToLowerInvariant()) { @@ -486,8 +486,7 @@ private async Task ProcessGraphsAsync(string packageId, IReadOnlyDictionary arguments, CancellationToken cancellationToken) + public override async Task RunAsync(IDictionary arguments, CancellationToken cancellationToken) { var intervalSec = arguments.GetOrDefault(Arguments.Interval, Arguments.DefaultInterval); Logger.LogInformation("Looping job at interval {Interval} seconds.", intervalSec); @@ -41,7 +41,7 @@ public override async Task Run(IDictionary arguments, Cancellati Logger.LogInformation("Running job."); try { - await RunInternal(cancellationToken); + await RunInternalAsync(cancellationToken); Logger.LogInformation("Job finished!"); } catch (TransientException te) diff --git a/src/Ng/Jobs/Monitoring2MonitoringJob.cs b/src/Ng/Jobs/Monitoring2MonitoringJob.cs index 13c9a2628..326e5be82 100644 --- a/src/Ng/Jobs/Monitoring2MonitoringJob.cs +++ b/src/Ng/Jobs/Monitoring2MonitoringJob.cs @@ -44,7 +44,7 @@ protected override void Init(IDictionary arguments, Cancellation _queue = CommandHelpers.CreateStorageQueue(arguments, PackageValidatorContext.Version); } - protected override async Task RunInternal(CancellationToken cancellationToken) + protected override async Task RunInternalAsync(CancellationToken cancellationToken) { var currentMessageCount = await _queue.GetMessageCount(cancellationToken); if (currentMessageCount > _maxRequeueQueueSize) diff --git a/src/Ng/Jobs/MonitoringProcessorJob.cs b/src/Ng/Jobs/MonitoringProcessorJob.cs index e726978ae..c45970948 100644 --- a/src/Ng/Jobs/MonitoringProcessorJob.cs +++ b/src/Ng/Jobs/MonitoringProcessorJob.cs @@ -72,7 +72,7 @@ protected override void Init(IDictionary arguments, Cancellation _client = new CollectorHttpClient(messageHandlerFactory()); } - protected override async Task RunInternal(CancellationToken cancellationToken) + protected override async Task RunInternalAsync(CancellationToken cancellationToken) { await ParallelAsync.Repeat(() => ProcessPackagesAsync(cancellationToken)); } diff --git a/src/Ng/Jobs/NgJob.cs b/src/Ng/Jobs/NgJob.cs index 4b2fc04a7..973c2069b 100644 --- a/src/Ng/Jobs/NgJob.cs +++ b/src/Ng/Jobs/NgJob.cs @@ -50,12 +50,12 @@ public virtual string GetUsage() protected abstract void Init(IDictionary arguments, CancellationToken cancellationToken); - protected abstract Task RunInternal(CancellationToken cancellationToken); - - public virtual async Task Run(IDictionary arguments, CancellationToken cancellationToken) + protected abstract Task RunInternalAsync(CancellationToken cancellationToken); + + public virtual async Task RunAsync(IDictionary arguments, CancellationToken cancellationToken) { Init(arguments, cancellationToken); - await RunInternal(cancellationToken); + await RunInternalAsync(cancellationToken); } } -} +} \ No newline at end of file diff --git a/src/Ng/Jobs/Package2CatalogJob.cs b/src/Ng/Jobs/Package2CatalogJob.cs index a73600632..357099d05 100644 --- a/src/Ng/Jobs/Package2CatalogJob.cs +++ b/src/Ng/Jobs/Package2CatalogJob.cs @@ -74,7 +74,7 @@ protected override void Init(IDictionary arguments, Cancellation _storage = storageFactory.Create(); } - protected override async Task RunInternal(CancellationToken cancellationToken) + protected override async Task RunInternalAsync(CancellationToken cancellationToken) { var timeout = TimeSpan.FromSeconds(300); diff --git a/src/Ng/Program.cs b/src/Ng/Program.cs index e9746c041..19d607d80 100644 --- a/src/Ng/Program.cs +++ b/src/Ng/Program.cs @@ -77,7 +77,7 @@ public static async Task MainAsync(string[] args) TelemetryConfiguration.Active.TelemetryInitializers.Add(new JobPropertiesTelemetryInitializer(telemetryService)); job = NgJobFactory.GetJob(jobName, telemetryService, loggerFactory); - await job.Run(arguments, cancellationTokenSource.Token); + await job.RunAsync(arguments, cancellationTokenSource.Token); } catch (ArgumentException ae) { diff --git a/src/Ng/SearchIndexFromCatalogCollector.cs b/src/Ng/SearchIndexFromCatalogCollector.cs index d512f901c..c590a14b9 100644 --- a/src/Ng/SearchIndexFromCatalogCollector.cs +++ b/src/Ng/SearchIndexFromCatalogCollector.cs @@ -46,7 +46,7 @@ public class SearchIndexFromCatalogCollector : CommitCollector _logger = logger; } - protected override async Task OnProcessBatch(CollectorHttpClient client, IEnumerable items, JToken context, DateTime commitTimeStamp, bool isLastBatch, CancellationToken cancellationToken) + protected override async Task OnProcessBatchAsync(CollectorHttpClient client, IEnumerable items, JToken context, DateTime commitTimeStamp, bool isLastBatch, CancellationToken cancellationToken) { JObject catalogIndex = null; if (_baseAddress != null) @@ -56,7 +56,7 @@ protected override async Task OnProcessBatch(CollectorHttpClient client, I _telemetryService.TrackCatalogIndexReadDuration(stopwatch.Elapsed, Index); } - IEnumerable catalogItems = await FetchCatalogItems(client, items, cancellationToken); + IEnumerable catalogItems = await FetchCatalogItemsAsync(client, items, cancellationToken); var numDocs = _indexWriter.NumDocs(); _logger.LogInformation(string.Format("Index contains {0} documents.", _indexWriter.NumDocs())); @@ -110,7 +110,10 @@ public void EnsureCommitted() _metadataForNextCommit = null; } - private static async Task> FetchCatalogItems(CollectorHttpClient client, IEnumerable items, CancellationToken cancellationToken) + private static async Task> FetchCatalogItemsAsync( + CollectorHttpClient client, + IEnumerable items, + CancellationToken cancellationToken) { IList> tasks = new List>(); @@ -327,4 +330,4 @@ private static IEnumerable GetCatalogStoragePaths(JObject index) return storagePaths; } } -} +} \ No newline at end of file diff --git a/src/V3PerPackage/EnqueueCollector.cs b/src/V3PerPackage/EnqueueCollector.cs index 1a37745a4..c4a91f39e 100644 --- a/src/V3PerPackage/EnqueueCollector.cs +++ b/src/V3PerPackage/EnqueueCollector.cs @@ -32,7 +32,7 @@ public class EnqueueCollector : CommitCollector _queue = queue ?? throw new ArgumentNullException(nameof(queue)); } - protected override Task> CreateBatches(IEnumerable catalogItems) + protected override Task> CreateBatchesAsync(IEnumerable catalogItems) { var catalogItemList = catalogItems.ToList(); @@ -41,7 +41,7 @@ protected override Task> CreateBatches(IEnumerable return Task.FromResult(new[] { new CatalogItemBatch(maxCommitTimestamp, catalogItemList) }.AsEnumerable()); } - protected override async Task OnProcessBatch( + protected override async Task OnProcessBatchAsync( CollectorHttpClient client, IEnumerable items, JToken context, @@ -78,4 +78,4 @@ private async Task EnqueueAsync(ConcurrentBag itemBag, CancellationToken } } } -} +} \ No newline at end of file diff --git a/src/V3PerPackage/EnqueueCommand.cs b/src/V3PerPackage/EnqueueCommand.cs index 8fce726a1..f6e71b3f4 100644 --- a/src/V3PerPackage/EnqueueCommand.cs +++ b/src/V3PerPackage/EnqueueCommand.cs @@ -40,7 +40,7 @@ public async Task ExecuteAsync(bool restart) var back = MemoryCursor.CreateMax(); - await _collector.Run(front, back, CancellationToken.None); + await _collector.RunAsync(front, back, CancellationToken.None); } } } \ No newline at end of file diff --git a/src/V3PerPackage/PerBatchProcessor.cs b/src/V3PerPackage/PerBatchProcessor.cs index 726dce597..50702affc 100644 --- a/src/V3PerPackage/PerBatchProcessor.cs +++ b/src/V3PerPackage/PerBatchProcessor.cs @@ -236,7 +236,7 @@ private async Task ExecuteCatalog2DnxAsync(PerBatchContext context, IReadOnlyLis var lowercasePackageIds = packageContexts.Select(x => x.PackageId.ToLowerInvariant()); using (await _stringLocker.AcquireAsync(lowercasePackageIds, TimeSpan.FromMinutes(5))) { - await collector.Run(CancellationToken.None); + await collector.RunAsync(CancellationToken.None); } } @@ -253,13 +253,11 @@ private async Task ExecuteCatalog2RegistrationAsync(PerBatchContext context, Uri catalogIndexUri, registrationStorageFactories.LegacyStorageFactory, registrationStorageFactories.SemVer2StorageFactory, + context.Global.ContentBaseAddress, serviceProvider.GetRequiredService(), - () => serviceProvider.GetRequiredService()) - { - ContentBaseAddress = context.Global.ContentBaseAddress, - }; + () => serviceProvider.GetRequiredService()); - await collector.Run(CancellationToken.None); + await collector.RunAsync(CancellationToken.None); } private async Task ExecuteCatalog2LuceneAsync(PerBatchContext context, Uri catalogIndexUri) @@ -284,7 +282,7 @@ private async Task ExecuteCatalog2LuceneAsync(PerBatchContext context, Uri catal serviceProvider.GetRequiredService(), () => serviceProvider.GetRequiredService()); - await collector.Run(CancellationToken.None); + await collector.RunAsync(CancellationToken.None); } } @@ -355,4 +353,4 @@ private async Task ExecuteCatalog2LuceneAsync(PerBatchContext context, Uri catal return serviceCollection.BuildServiceProvider(); } } -} +} \ No newline at end of file diff --git a/tests/CatalogTests/SortingGraphCollectorTests.cs b/tests/CatalogTests/SortingGraphCollectorTests.cs index e11a4b8b4..8c5452848 100644 --- a/tests/CatalogTests/SortingGraphCollectorTests.cs +++ b/tests/CatalogTests/SortingGraphCollectorTests.cs @@ -44,7 +44,7 @@ public async Task AllGraphsAreReadOnly() await _mockServer.AddStorageAsync(catalogStorage); // Act - var result = await _collector.Run(CancellationToken.None); + var result = await _collector.RunAsync(CancellationToken.None); // Assert foreach (var graphs in _collector.AllSortedGraphs) @@ -68,7 +68,7 @@ public async Task GraphsAreBatchedById() await _mockServer.AddStorageAsync(catalogStorage); // Act - var result = await _collector.Run(CancellationToken.None); + var result = await _collector.RunAsync(CancellationToken.None); // Assert Assert.Equal( diff --git a/tests/NgTests/DnxCatalogCollectorTests.cs b/tests/NgTests/DnxCatalogCollectorTests.cs index f676964cb..4a03bbfec 100644 --- a/tests/NgTests/DnxCatalogCollectorTests.cs +++ b/tests/NgTests/DnxCatalogCollectorTests.cs @@ -48,7 +48,6 @@ public DnxCatalogCollectorTests() var loggerFactory = new LoggerFactory(); _logger = loggerFactory.CreateLogger(); - // Setup collector _target = new DnxCatalogCollector( new Uri("http://tempuri.org/index.json"), _catalogToDnxStorageFactory, @@ -210,7 +209,7 @@ public void Constructor_WhenHttpClientTimeoutIsNull_InstantiatesClass() } [Fact] - public async Task Run_WhenPackageDoesNotHaveNuspec_SkipsPackage() + public async Task RunAsync_WhenPackageDoesNotHaveNuspec_SkipsPackage() { var zipWithNoNuspec = CreateZipStreamWithEntry("readme.txt", "content"); var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json"); @@ -225,7 +224,7 @@ public async Task Run_WhenPackageDoesNotHaveNuspec_SkipsPackage() var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); ReadCursor back = MemoryCursor.CreateMax(); - await _target.Run(front, back, CancellationToken.None); + await _target.RunAsync(front, back, CancellationToken.None); Assert.Equal(1, _catalogToDnxStorage.Content.Count); Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); @@ -235,7 +234,7 @@ public async Task Run_WhenPackageDoesNotHaveNuspec_SkipsPackage() } [Fact] - public async Task Run_WhenPackageHasNuspecWithWrongName_ProcessesPackage() + public async Task RunAsync_WhenPackageHasNuspecWithWrongName_ProcessesPackage() { var zipWithWrongNameNuspec = CreateZipStreamWithEntry("Newtonsoft.Json.nuspec", _nuspecData); var indexJsonUri = _catalogToDnxStorage.ResolveUri("/unlistedpackage/index.json"); @@ -252,7 +251,7 @@ public async Task Run_WhenPackageHasNuspecWithWrongName_ProcessesPackage() var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); ReadCursor back = MemoryCursor.CreateMax(); - await _target.Run(front, back, CancellationToken.None); + await _target.RunAsync(front, back, CancellationToken.None); Assert.Equal(4, _catalogToDnxStorage.Content.Count); Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); @@ -270,7 +269,7 @@ public async Task Run_WhenPackageHasNuspecWithWrongName_ProcessesPackage() } [Fact] - public async Task Run_WhenSourceNupkgIsNotFound_SkipsPackage() + public async Task RunAsync_WhenSourceNupkgIsNotFound_SkipsPackage() { var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json"); var nupkgUri = _catalogToDnxStorage.ResolveUri("/unlistedpackage/1.0.0/unlistedpackage.1.0.0.nupkg"); @@ -286,7 +285,7 @@ public async Task Run_WhenSourceNupkgIsNotFound_SkipsPackage() var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); ReadCursor back = MemoryCursor.CreateMax(); - await _target.Run(front, back, CancellationToken.None); + await _target.RunAsync(front, back, CancellationToken.None); Assert.Equal(1, _catalogToDnxStorage.Content.Count); Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); @@ -300,7 +299,7 @@ public async Task Run_WhenSourceNupkgIsNotFound_SkipsPackage() } [Fact] - public async Task Run_WithValidPackage_CreatesFlatContainer() + public async Task RunAsync_WithValidPackage_CreatesFlatContainer() { var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json"); var nupkgUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.0/listedpackage.1.0.0.nupkg"); @@ -318,7 +317,7 @@ public async Task Run_WithValidPackage_CreatesFlatContainer() var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); ReadCursor back = MemoryCursor.CreateMax(); - await _target.Run(front, back, CancellationToken.None); + await _target.RunAsync(front, back, CancellationToken.None); Assert.Equal(4, _catalogToDnxStorage.Content.Count); Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); @@ -338,7 +337,7 @@ public async Task Run_WithValidPackage_CreatesFlatContainer() } [Fact] - public async Task Run_WithValidPackage_RespectsDeletion() + public async Task RunAsync_WithValidPackage_RespectsDeletion() { var indexJsonUri = _catalogToDnxStorage.ResolveUri("/otherpackage/index.json"); var nupkgUri = _catalogToDnxStorage.ResolveUri("/otherpackage/1.0.0/otherpackage.1.0.0.nupkg"); @@ -354,7 +353,7 @@ public async Task Run_WithValidPackage_RespectsDeletion() var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); ReadCursor back = MemoryCursor.CreateMax(); - await _target.Run(front, back, CancellationToken.None); + await _target.RunAsync(front, back, CancellationToken.None); Assert.Equal(1, _catalogToDnxStorage.Content.Count); Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); @@ -368,7 +367,7 @@ public async Task Run_WithValidPackage_RespectsDeletion() } [Fact] - public async Task Run_WithPackageCreatedThenDeleted_LeavesNoArtifacts() + public async Task RunAsync_WithPackageCreatedThenDeleted_LeavesNoArtifacts() { var indexJsonUri = _catalogToDnxStorage.ResolveUri("/otherpackage/index.json"); var nupkgUri = _catalogToDnxStorage.ResolveUri("/otherpackage/1.0.0/otherpackage.1.0.0.nupkg"); @@ -384,7 +383,7 @@ public async Task Run_WithPackageCreatedThenDeleted_LeavesNoArtifacts() var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); ReadCursor back = MemoryCursor.CreateMax(); - await _target.Run(front, back, CancellationToken.None); + await _target.RunAsync(front, back, CancellationToken.None); Assert.Equal(1, _catalogToDnxStorage.Content.Count); Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); @@ -398,7 +397,7 @@ public async Task Run_WithPackageCreatedThenDeleted_LeavesNoArtifacts() } [Fact] - public async Task Run_WithNonIAzureStorage_WhenPackageIsAlreadySynchronizedAndHasRequiredProperties_SkipsPackage() + public async Task RunAsync_WithNonIAzureStorage_WhenPackageIsAlreadySynchronizedAndHasRequiredProperties_SkipsPackage() { _catalogToDnxStorage = new SynchronizedMemoryStorage(new[] { @@ -437,7 +436,7 @@ public async Task Run_WithNonIAzureStorage_WhenPackageIsAlreadySynchronizedAndHa var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); ReadCursor back = MemoryCursor.CreateMax(); - await _target.Run(front, back, CancellationToken.None); + await _target.RunAsync(front, back, CancellationToken.None); Assert.Equal(2, _catalogToDnxStorage.Content.Count); Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); @@ -453,7 +452,7 @@ public async Task Run_WithNonIAzureStorage_WhenPackageIsAlreadySynchronizedAndHa } [Fact] - public async Task Run_WithIAzureStorage_WhenPackageIsAlreadySynchronizedAndHasRequiredProperties_SkipsPackage() + public async Task RunAsync_WithIAzureStorage_WhenPackageIsAlreadySynchronizedAndHasRequiredProperties_SkipsPackage() { _catalogToDnxStorage = new AzureSynchronizedMemoryStorageStub(new[] { @@ -492,7 +491,7 @@ public async Task Run_WithIAzureStorage_WhenPackageIsAlreadySynchronizedAndHasRe var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); ReadCursor back = MemoryCursor.CreateMax(); - await _target.Run(front, back, CancellationToken.None); + await _target.RunAsync(front, back, CancellationToken.None); Assert.Equal(2, _catalogToDnxStorage.Content.Count); Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); @@ -508,7 +507,7 @@ public async Task Run_WithIAzureStorage_WhenPackageIsAlreadySynchronizedAndHasRe } [Fact] - public async Task Run_WithFakeIAzureStorage_WhenPackageIsAlreadySynchronizedButDoesNotHaveRequiredProperties_ProcessesPackage() + public async Task RunAsync_WithFakeIAzureStorage_WhenPackageIsAlreadySynchronizedButDoesNotHaveRequiredProperties_ProcessesPackage() { _catalogToDnxStorage = new AzureSynchronizedMemoryStorageStub(new[] { @@ -547,7 +546,7 @@ public async Task Run_WithFakeIAzureStorage_WhenPackageIsAlreadySynchronizedButD var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); ReadCursor back = MemoryCursor.CreateMax(); - await _target.Run(front, back, CancellationToken.None); + await _target.RunAsync(front, back, CancellationToken.None); Assert.Equal(4, _catalogToDnxStorage.Content.Count); Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); @@ -563,7 +562,7 @@ public async Task Run_WithFakeIAzureStorage_WhenPackageIsAlreadySynchronizedButD } [Fact] - public async Task Run_WhenPackageIsAlreadySynchronizedButNotInIndex_ProcessesPackage() + public async Task RunAsync_WhenPackageIsAlreadySynchronizedButNotInIndex_ProcessesPackage() { _catalogToDnxStorage = new SynchronizedMemoryStorage(new[] { @@ -598,7 +597,7 @@ public async Task Run_WhenPackageIsAlreadySynchronizedButNotInIndex_ProcessesPac var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); ReadCursor back = MemoryCursor.CreateMax(); - await _target.Run(front, back, CancellationToken.None); + await _target.RunAsync(front, back, CancellationToken.None); Assert.Equal(2, _catalogToDnxStorage.Content.Count); Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); @@ -620,7 +619,7 @@ public async Task Run_WhenPackageIsAlreadySynchronizedButNotInIndex_ProcessesPac [InlineData(HttpStatusCode.NoContent)] [InlineData(HttpStatusCode.InternalServerError)] [InlineData(HttpStatusCode.ServiceUnavailable)] - public async Task Run_WhenDownloadingPackage_RejectsUnexpectedHttpStatusCode(HttpStatusCode statusCode) + public async Task RunAsync_WhenDownloadingPackage_RejectsUnexpectedHttpStatusCode(HttpStatusCode statusCode) { var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete(); @@ -636,7 +635,7 @@ public async Task Run_WhenDownloadingPackage_RejectsUnexpectedHttpStatusCode(Htt ReadCursor back = MemoryCursor.CreateMax(); var exception = await Assert.ThrowsAsync( - () => _target.Run(front, back, CancellationToken.None)); + () => _target.RunAsync(front, back, CancellationToken.None)); Assert.Equal( $"Expected status code OK for package download, actual: {statusCode}", exception.Message); @@ -644,7 +643,7 @@ public async Task Run_WhenDownloadingPackage_RejectsUnexpectedHttpStatusCode(Htt } [Fact] - public async Task Run_WhenDownloadingPackage_OnlyDownloadsNupkgOncePerCatalogLeaf() + public async Task RunAsync_WhenDownloadingPackage_OnlyDownloadsNupkgOncePerCatalogLeaf() { // Arrange var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete(); @@ -663,12 +662,11 @@ public async Task Run_WhenDownloadingPackage_OnlyDownloadsNupkgOncePerCatalogLea "/packages/otherpackage.1.0.0.nupkg", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(File.OpenRead("Packages\\OtherPackage.1.0.0.zip")) })); - // Setup collector ReadWriteCursor front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); ReadCursor back = MemoryCursor.CreateMax(); // Act - await _target.Run(front, back, CancellationToken.None); + await _target.RunAsync(front, back, CancellationToken.None); // Assert Assert.Equal(9, _catalogToDnxStorage.Content.Count); @@ -694,7 +692,7 @@ public async Task Run_WhenDownloadingPackage_OnlyDownloadsNupkgOncePerCatalogLea [InlineData("/packages/unlistedpackage.1.0.0.nupkg", null)] [InlineData("/packages/listedpackage.1.0.1.nupkg", "2015-10-12T10:08:54.1506742Z")] [InlineData("/packages/anotherpackage.1.0.0.nupkg", "2015-10-12T10:08:54.1506742Z")] - public async Task Run_WhenExceptionOccurs_DoesNotSkipPackage(string catalogUri, string expectedCursorBeforeRetry) + public async Task RunAsync_WhenExceptionOccurs_DoesNotSkipPackage(string catalogUri, string expectedCursorBeforeRetry) { // Arrange var catalogStorage = Catalogs.CreateTestCatalogWithCommitThenTwoPackageCommit(); @@ -723,9 +721,9 @@ public async Task Run_WhenExceptionOccurs_DoesNotSkipPackage(string catalogUri, ReadCursor back = MemoryCursor.CreateMax(); // Act - await Assert.ThrowsAsync(() => _target.Run(front, back, CancellationToken.None)); + await Assert.ThrowsAsync(() => _target.RunAsync(front, back, CancellationToken.None)); var cursorBeforeRetry = front.Value; - await _target.Run(front, back, CancellationToken.None); + await _target.RunAsync(front, back, CancellationToken.None); var cursorAfterRetry = front.Value; // Assert @@ -749,7 +747,7 @@ public async Task Run_WhenExceptionOccurs_DoesNotSkipPackage(string catalogUri, } [Fact] - public async Task Run_WhenMultipleEntriesWithSamePackageIdentityInSameBatch_Throws() + public async Task RunAsync_WhenMultipleEntriesWithSamePackageIdentityInSameBatch_Throws() { var zipWithWrongNameNuspec = CreateZipStreamWithEntry("Newtonsoft.Json.nuspec", _nuspecData); var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json"); @@ -769,7 +767,7 @@ public async Task Run_WhenMultipleEntriesWithSamePackageIdentityInSameBatch_Thro ReadCursor back = MemoryCursor.CreateMax(); var exception = await Assert.ThrowsAsync( - () => _target.Run(front, back, CancellationToken.None)); + () => _target.RunAsync(front, back, CancellationToken.None)); Assert.Equal("The catalog batch 10/13/2015 6:40:07 AM contains multiple entries for the same package identity. Package(s): listedpackage 1.0.0", exception.Message); } diff --git a/tests/NgTests/Feed2CatalogTests.cs b/tests/NgTests/Feed2CatalogTests.cs index 91c6f0dad..8965914b5 100644 --- a/tests/NgTests/Feed2CatalogTests.cs +++ b/tests/NgTests/Feed2CatalogTests.cs @@ -497,7 +497,7 @@ public async Task CreatesNewCatalogFromCreatedAndEditedAndDeletedPackages() timeout: TimeSpan.FromMinutes(5), top: 20, verbose: true); - await feed2catalogTestJob.RunOnce(CancellationToken.None); + await feed2catalogTestJob.RunOnceAsync(CancellationToken.None); // Assert Assert.Equal(7, catalogStorage.Content.Count); @@ -600,7 +600,7 @@ public async Task AppendsDeleteToExistingCatalog() timeout: TimeSpan.FromMinutes(5), top: 20, verbose: true); - await feed2catalogTestJob.RunOnce(CancellationToken.None); + await feed2catalogTestJob.RunOnceAsync(CancellationToken.None); // Assert Assert.Equal(6, catalogStorage.Content.Count); @@ -696,7 +696,7 @@ public async Task AppendsDeleteAndReinsertToExistingCatalog() timeout: TimeSpan.FromMinutes(5), top: 20, verbose: true); - await feed2catalogTestJob.RunOnce(CancellationToken.None); + await feed2catalogTestJob.RunOnceAsync(CancellationToken.None); // Assert Assert.Equal(7, catalogStorage.Content.Count); @@ -798,7 +798,7 @@ public async Task RunInternal_CallsCatalogStorageLoadStringExactlyOnce() top: 20, verbose: true); - await feed2catalogTestJob.RunOnce(CancellationToken.None); + await feed2catalogTestJob.RunOnceAsync(CancellationToken.None); catalogStorage.Verify(x => x.ResolveUri(It.IsNotNull()), Times.AtLeastOnce()); catalogStorage.Verify(x => x.LoadStringAsync(It.IsNotNull(), It.IsAny()), Times.Once()); @@ -1039,7 +1039,7 @@ private PackageCreationOrEdit AddEditedPackageToFeed(PackageCreationOrEdit entry PrepareFeed(); - await _job.RunOnce(CancellationToken.None); + await _job.RunOnceAsync(CancellationToken.None); VerifyCatalog(expectedLastCreated, expectedLastDeleted, expectedLastEdited, skippedPackage); } diff --git a/tests/NgTests/Package2CatalogJobTests.cs b/tests/NgTests/Package2CatalogJobTests.cs index 829cc0eb8..aa461000c 100644 --- a/tests/NgTests/Package2CatalogJobTests.cs +++ b/tests/NgTests/Package2CatalogJobTests.cs @@ -104,7 +104,7 @@ protected override HttpClient CreateHttpClient() internal Task RunOnceAsync(CancellationToken cancellationToken) { - return RunInternal(cancellationToken); + return RunInternalAsync(cancellationToken); } } } diff --git a/tests/NgTests/RegistrationCollectorTests.cs b/tests/NgTests/RegistrationCollectorTests.cs index 8c18f8e2d..c7a85ae91 100644 --- a/tests/NgTests/RegistrationCollectorTests.cs +++ b/tests/NgTests/RegistrationCollectorTests.cs @@ -43,16 +43,13 @@ private void SharedInit(bool useLegacy, bool useSemVer2) _mockServer = new MockServerHttpClientHandler(); _mockServer.SetAction("/", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK))); - // Setup collector _target = new RegistrationCollector( new Uri("http://tempuri.org/index.json"), _legacyStorageFactory, _semVer2StorageFactory, + new Uri("http://tempuri.org/packages"), new Mock().Object, - handlerFunc: () => _mockServer) - { - ContentBaseAddress = new Uri("http://tempuri.org/packages") - }; + handlerFunc: () => _mockServer); RegistrationMakerCatalogItem.PackagePathProvider = new PackagesFolderPackagePathProvider(); } @@ -82,9 +79,9 @@ public async Task DoesNotSkipPackagesWhenExceptionOccurs(string catalogUri, stri ReadCursor back = MemoryCursor.CreateMax(); // Act - await Assert.ThrowsAsync(() => _target.Run(front, back, CancellationToken.None)); + await Assert.ThrowsAsync(() => _target.RunAsync(front, back, CancellationToken.None)); var cursorBeforeRetry = front.Value; - await _target.Run(front, back, CancellationToken.None); + await _target.RunAsync(front, back, CancellationToken.None); var cursorAfterRetry = front.Value; // Assert @@ -137,7 +134,7 @@ public async Task CreatesRegistrationsAndRespectsDeletes() ReadCursor back = MemoryCursor.CreateMax(); // Act - await _target.Run(front, back, CancellationToken.None); + await _target.RunAsync(front, back, CancellationToken.None); // Assert Assert.Equal(6, _legacyStorage.Content.Count); @@ -201,7 +198,7 @@ public async Task WhenPackageHasMultipleCommitsRespectsOrder(string pageContent) ReadCursor back = MemoryCursor.CreateMax(); // Act - await _target.Run(front, back, CancellationToken.None); + await _target.RunAsync(front, back, CancellationToken.None); // Assert Assert.Equal(3, _legacyStorage.Content.Count); @@ -258,7 +255,7 @@ public async Task CreatesRegistrationsWithSemVer2() var back = MemoryCursor.CreateMax(); // Act - await _target.Run(front, back, CancellationToken.None); + await _target.RunAsync(front, back, CancellationToken.None); // Assert // Verify the contents of the legacy (non-SemVer 2.0.0) storage @@ -305,7 +302,7 @@ public async Task IgnoresSemVer2PackagesInLegacyStorageWhenSemVer2IsEnabled() var back = MemoryCursor.CreateMax(); // Act - await _target.Run(front, back, CancellationToken.None); + await _target.RunAsync(front, back, CancellationToken.None); // Assert var legacyCursor = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("cursor.json")); @@ -338,7 +335,7 @@ public async Task PutsSemVer2PackagesInLegacyStorageWhenSemVer2IsDisabled() var back = MemoryCursor.CreateMax(); // Act - await _target.Run(front, back, CancellationToken.None); + await _target.RunAsync(front, back, CancellationToken.None); // Assert var legacyCursor = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("cursor.json")); @@ -362,7 +359,7 @@ public async Task HandlesDeleteCatalogItemWithNonNormalizedVersion() ReadWriteCursor front = new DurableCursor(_legacyStorage.ResolveUri("cursor.json"), _legacyStorage, MemoryCursor.MinValue); // Act - await _target.Run( + await _target.RunAsync( front, new MemoryCursor(DateTime.Parse("2015-10-12T10:08:54.1506742")), CancellationToken.None); @@ -371,7 +368,7 @@ public async Task HandlesDeleteCatalogItemWithNonNormalizedVersion() .Select(pair => pair.Key.ToString()) .OrderBy(uri => uri) .ToList(); - await _target.Run( + await _target.RunAsync( front, MemoryCursor.CreateMax(), CancellationToken.None); diff --git a/tests/NgTests/SearchIndexFromCatalogCollectorTests.cs b/tests/NgTests/SearchIndexFromCatalogCollectorTests.cs index f1c3d4fb8..a4957a337 100644 --- a/tests/NgTests/SearchIndexFromCatalogCollectorTests.cs +++ b/tests/NgTests/SearchIndexFromCatalogCollectorTests.cs @@ -66,9 +66,9 @@ public async Task DoesNotSkipPackagesWhenExceptionOccurs(string catalogUri, stri handlerFunc: () => mockServer); // Act - await Assert.ThrowsAsync(() => target.Run(front, back, CancellationToken.None)); + await Assert.ThrowsAsync(() => target.RunAsync(front, back, CancellationToken.None)); var cursorBeforeRetry = front.Value; - await target.Run(front, back, CancellationToken.None); + await target.RunAsync(front, back, CancellationToken.None); var cursorAfterRetry = front.Value; // Assert diff --git a/tests/NgTests/SortingIdVersionCollectorTests.cs b/tests/NgTests/SortingIdVersionCollectorTests.cs index fce5cf2bd..8db71735b 100644 --- a/tests/NgTests/SortingIdVersionCollectorTests.cs +++ b/tests/NgTests/SortingIdVersionCollectorTests.cs @@ -88,7 +88,7 @@ public async Task OnProcessBatch_BatchesCorrectly(IEnumerable items) }); // Act - var result = await collectorMock.Object.OnProcessBatch(items); + var result = await collectorMock.Object.OnProcessBatchAsync(items); } private static JToken CreatePackage(string id, string version) @@ -110,9 +110,9 @@ public TestableSortingIdVersionCollector() { } - public Task OnProcessBatch(IEnumerable items) + public Task OnProcessBatchAsync(IEnumerable items) { - return base.OnProcessBatch(null, items, null, DateTime.MinValue, false, CancellationToken.None); + return base.OnProcessBatchAsync(null, items, null, DateTime.MinValue, false, CancellationToken.None); } protected override Task ProcessSortedBatchAsync( diff --git a/tests/NgTests/TestableFeed2CatalogJob.cs b/tests/NgTests/TestableFeed2CatalogJob.cs index 417a1c95b..1d254b479 100644 --- a/tests/NgTests/TestableFeed2CatalogJob.cs +++ b/tests/NgTests/TestableFeed2CatalogJob.cs @@ -47,9 +47,9 @@ protected override HttpClient CreateHttpClient() return new HttpClient(_handler); } - public async Task RunOnce(CancellationToken cancellationToken) + public async Task RunOnceAsync(CancellationToken cancellationToken) { - await RunInternal(cancellationToken); + await RunInternalAsync(cancellationToken); } }