From 2b82dafd098ca5f4a8d97db7899b5e84992c28c3 Mon Sep 17 00:00:00 2001 From: Mathew Charles Date: Tue, 14 Jul 2015 18:07:37 -0700 Subject: [PATCH] Fixing a bug in Blob scan continuation token handling --- .../Listeners/StorageBlobClientExtensions.cs | 11 +- .../StorageBlobContainerExtensions.cs | 9 +- .../StorageBlobClientExtensionsTests.cs | 103 ++++++++++++++++++ .../StorageBlobContainerExtensionsTests.cs | 103 ++++++++++++++++++ .../WebJobs.Host.UnitTests.csproj | 2 + 5 files changed, 219 insertions(+), 9 deletions(-) create mode 100644 test/Microsoft.Azure.WebJobs.Host.UnitTests/Blobs/Listeners/StorageBlobClientExtensionsTests.cs create mode 100644 test/Microsoft.Azure.WebJobs.Host.UnitTests/Blobs/Listeners/StorageBlobContainerExtensionsTests.cs diff --git a/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/StorageBlobClientExtensions.cs b/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/StorageBlobClientExtensions.cs index fe921e355..805804c11 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/StorageBlobClientExtensions.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/StorageBlobClientExtensions.cs @@ -22,26 +22,27 @@ internal static class StorageBlobClientExtensions } List allResults = new List(); - BlobContinuationToken currentToken = null; + BlobContinuationToken continuationToken = null; IStorageBlobResultSegment result; do { result = await client.ListBlobsSegmentedAsync(prefix, useFlatBlobListing, blobListingDetails, - maxResults: null, currentToken: currentToken, options: null, operationContext: null, + maxResults: null, currentToken: continuationToken, options: null, operationContext: null, cancellationToken: cancellationToken); if (result != null) { IEnumerable currentResults = result.Results; - if (currentResults != null) { allResults.AddRange(currentResults); } + + continuationToken = result.ContinuationToken; } - } - while (result != null && result.ContinuationToken != null); + } + while (result != null && continuationToken != null); return allResults; } diff --git a/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/StorageBlobContainerExtensions.cs b/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/StorageBlobContainerExtensions.cs index 63365e317..8eded44a7 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/StorageBlobContainerExtensions.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/StorageBlobContainerExtensions.cs @@ -21,26 +21,27 @@ internal static class StorageBlobContainerExtensions } List allResults = new List(); - BlobContinuationToken currentToken = null; + BlobContinuationToken continuationToken = null; IStorageBlobResultSegment result; do { result = await container.ListBlobsSegmentedAsync(prefix: null, useFlatBlobListing: useFlatBlobListing, - blobListingDetails: BlobListingDetails.None, maxResults: null, currentToken: currentToken, + blobListingDetails: BlobListingDetails.None, maxResults: null, currentToken: continuationToken, options: null, operationContext: null, cancellationToken: cancellationToken); if (result != null) { IEnumerable currentResults = result.Results; - if (currentResults != null) { allResults.AddRange(currentResults); } + + continuationToken = result.ContinuationToken; } } - while (result != null && result.ContinuationToken != null); + while (result != null && continuationToken != null); return allResults; } diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Blobs/Listeners/StorageBlobClientExtensionsTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Blobs/Listeners/StorageBlobClientExtensionsTests.cs new file mode 100644 index 000000000..cf01d233e --- /dev/null +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Blobs/Listeners/StorageBlobClientExtensionsTests.cs @@ -0,0 +1,103 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Azure.WebJobs.Host.Blobs.Listeners; +using Microsoft.Azure.WebJobs.Host.Storage.Blob; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; +using Moq; +using Xunit; + +namespace Microsoft.Azure.WebJobs.Host.UnitTests.Blobs.Listeners +{ + public class StorageBlobClientExtensionsTests + { + [Theory] + [InlineData(4000)] + [InlineData(30000)] + public async Task ListBlobsAsync_FollowsContinuationTokensToEnd(int blobCount) + { + Mock mockClient = new Mock(MockBehavior.Strict); + + int maxResults = 5000; + List blobs = GetMockBlobs(blobCount); + int numPages = (int)Math.Ceiling(((decimal)blobCount / maxResults)); + + // create the test data pages with continuation tokens + List blobSegments = new List(); + IStorageBlobResultSegment initialSegement = null; + for (int i = 0; i < numPages; i++) + { + BlobContinuationToken continuationToken = null; + if (i < (numPages - 1)) + { + // add a token for all but the last page + continuationToken = new BlobContinuationToken() + { + NextMarker = i.ToString() + }; + } + + Mock mockSegment = new Mock(MockBehavior.Strict); + mockSegment.SetupGet(p => p.Results).Returns(blobs.Skip(i * maxResults).Take(maxResults).ToArray()); + mockSegment.SetupGet(p => p.ContinuationToken).Returns(continuationToken); + + if (i == 0) + { + initialSegement = mockSegment.Object; + } + else + { + blobSegments.Add(mockSegment.Object); + } + } + + // Mock the List function to return the correct blob page + HashSet seenTokens = new HashSet(); + IStorageBlobResultSegment resultSegment = null; + mockClient.Setup(p => p.ListBlobsSegmentedAsync("test", true, BlobListingDetails.None, null, It.IsAny(), null, null, CancellationToken.None)) + .Callback( + (prefix, useFlatBlobListing, blobListingDetails, maxResultsValue, currentToken, options, operationContext, cancellationToken) => + { + // Previously this is where a bug existed - ListBlobsAsync wasn't handling + // continuation tokens properly and kept sending the same initial token + Assert.False(seenTokens.Contains(currentToken)); + seenTokens.Add(currentToken); + + if (currentToken == null) + { + resultSegment = initialSegement; + } + else + { + int idx = int.Parse(currentToken.NextMarker); + resultSegment = blobSegments[idx]; + } + }) + .Returns(() => { return Task.FromResult(resultSegment); }); + + IEnumerable results = await mockClient.Object.ListBlobsAsync("test", true, BlobListingDetails.None, CancellationToken.None); + + Assert.Equal(blobCount, results.Count()); + } + + private class FakeBlobItem : IStorageListBlobItem + { + } + + private List GetMockBlobs(int count) + { + List blobs = new List(); + for (int i = 0; i < count; i++) + { + blobs.Add(new FakeBlobItem()); + } + return blobs; + } + } +} diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Blobs/Listeners/StorageBlobContainerExtensionsTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Blobs/Listeners/StorageBlobContainerExtensionsTests.cs new file mode 100644 index 000000000..07b0dbfbf --- /dev/null +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Blobs/Listeners/StorageBlobContainerExtensionsTests.cs @@ -0,0 +1,103 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Azure.WebJobs.Host.Blobs.Listeners; +using Microsoft.Azure.WebJobs.Host.Storage.Blob; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; +using Moq; +using Xunit; + +namespace Microsoft.Azure.WebJobs.Host.UnitTests.Blobs.Listeners +{ + public class StorageBlobContainerExtensionsTests + { + [Theory] + [InlineData(4000)] + [InlineData(30000)] + public async Task ListBlobsAsync_FollowsContinuationTokensToEnd(int blobCount) + { + Mock mockContainer = new Mock(MockBehavior.Strict); + + int maxResults = 5000; + List blobs = GetMockBlobs(blobCount); + int numPages = (int)Math.Ceiling(((decimal)blobCount / maxResults)); + + // create the test data pages with continuation tokens + List blobSegments = new List(); + IStorageBlobResultSegment initialSegement = null; + for (int i = 0; i < numPages; i++) + { + BlobContinuationToken continuationToken = null; + if (i < (numPages - 1)) + { + // add a token for all but the last page + continuationToken = new BlobContinuationToken() + { + NextMarker = i.ToString() + }; + } + + Mock mockSegment = new Mock(MockBehavior.Strict); + mockSegment.SetupGet(p => p.Results).Returns(blobs.Skip(i * maxResults).Take(maxResults).ToArray()); + mockSegment.SetupGet(p => p.ContinuationToken).Returns(continuationToken); + + if (i == 0) + { + initialSegement = mockSegment.Object; + } + else + { + blobSegments.Add(mockSegment.Object); + } + } + + // Mock the List function to return the correct blob page + HashSet seenTokens = new HashSet(); + IStorageBlobResultSegment resultSegment = null; + mockContainer.Setup(p => p.ListBlobsSegmentedAsync(null, true, BlobListingDetails.None, null, It.IsAny(), null, null, CancellationToken.None)) + .Callback( + (prefix, useFlatBlobListing, blobListingDetails, maxResultsValue, currentToken, options, operationContext, cancellationToken) => + { + // Previously this is where a bug existed - ListBlobsAsync wasn't handling + // continuation tokens properly and kept sending the same initial token + Assert.False(seenTokens.Contains(currentToken)); + seenTokens.Add(currentToken); + + if (currentToken == null) + { + resultSegment = initialSegement; + } + else + { + int idx = int.Parse(currentToken.NextMarker); + resultSegment = blobSegments[idx]; + } + }) + .Returns(() => { return Task.FromResult(resultSegment); }); + + IEnumerable results = await mockContainer.Object.ListBlobsAsync(true, CancellationToken.None); + + Assert.Equal(blobCount, results.Count()); + } + + private class FakeBlobItem : IStorageListBlobItem + { + } + + private List GetMockBlobs(int count) + { + List blobs = new List(); + for (int i = 0; i < count; i++) + { + blobs.Add(new FakeBlobItem()); + } + return blobs; + } + } +} diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/WebJobs.Host.UnitTests.csproj b/test/Microsoft.Azure.WebJobs.Host.UnitTests/WebJobs.Host.UnitTests.csproj index e25e6fad8..eadc2b00f 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/WebJobs.Host.UnitTests.csproj +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/WebJobs.Host.UnitTests.csproj @@ -118,6 +118,8 @@ + +