Skip to content

Commit

Permalink
Datamovement tests ShareUploadDirectory (#39718)
Browse files Browse the repository at this point in the history
* initial testbase implementation

* hook up shares to testbase, some failures

* fixes

* fix a test

* get root vs nonroot tests working together

* fix build

* cleanup

* testproxy

* rerecord

* rerecord

* rerecord (exceeding file path limits)

* change path generation, hopefully fix linux tests

* recorded test attr

* disable bad test

* use system path separator

* bugfix
  • Loading branch information
jaschrep-msft committed Nov 10, 2023
1 parent bb20017 commit 19c5fd2
Show file tree
Hide file tree
Showing 8 changed files with 638 additions and 71 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@
"AssetsRepo": "Azure/azure-sdk-assets",
"AssetsRepoPrefixPath": "net",
"TagPrefix": "net/storage/Azure.Storage.DataMovement.Files.Shares",
"Tag": "net/storage/Azure.Storage.DataMovement.Files.Shares_876c524ce2"
"Tag": "net/storage/Azure.Storage.DataMovement.Files.Shares_4e52c7c39c"
}
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
<Compile Include="$(AzureStorageDataMovementTestSharedSources)TransferValidator.Local.cs" LinkBase="Shared\DataMovement" />
<Compile Include="$(AzureStorageDataMovementTestSharedSources)StartTransferUploadTestBase.cs" LinkBase="Shared\DataMovement" />
<Compile Include="$(AzureStorageDataMovementTestSharedSources)StartTransferDirectoryCopyTestBase.cs" LinkBase="Shared\DataMovement" />
<Compile Include="$(AzureStorageDataMovementTestSharedSources)StartTransferUploadDirectoryTestBase.cs" LinkBase="Shared\DataMovement" />
<Compile Include="$(AzureStorageDataMovementTestSharedSources)StartTransferCopyTestBase.cs" LinkBase="Shared\DataMovement" />
<Compile Include="$(AzureStorageDataMovementTestSharedSources)StartTransferDownloadTestBase.cs" LinkBase="Shared\DataMovement" />
<Compile Include="$(AzureStorageDataMovementTestSharedSources)StartTransferDirectoryDownloadTestBase.cs" LinkBase="Shared\DataMovement" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ protected override async Task<IDisposingContainer<ShareClient>> GetDisposingCont
}

protected override TransferValidator.ListFilesAsync GetSourceLister(ShareClient container, string prefix)
=> TransferValidator.GetShareFilesLister(container, prefix);
=> TransferValidator.GetShareFileLister(container.GetDirectoryClient(prefix));

protected override StorageResourceContainer GetStorageResourceContainer(ShareClient container, string directoryPath)
{
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Azure.Storage.DataMovement.Tests;
using Azure.Storage.Files.Shares;
using Azure.Storage.Files.Shares.Tests;
using Azure.Storage.Test.Shared;

namespace Azure.Storage.DataMovement.Files.Shares.Tests
{
[ShareClientTestFixture(true)]
[ShareClientTestFixture(false)]
internal class ShareDirectoryStartTransferUploadTests : StartTransferUploadDirectoryTestBase<
ShareServiceClient,
ShareDirectoryClient,
ShareFileClient,
ShareClientOptions,
StorageTestEnvironment>
{
/// <summary>
/// A <see cref="DisposingShare"/> but exposes a directory client within that share.
/// Still cleans up the whole share. Helpful for parameterizing tests to use a root
/// directory vs a subdir.
/// </summary>
private class DisposingShareDirectory : IDisposingContainer<ShareDirectoryClient>
{
private readonly DisposingShare _disposingShare;
public ShareDirectoryClient Container { get; }

public DisposingShareDirectory(DisposingShare disposingShare, ShareDirectoryClient dirClient)
{
_disposingShare = disposingShare;
Container = dirClient;
}

public async ValueTask DisposeAsync()
{
if (_disposingShare != default)
{
await _disposingShare.DisposeAsync();
}
}
}

public bool UseNonRootDirectory { get; }

public ShareDirectoryStartTransferUploadTests(bool async, ShareClientOptions.ServiceVersion serviceVersion, bool useNonRootDirectory)
: base(async, null /* RecordedTestMode.Record /* to re-record */)
{
ClientBuilder = ClientBuilderExtensions.GetNewShareClientBuilder(Tenants, serviceVersion);
UseNonRootDirectory = useNonRootDirectory;
}

protected override async Task<IDisposingContainer<ShareDirectoryClient>> GetDisposingContainerAsync(ShareServiceClient service = null, string containerName = null)
{
DisposingShare disposingShare = await ClientBuilder.GetTestShareAsync(service, containerName);
ShareDirectoryClient directoryClient = disposingShare.Container.GetRootDirectoryClient();
if (UseNonRootDirectory)
{
foreach (var _ in Enumerable.Range(0, 2))
{
directoryClient = directoryClient.GetSubdirectoryClient(GetNewObjectName());
await directoryClient.CreateAsync();
}
}
return new DisposingShareDirectory(disposingShare, directoryClient);
}

protected override StorageResourceContainer GetStorageResourceContainer(ShareDirectoryClient containerClient)
{
return new ShareDirectoryStorageResourceContainer(containerClient, null);
}

protected override TransferValidator.ListFilesAsync GetStorageResourceLister(ShareDirectoryClient containerClient)
{
return TransferValidator.GetShareFileLister(containerClient);
}

protected override async Task InitializeDestinationDataAsync(ShareDirectoryClient containerClient, List<(string FilePath, long Size)> fileSizes, CancellationToken cancellationToken)
{
foreach ((string filePath, long size) in fileSizes)
{
ShareDirectoryClient directory = containerClient;

string[] pathSegments = filePath.Split('/');
foreach (string pathSegment in pathSegments.Take(pathSegments.Length - 1))
{
directory = directory.GetSubdirectoryClient(pathSegment);
await directory.CreateIfNotExistsAsync(cancellationToken: cancellationToken);
}
ShareFileClient file = directory.GetFileClient(pathSegments.Last());
await file.CreateAsync(size, cancellationToken: cancellationToken);
await file.UploadAsync(await CreateLimitedMemoryStream(size), cancellationToken: cancellationToken);
}
}
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

using System;
using System.Collections.Generic;
using System.IO;
using System.Threading;
Expand All @@ -12,13 +13,13 @@ namespace Azure.Storage.DataMovement.Tests
{
public partial class TransferValidator
{
public class ShareResourceEnumerationItem : IResourceEnumerationItem
public class ShareFileResourceEnumerationItem : IResourceEnumerationItem
{
private readonly ShareFileClient _client;

public string RelativePath { get; }

public ShareResourceEnumerationItem(ShareFileClient client, string relativePath)
public ShareFileResourceEnumerationItem(ShareFileClient client, string relativePath)
{
_client = client;
RelativePath = relativePath;
Expand All @@ -30,66 +31,39 @@ public async Task<Stream> OpenReadAsync(CancellationToken cancellationToken)
}
}

public static ListFilesAsync GetShareFilesLister(ShareClient container, string prefix)
public static ListFilesAsync GetShareFileLister(ShareDirectoryClient container)
{
async Task<List<IResourceEnumerationItem>> ListFiles(CancellationToken cancellationToken)
async Task<List<IResourceEnumerationItem>> ListFilesRecursive(ShareDirectoryClient dir, CancellationToken cancellationToken)
{
List<IResourceEnumerationItem> result = new();
ShareDirectoryClient directory = string.IsNullOrEmpty(prefix) ?
container.GetRootDirectoryClient() :
container.GetDirectoryClient(prefix);

Queue<ShareDirectoryClient> toScan = new();
toScan.Enqueue(directory);

while (toScan.Count > 0)
await foreach (ShareFileItem fileItem in dir.GetFilesAndDirectoriesAsync(cancellationToken: cancellationToken))
{
ShareDirectoryClient current = toScan.Dequeue();
await foreach (ShareFileItem item in current.GetFilesAndDirectoriesAsync(
cancellationToken: cancellationToken).ConfigureAwait(false))
if (fileItem.IsDirectory)
{
result.AddRange(await ListFilesRecursive(dir.GetSubdirectoryClient(fileItem.Name), cancellationToken));
}
else
{
if (item.IsDirectory)
{
ShareDirectoryClient subdir = current.GetSubdirectoryClient(item.Name);
toScan.Enqueue(subdir);
}
else
{
string relativePath = "";
if (string.IsNullOrEmpty(current.Path))
{
relativePath = item.Name;
}
else if (string.IsNullOrEmpty(prefix))
{
relativePath = string.Join("/", current.Path, item.Name);
}
else
{
relativePath =
prefix != current.Name ?
string.Join("/", current.Path.Substring(prefix.Length + 1), item.Name) :
item.Name;
}
result.Add(new ShareResourceEnumerationItem(current.GetFileClient(item.Name), relativePath));
}
ShareFileClient fileClient = dir.GetFileClient(fileItem.Name);
result.Add(new ShareFileResourceEnumerationItem(
fileClient, fileClient.Path.Substring(container.Path.Length).Trim('/')));
}
}
return result;
}
return ListFiles;
return (cancellationToken) => ListFilesRecursive(container, cancellationToken);
}

public static ListFilesAsync GetFileListerSingle(ShareFileClient file, string relativePath)
public static ListFilesAsync GetShareFileListerSingle(ShareFileClient file, string relativePath)
{
Task<List<IResourceEnumerationItem>> ListFiles(CancellationToken cancellationToken)
Task<List<IResourceEnumerationItem>> ListFile(CancellationToken cancellationToken)
{
return Task.FromResult(new List<IResourceEnumerationItem>
{
new ShareResourceEnumerationItem(file, relativePath)
new ShareFileResourceEnumerationItem(file, relativePath)
});
}
return ListFiles;
return ListFile;
}
}
}

0 comments on commit 19c5fd2

Please sign in to comment.