Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #47 from AzureCosmosDB/develop
Adding prototype implementation of split storage+formatter extension
- Loading branch information
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| using Cosmos.DataTransfer.Interfaces; | ||
| using Microsoft.Extensions.Configuration; | ||
| using Microsoft.Extensions.Logging; | ||
|
|
||
| namespace Cosmos.DataTransfer.Common; | ||
|
|
||
| public abstract class CompositeSinkExtension<TSink, TFormatter> : IDataSinkExtension | ||
| where TSink : class, IComposableDataSink, new() | ||
| where TFormatter : class, IFormattedDataWriter, new() | ||
| { | ||
| public abstract string DisplayName { get; } | ||
|
|
||
| public async Task WriteAsync(IAsyncEnumerable<IDataItem> dataItems, IConfiguration config, IDataSourceExtension dataSource, ILogger logger, CancellationToken cancellationToken = default) | ||
| { | ||
| var sink = new TSink(); | ||
| var formatter = new TFormatter(); | ||
|
|
||
| await sink.WriteToTargetAsync(formatter, dataItems, config, dataSource, logger, cancellationToken); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| using Cosmos.DataTransfer.Interfaces; | ||
| using Microsoft.Extensions.Configuration; | ||
| using Microsoft.Extensions.Logging; | ||
|
|
||
| namespace Cosmos.DataTransfer.Common | ||
| { | ||
| public abstract class CompositeSourceExtension<TSource, TFormatter> : IDataSourceExtension | ||
| where TSource : class, IComposableDataSource, new() | ||
| where TFormatter : class, IFormattedDataReader, new() | ||
| { | ||
| public abstract string DisplayName { get; } | ||
|
|
||
| public IAsyncEnumerable<IDataItem> ReadAsync(IConfiguration config, ILogger logger, CancellationToken cancellationToken = default) | ||
| { | ||
| var source = new TSource(); | ||
| var formatter = new TFormatter(); | ||
|
|
||
| return formatter.ParseDataAsync(source, config, logger, cancellationToken); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
|
|
||
| <PropertyGroup> | ||
| <TargetFramework>net6.0</TargetFramework> | ||
| <ImplicitUsings>enable</ImplicitUsings> | ||
| <Nullable>enable</Nullable> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="6.0.0" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <ProjectReference Include="..\Interfaces\Cosmos.DataTransfer.Interfaces\Cosmos.DataTransfer.Interfaces.csproj" /> | ||
| </ItemGroup> | ||
|
|
||
| </Project> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| using Cosmos.DataTransfer.Interfaces; | ||
| using Microsoft.Extensions.Configuration; | ||
| using Microsoft.Extensions.Logging; | ||
|
|
||
| namespace Cosmos.DataTransfer.Common; | ||
|
|
||
| public class FileDataSink : IComposableDataSink | ||
| { | ||
| public async Task WriteToTargetAsync(IFormattedDataWriter dataWriter, IAsyncEnumerable<IDataItem> dataItems, IConfiguration config, IDataSourceExtension dataSource, ILogger logger, CancellationToken cancellationToken = default) | ||
| { | ||
| var settings = config.Get<FileSinkSettings>(); | ||
| settings.Validate(); | ||
| if (settings.FilePath != null) | ||
| { | ||
| await using var writer = File.Create(settings.FilePath); | ||
| await dataWriter.FormatDataAsync(dataItems, writer, config, logger, cancellationToken); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,56 @@ | ||
| using Cosmos.DataTransfer.Interfaces; | ||
| using Microsoft.Extensions.Configuration; | ||
| using Microsoft.Extensions.Logging; | ||
|
|
||
| namespace Cosmos.DataTransfer.Common; | ||
|
|
||
| public class FileDataSource : IComposableDataSource | ||
| { | ||
| public async IAsyncEnumerable<Stream?> ReadSourceAsync(IConfiguration config, ILogger logger, CancellationToken cancellationToken = default) | ||
|
Check warning on line 9 in Cosmos.DataTransfer.Common/FileDataSource.cs
|
||
| { | ||
| var settings = config.Get<FileSourceSettings>(); | ||
| settings.Validate(); | ||
| if (settings.FilePath != null) | ||
| { | ||
|
|
||
| if (File.Exists(settings.FilePath)) | ||
| { | ||
| logger.LogInformation("Reading file '{FilePath}'", settings.FilePath); | ||
| yield return File.OpenRead(settings.FilePath); | ||
| } | ||
| else if (Directory.Exists(settings.FilePath)) | ||
| { | ||
| string[] files = Directory.GetFiles(settings.FilePath, "*.json", SearchOption.AllDirectories); | ||
| logger.LogInformation("Reading {FileCount} files from '{Folder}'", files.Length, settings.FilePath); | ||
| foreach (string filePath in files.OrderBy(f => f)) | ||
| { | ||
| logger.LogInformation("Reading file '{FilePath}'", filePath); | ||
| yield return File.OpenRead(filePath); | ||
| } | ||
| } | ||
| else if (Uri.IsWellFormedUriString(settings.FilePath, UriKind.RelativeOrAbsolute)) | ||
| { | ||
| logger.LogInformation("Reading from URI '{FilePath}'", settings.FilePath); | ||
|
|
||
| HttpClient client = new HttpClient(); | ||
| var response = await client.GetAsync(settings.FilePath, cancellationToken); | ||
| if (!response.IsSuccessStatusCode) | ||
| { | ||
| logger.LogError("Failed to read {FilePath}. Response was: {ResponseCode} {ResponseMessage}", settings.FilePath, response.StatusCode, response.ReasonPhrase); | ||
| yield break; | ||
| } | ||
|
|
||
| var json = await response.Content.ReadAsStreamAsync(cancellationToken); | ||
|
|
||
| yield return json; | ||
| } | ||
| else | ||
| { | ||
| logger.LogWarning("No content was found at configured path '{FilePath}'", settings.FilePath); | ||
| yield break; | ||
| } | ||
|
|
||
| logger.LogInformation("Completed reading '{FilePath}'", settings.FilePath); | ||
| } | ||
| } | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| using System.ComponentModel.DataAnnotations; | ||
| using Cosmos.DataTransfer.Interfaces; | ||
|
|
||
| namespace Cosmos.DataTransfer.Common; | ||
|
|
||
| public class FileSinkSettings : IDataExtensionSettings | ||
| { | ||
| [Required] | ||
| public string? FilePath { get; set; } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| using System.ComponentModel.DataAnnotations; | ||
| using Cosmos.DataTransfer.Interfaces; | ||
|
|
||
| namespace Cosmos.DataTransfer.Common; | ||
|
|
||
| public class FileSourceSettings : IDataExtensionSettings | ||
| { | ||
| [Required] | ||
| public string? FilePath { get; set; } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,86 @@ | ||
| using Microsoft.Extensions.Logging.Abstractions; | ||
| using Newtonsoft.Json.Linq; | ||
|
|
||
| namespace Cosmos.DataTransfer.JsonExtension.UnitTests | ||
| { | ||
| [TestClass] | ||
| public class JsonFileRoundTripTests | ||
| { | ||
| [TestMethod] | ||
| public async Task WriteAsync_fromReadAsync_ProducesIdenticalFile() | ||
| { | ||
| var input = new JsonFileSource(); | ||
| var output = new JsonFileSink(); | ||
|
|
||
| const string fileIn = "Data/ArraysTypesNesting.json"; | ||
| const string fileOut = $"{nameof(WriteAsync_fromReadAsync_ProducesIdenticalFile)}_out.json"; | ||
|
|
||
| var sourceConfig = TestHelpers.CreateConfig(new Dictionary<string, string> | ||
| { | ||
| { "FilePath", fileIn } | ||
| }); | ||
| var sinkConfig = TestHelpers.CreateConfig(new Dictionary<string, string> | ||
| { | ||
| { "FilePath", fileOut }, | ||
| { "Indented", "true" }, | ||
| }); | ||
|
|
||
| await output.WriteAsync(input.ReadAsync(sourceConfig, NullLogger.Instance), sinkConfig, input, NullLogger.Instance); | ||
|
|
||
| bool areEqual = JToken.DeepEquals(JToken.Parse(await File.ReadAllTextAsync(fileIn)), JToken.Parse(await File.ReadAllTextAsync(fileOut))); | ||
| Assert.IsTrue(areEqual); | ||
| } | ||
|
|
||
| [TestMethod] | ||
| public async Task WriteAsync_fromFolderReadAsync_ProducesExpectedCombinedFile() | ||
| { | ||
| var input = new JsonFileSource(); | ||
| var output = new JsonFileSink(); | ||
|
|
||
| const string fileIn = "Data/SingleObjects"; | ||
| const string fileCompare = "Data/SimpleIdName.json"; | ||
| const string fileOut = $"{nameof(WriteAsync_fromFolderReadAsync_ProducesExpectedCombinedFile)}_out.json"; | ||
|
|
||
| var sourceConfig = TestHelpers.CreateConfig(new Dictionary<string, string> | ||
| { | ||
| { "FilePath", fileIn } | ||
| }); | ||
| var sinkConfig = TestHelpers.CreateConfig(new Dictionary<string, string> | ||
| { | ||
| { "FilePath", fileOut }, | ||
| { "Indented", "true" }, | ||
| }); | ||
|
|
||
| await output.WriteAsync(input.ReadAsync(sourceConfig, NullLogger.Instance), sinkConfig, input, NullLogger.Instance); | ||
|
|
||
| bool areEqual = JToken.DeepEquals(JToken.Parse(await File.ReadAllTextAsync(fileCompare)), JToken.Parse(await File.ReadAllTextAsync(fileOut))); | ||
| Assert.IsTrue(areEqual); | ||
| } | ||
|
|
||
| [TestMethod] | ||
| public async Task WriteAsync_fromReadUriAsync_ProducesIdenticalFile() | ||
| { | ||
| var input = new JsonFileSource(); | ||
| var output = new JsonFileSink(); | ||
|
|
||
| const string urlIn = "https://raw.githubusercontent.com/AzureCosmosDB/data-migration-desktop-tool/main/Extensions/Json/Cosmos.DataTransfer.JsonExtension.UnitTests/Data/ArraysTypesNesting.json"; | ||
| const string compareFile = "Data/ArraysTypesNesting.json"; | ||
| const string fileOut = $"{nameof(WriteAsync_fromReadAsync_ProducesIdenticalFile)}_out.json"; | ||
|
|
||
| var sourceConfig = TestHelpers.CreateConfig(new Dictionary<string, string> | ||
| { | ||
| { "FilePath", urlIn } | ||
| }); | ||
| var sinkConfig = TestHelpers.CreateConfig(new Dictionary<string, string> | ||
| { | ||
| { "FilePath", fileOut }, | ||
| { "Indented", "true" }, | ||
| }); | ||
|
|
||
| await output.WriteAsync(input.ReadAsync(sourceConfig, NullLogger.Instance), sinkConfig, input, NullLogger.Instance); | ||
|
|
||
| bool areEqual = JToken.DeepEquals(JToken.Parse(await File.ReadAllTextAsync(compareFile)), JToken.Parse(await File.ReadAllTextAsync(fileOut))); | ||
| Assert.IsTrue(areEqual); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| using Cosmos.DataTransfer.Interfaces; | ||
| using Microsoft.Extensions.Logging.Abstractions; | ||
| using Newtonsoft.Json; | ||
|
|
||
| namespace Cosmos.DataTransfer.JsonExtension.UnitTests | ||
| { | ||
| [TestClass] | ||
| public class JsonFileSinkTests | ||
| { | ||
| [TestMethod] | ||
| public async Task WriteAsync_WithFlatObjects_WritesToValidFile() | ||
| { | ||
| var sink = new JsonFileSink(); | ||
|
|
||
| var data = new List<DictionaryDataItem> | ||
| { | ||
| new(new Dictionary<string, object?> | ||
| { | ||
| { "Id", 1 }, | ||
| { "Name", "One" }, | ||
| }), | ||
| new(new Dictionary<string, object?> | ||
| { | ||
| { "Id", 2 }, | ||
| { "Name", "Two" }, | ||
| }), | ||
| new(new Dictionary<string, object?> | ||
| { | ||
| { "Id", 3 }, | ||
| { "Name", "Three" }, | ||
| }), | ||
| }; | ||
| string outputFile = $"{DateTime.Now:yy-MM-dd}_FS_Output.json"; | ||
| var config = TestHelpers.CreateConfig(new Dictionary<string, string> | ||
| { | ||
| { "FilePath", outputFile } | ||
| }); | ||
|
|
||
| await sink.WriteAsync(data.ToAsyncEnumerable(), config, new JsonDataSourceExtension(), NullLogger.Instance); | ||
|
|
||
| var outputData = JsonConvert.DeserializeObject<List<TestDataObject>>(await File.ReadAllTextAsync(outputFile)); | ||
|
|
||
| Assert.IsTrue(outputData.Any(o => o.Id == 1 && o.Name == "One")); | ||
| Assert.IsTrue(outputData.Any(o => o.Id == 2 && o.Name == "Two")); | ||
| Assert.IsTrue(outputData.Any(o => o.Id == 3 && o.Name == "Three")); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| namespace Cosmos.DataTransfer.JsonExtension.UnitTests | ||
| { | ||
| public class TestDataObject | ||
| { | ||
| public int Id { get; set; } | ||
| public string? Name { get; set; } | ||
| public DateTime? Created { get; set; } | ||
| public List<DateTime>? Dates { get; set; } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.