From 726442cbc577586d5c72c66fac628fdec4d3145e Mon Sep 17 00:00:00 2001 From: dotnetprog <24593889+dotnetprog@users.noreply.github.com> Date: Fri, 19 Sep 2025 10:37:51 -0400 Subject: [PATCH 1/5] Add new parameters for interactive login, new params to export empty fields with reandmes adjusted. --- Guide-XrmToolBox.md | 11 +- README.md | 68 +++++- ...CommandLineConfigurationExtensionsTests.cs | 26 +++ ...omCommandLineConfigurationProviderTests.cs | 205 ++++++++++++++++++ ...stomCommandLineConfigurationSourceTests.cs | 25 +++ .../Console.Tests/FakeSchemas.cs | 7 +- .../DataverseRecordToRecordMapperTests.cs | 32 ++- .../EntityFieldValueToFieldMapperTests.cs | 2 +- .../Dataverse/DataverseDomainServiceTests.cs | 8 +- ...ustomCommandLineConfigurationExtensions.cs | 160 ++++++++++++++ .../CustomCommandLineConfigurationProvider.cs | 159 ++++++++++++++ .../CustomCommandLineConfigurationSource.cs | 23 ++ .../Mappers/DataverseRecordToRecordMapper.cs | 21 +- .../Mappers/EntityFieldValueToFieldMapper.cs | 3 +- .../Features/Shared/Domain/EntityImport.cs | 3 + .../Program.cs | 23 +- .../Properties/launchSettings.json | 23 +- .../DataverseDomainServiceOptions.cs | 5 + .../SdkDataverseServiceFactoryOptions.cs | 1 + .../Connection/SdkDataverseServiceFactory.cs | 12 + .../Dataverse/DataverseDomainService.cs | 9 +- .../ConfigurationMigrationControl.cs | 5 +- 22 files changed, 797 insertions(+), 34 deletions(-) create mode 100644 src/Dataverse.ConfigurationMigrationTool/Console.Tests/ConfigurationProviders/CustomCommandLineConfigurationExtensionsTests.cs create mode 100644 src/Dataverse.ConfigurationMigrationTool/Console.Tests/ConfigurationProviders/CustomCommandLineConfigurationProviderTests.cs create mode 100644 src/Dataverse.ConfigurationMigrationTool/Console.Tests/ConfigurationProviders/CustomCommandLineConfigurationSourceTests.cs create mode 100644 src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/ConfigurationProviders/CustomCommandLineConfigurationExtensions.cs create mode 100644 src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/ConfigurationProviders/CustomCommandLineConfigurationProvider.cs create mode 100644 src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/ConfigurationProviders/CustomCommandLineConfigurationSource.cs create mode 100644 src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Services/Dataverse/Configuration/DataverseDomainServiceOptions.cs diff --git a/Guide-XrmToolBox.md b/Guide-XrmToolBox.md index 33c083d..ac2d722 100644 --- a/Guide-XrmToolBox.md +++ b/Guide-XrmToolBox.md @@ -9,10 +9,15 @@ This tool was created to support more data types that the official tool does. โœ”๏ธ Schema definition file for export/import ### Upcoming features ๐Ÿ”œ -๐Ÿ”œ Configuration Data Importation \ -๐Ÿ”œ Configuration Data Exportation +๐Ÿ”œ Supports for multiselect optionset \ +๐Ÿ”œ ~~Configuration Data Importation~~ This is no longer planned.\ +๐Ÿ”œ ~~Configuration Data Exportation~~ This is no longer planned. + +> [!IMPORTANT] +> Data import/export features will only be available through the cli tool. +> Since the cli tool is made with .Net Core and XTB Plugins is in .Net Framework, It's really hard to keep a sharable codebase and retricts the usage of some modern libraries. +> CLI tool documentation [here](https://github.com/dotnetprog/dataverse-configuration-migration-tool) ->**Note**: Those features are available through the cli tool. More Info [here](https://github.com/dotnetprog/dataverse-configuration-migration-too) ## What's a schema definition file exactly ๐Ÿค”โ“ diff --git a/README.md b/README.md index 4ff46b1..383c84d 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,8 @@ This repository contains a custom .NET CLI tool designed to export and import co ### Download latest release Get latest version of the tool built on this [release](https://github.com/dotnetprog/dataverse-configuration-migration-tool/releases/latest) > [!NOTE] -> If you want to use the built version of the tool , `appsettings.Production.json` will need to be setup manually with your azure service principal credentials. +> Interactive Login is now available with the CLI tool. The use of service principal is no more mandatory but still recommanded for automation scenarios. \ +> To automate the use of the tool, let's say within a pipeline (e.g Github Action,Azure Devops Pipeline),`appsettings.Production.json` will need to be setup manually with your azure service principal credentials. > [Quick Guide](https://recursion.no/blogs/dataverse-setup-service-principal-access-for-environment/) to create an azure service principal ## Why โ“ @@ -17,7 +18,8 @@ This new tool enables you to: - runs on windows and **linux** ## โญFeaturesโญ - +๐Ÿ†• :heavy_check_mark: **Interactive Login** is now available by using `--il` parameter. Service Principal is no more required.๐Ÿ†• \ +๐Ÿ†• :heavy_check_mark: Data export now supports a new parameter `--AllowEmptyFields` or `--enable-empty-fields` to export empty fields. *Useful to clear values on target environments* ๐Ÿ†• \ :heavy_check_mark: Import configuration data into Dataverse \ :heavy_check_mark: Export configuration data from Dataverse \ :heavy_check_mark: Schema validation and rule-based checks \ @@ -36,6 +38,13 @@ This new tool enables you to: - Image - File +### Upcoming features ๐Ÿ”œ +๐Ÿ”œ Supports for multiselect optionset + + +> [!IMPORTANT] +> Data import/export features for XrmToolBox is no longer planned. +> Since the cli tool is made with .Net Core and XTB Plugins is in .Net Framework, It's really hard to keep a sharable codebase and retricts the usage of some modern libraries. @@ -59,15 +68,20 @@ This new tool enables you to: ### Usage +Before running the tool, set your dataverse variables securely using [dotnet user-secrets](https://learn.microsoft.com/en-us/aspnet/core/security/app-secrets): -Before running the tool, set your `clientId`, `clientSecret` and `url` securely using [dotnet user-secrets](https://learn.microsoft.com/en-us/aspnet/core/security/app-secrets): - +#### For Service principal ```powershell cd src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console dotnet user-secrets set "Dataverse:ClientId" "" dotnet user-secrets set "Dataverse:ClientSecret" "" dotnet user-secrets set "Dataverse:Url" "" ``` +#### For Interactive Login +```powershell +cd src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console +dotnet user-secrets set "Dataverse:Url" "" +``` Run the CLI tool with the required arguments (no need to pass clientId or clientSecret on the command line): #### example @@ -77,13 +91,51 @@ dotnet run --environment DOTNET_ENVIRONMENT=Development --project Dataverse.Conf #### ๐Ÿ’ป Command Line Arguments ๐Ÿ’ป + Verb: `import` -- `--data` : Path to the data xml file, you can use `export-data` command or the microsoft tool (see last section). -- `--schema` : Path to the schema XML file +- `--data` : Path to the data xml file, you can use `export-data` command or the microsoft tool (see last section).(Absolute or relative path) +- `--schema` : Path to the schema XML file (Absolute or relative path) +- ๐Ÿ†•`--il` or `-il` : **Optional** flag to use interactive login (user will be prompted to log in) instead of service principal. Useful for local testing/execution. + +> [!NOTE] +> using interactive login to import data will request user to login twice as it instantiates two different connections to dataverse for performance purposes. + +**Example** + +using dotnet run from the solution folder (.sln) +```powershell +dotnet run --environment DOTNET_ENVIRONMENT=Development --project Dataverse.ConfigurationMigrationTool.Console -- import --data "C:\temp\data.xml" --schema "C:\temp\data_schema.xml" --il +``` +using directly the tool executable +```powershell +cd path/to/tool__executable_folder +Dataverse.ConfigurationMigrationTool.Console.exe import --data "C:\temp\data.xml" --schema "C:\temp\data_schema.xml" --il +``` Verb: `export-data` -- `--schema` : Path to the schema XML file -- `--output` : output file path to save the exported data. This file can be used for the `import` command. +- `--schema` : Path to the schema XML file (Absolute or relative path) +- `--output` : output file path to save the exported data. This file can be used for the `import` command. (Absolute or relative path) +- ๐Ÿ†•`--AllowEmptyFields` or `--enable-empty-fields` : **Optional** flag to include fields with empty values in the export. Useful for clearing values in the target environment. +- ๐Ÿ†•`--il` or `-il` : **Optional** flag to use interactive login (user will be prompted to log in) instead of service principal. Useful for local testing/execution. + +**Example** + +using dotnet run from the solution folder (.sln) +```powershell +dotnet run --environment DOTNET_ENVIRONMENT=Development --project Dataverse.ConfigurationMigrationTool.Console -- export-data --schema "C:\temp\data_schema.xml" --output "C:\temp\exported_data.xml" --enable-empty-fields --il +``` +using directly the tool executable +```powershell +cd path/to/tool__executable_folder +Dataverse.ConfigurationMigrationTool.Console.exe export-data --schema "C:\temp\data_schema.xml" --output "C:\temp\exported_data.xml" --enable-empty-fields --il +``` + +> [!TIP] +> To use service principal credentials, the interactive login flag `--il` or `-il` must be omitted. +> If you use the executable directly, you must edit the `appsettings.Production.json` file with your service principal credentials +> If you use `dotnet run`, you can set the secrets using `dotnet user-secrets` as shown above or edit the `appsettings.Development.json` file. +> you can also set environment variables `Dataverse__ClientId`, `Dataverse__ClientSecret` and `Dataverse__Url` to override the settings in the json files. +> you can also use these commandline arguments `--Dataverse:ClientId`, `--Dataverse:ClientSecret` and `--Dataverse:Url` to override the settings in the json files. ## ๐Ÿค Contributing ๐Ÿค diff --git a/src/Dataverse.ConfigurationMigrationTool/Console.Tests/ConfigurationProviders/CustomCommandLineConfigurationExtensionsTests.cs b/src/Dataverse.ConfigurationMigrationTool/Console.Tests/ConfigurationProviders/CustomCommandLineConfigurationExtensionsTests.cs new file mode 100644 index 0000000..f108577 --- /dev/null +++ b/src/Dataverse.ConfigurationMigrationTool/Console.Tests/ConfigurationProviders/CustomCommandLineConfigurationExtensionsTests.cs @@ -0,0 +1,26 @@ +๏ปฟusing Dataverse.ConfigurationMigrationTool.Console.ConfigurationProviders; +using Microsoft.Extensions.Configuration; +using NSubstitute; + +namespace Dataverse.ConfigurationMigrationTool.Console.Tests.ConfigurationProviders; +public class CustomCommandLineConfigurationExtensionsTests +{ + private readonly IConfigurationBuilder _builder = Substitute.For(); + public CustomCommandLineConfigurationExtensionsTests() + { + _builder.Add(Arg.Any()).Returns(_builder); + } + [Fact] + public void GivenAConfigurationBuilder_WhenItAddsCustomCommandline_ThenItAddsTheCustomCommandLineConfigurationProvider() + { + // Arrange + var args = new[] { "--key=value" }; + // Act + _builder.AddCustomCommandline(args); + // Assert + _builder.Received(1).Add(Arg.Is(source => + source.Args.SequenceEqual(args) && + source.SwitchMappings == null && + source.FlagMappings == null)); + } +} diff --git a/src/Dataverse.ConfigurationMigrationTool/Console.Tests/ConfigurationProviders/CustomCommandLineConfigurationProviderTests.cs b/src/Dataverse.ConfigurationMigrationTool/Console.Tests/ConfigurationProviders/CustomCommandLineConfigurationProviderTests.cs new file mode 100644 index 0000000..18fee4b --- /dev/null +++ b/src/Dataverse.ConfigurationMigrationTool/Console.Tests/ConfigurationProviders/CustomCommandLineConfigurationProviderTests.cs @@ -0,0 +1,205 @@ +๏ปฟusing Dataverse.ConfigurationMigrationTool.Console.ConfigurationProviders; +using Shouldly; + +namespace Dataverse.ConfigurationMigrationTool.Console.Tests.ConfigurationProviders; + +public class CustomCommandLineConfigurationProviderTests +{ + [Fact] + public void Given_ArgsWithDoubleDash_When_Load_Then_ParsesKeyValueCorrectly() + { + // Arrange + var args = new[] { "--key=value" }; + var provider = BuildProvider(args); + // Act + provider.Load(); + // Assert + GetValue(provider, "key").ShouldBe("value"); + } + [Fact] + public void Given_ArgsWithDoubleDashWithNoSeparator_When_Load_Then_ParsesKeyValueCorrectly() + { + // Arrange + var args = new[] { "--key", "value" }; + var provider = BuildProvider(args); + // Act + provider.Load(); + // Assert + GetValue(provider, "key").ShouldBe("value"); + } + [Fact] + public void Given_ArgsWithSingleDashWithNoSeparator_When_Load_Then_ParsesKeyValueCorrectly() + { + // Arrange + var args = new[] { "-key", "value" }; + var switchMappings = new Dictionary { { "-key", "key" } }; + var provider = BuildProvider(args, switchMappings); + // Act + provider.Load(); + // Assert + GetValue(provider, "key").ShouldBe("value"); + } + [Fact] + public void Given_ArgsWithSingleDash_When_Load_Then_ParsesKeyValueCorrectly() + { + // Arrange + var args = new[] { "-key=value" }; + var switchMappings = new Dictionary { { "-key", "key" } }; + var provider = BuildProvider(args, switchMappings); + // Act + provider.Load(); + // Assert + GetValue(provider, "key").ShouldBe("value"); + } + + [Fact] + public void Given_ArgsWithSlash_When_Load_Then_ParsesKeyValueCorrectly() + { + // Arrange + var args = new[] { "/key=value" }; + var provider = BuildProvider(args); + // Act + provider.Load(); + // Assert + GetValue(provider, "key").ShouldBe("value"); + } + + [Fact] + public void Given_ArgsWithSwitchMappings_When_Load_Then_MapsSwitchToCustomKey() + { + // Arrange + var args = new[] { "--customSwitch=value" }; + var switchMappings = new Dictionary { { "--customSwitch", "mappedKey" } }; + var provider = BuildProvider(args, switchMappings); + // Act + provider.Load(); + // Assert + GetValue(provider, "mappedKey").ShouldBe("value"); + } + + [Fact] + public void Given_ArgsWithFlagMappings_When_Load_Then_SetsFlagValueToTrue() + { + // Arrange + var args = new[] { "--flag" }; + var flagMappings = new[] { "--flag" }; + var provider = BuildProvider(args, null, flagMappings); + // Act + provider.Load(); + // Assert + GetValue(provider, "flag").ShouldBe("True"); + } + + [Fact] + public void Given_ArgsWithDuplicateKeys_When_Load_Then_LastValueWins() + { + // Arrange + var args = new[] { "--key=first", "--key=second" }; + var provider = BuildProvider(args); + // Act + provider.Load(); + // Assert + GetValue(provider, "key").ShouldBe("second"); + } + + [Fact] + public void Given_ArgsWithInvalidFormat_When_Load_Then_IgnoresInvalidArgs() + { + // Arrange + var args = new[] { "invalidArg", "--valid=value" }; + var provider = BuildProvider(args); + // Act + provider.Load(); + // Assert + GetValue(provider, "invalidArg").ShouldBeNull(); + GetValue(provider, "valid").ShouldBe("value"); + } + + [Fact] + public void Given_ShortDashWithoutMapping_When_Load_Then_IgnoresArg() + { + // Arrange + var args = new[] { "-short" }; + var provider = BuildProvider(args); + // Act + provider.Load(); + // Assert + GetValue(provider, "short").ShouldBeNull(); + } + + [Fact] + public void Given_ShortDashWithMapping_When_Load_Then_MapsKey() + { + // Arrange + var args = new[] { "-s=value" }; + var switchMappings = new Dictionary { { "-s", "shortKey" } }; + var provider = BuildProvider(args, switchMappings); + // Act + provider.Load(); + // Assert + GetValue(provider, "shortKey").ShouldBe("value"); + } + + [Fact] + public void Given_ShortDashWithNoMappingAndEquals_When_Load_Then_ThrowsFormatException() + { + // Arrange + var args = new[] { "-s=value" }; + var provider = BuildProvider(args); + // Act + var act = () => provider.Load(); + // Assert + act.ShouldThrow(); + } + + [Fact] + public void Given_SwitchMappingsWithInvalidKey_When_Construct_Then_ThrowsArgumentException() + { + // Arrange + var switchMappings = new Dictionary { { "invalidKey", "mappedKey" } }; + var args = new[] { "--key=value" }; + // Act + var act = () => BuildProvider(args, switchMappings); + // Assert + act.ShouldThrow(); + } + + [Fact] + public void Given_SwitchMappingsWithDuplicateKeys_When_Construct_Then_ThrowsArgumentException() + { + // Arrange + var switchMappings = new Dictionary + { + { "--dup", "key1" }, + { "--DUP", "key2" } + }; + var args = new[] { "--dup=value" }; + // Act + var act = () => BuildProvider(args, switchMappings); + // Assert + act.ShouldThrow(); + } + + [Fact] + public void Given_ArgsIsNull_When_Construct_Then_ThrowsArgumentNullException() + { + // Act + var act = () => BuildProvider(null!); + //Assert + act.ShouldThrow(); + } + private static CustomCommandLineConfigurationProvider BuildProvider(IEnumerable args, IDictionary? switchMappings = null, IEnumerable? flagMappings = null) + { + return new CustomCommandLineConfigurationProvider(args, switchMappings, flagMappings); + } + + // Helper to get value from provider's TryGet method using reflection + private static string? GetValue(CustomCommandLineConfigurationProvider provider, string key) + { + if (provider.TryGet(key, out var value)) + { + return value; + } + return null; + } +} diff --git a/src/Dataverse.ConfigurationMigrationTool/Console.Tests/ConfigurationProviders/CustomCommandLineConfigurationSourceTests.cs b/src/Dataverse.ConfigurationMigrationTool/Console.Tests/ConfigurationProviders/CustomCommandLineConfigurationSourceTests.cs new file mode 100644 index 0000000..21d49f9 --- /dev/null +++ b/src/Dataverse.ConfigurationMigrationTool/Console.Tests/ConfigurationProviders/CustomCommandLineConfigurationSourceTests.cs @@ -0,0 +1,25 @@ +๏ปฟusing Dataverse.ConfigurationMigrationTool.Console.ConfigurationProviders; +using Shouldly; + +namespace Dataverse.ConfigurationMigrationTool.Console.Tests.ConfigurationProviders; +public class CustomCommandLineConfigurationSourceTests +{ + private CustomCommandLineConfigurationSource ConfigSource { get; } + public CustomCommandLineConfigurationSourceTests() + { + ConfigSource = new CustomCommandLineConfigurationSource() + { + Args = [], + }; + } + + [Fact] + public void GivenACommandLineSource_WhenItBuildsAConfigProvider_ThenACommandLineConfigurationProviderIsReturned() + { + // Act + var provider = ConfigSource.Build(null); + // Assert + provider.ShouldBeOfType(); + } + +} diff --git a/src/Dataverse.ConfigurationMigrationTool/Console.Tests/FakeSchemas.cs b/src/Dataverse.ConfigurationMigrationTool/Console.Tests/FakeSchemas.cs index 05bd27b..a36c456 100644 --- a/src/Dataverse.ConfigurationMigrationTool/Console.Tests/FakeSchemas.cs +++ b/src/Dataverse.ConfigurationMigrationTool/Console.Tests/FakeSchemas.cs @@ -26,7 +26,12 @@ internal static class FakeSchemas Displayname = "Primary Contact", LookupType = "contact", PrimaryKey = false - } + }, + new FieldSchema + { + Name = "revenue", + Type = "money" + } } }, }; diff --git a/src/Dataverse.ConfigurationMigrationTool/Console.Tests/Features/Export/Mappers/DataverseRecordToRecordMapperTests.cs b/src/Dataverse.ConfigurationMigrationTool/Console.Tests/Features/Export/Mappers/DataverseRecordToRecordMapperTests.cs index eba0964..edb564e 100644 --- a/src/Dataverse.ConfigurationMigrationTool/Console.Tests/Features/Export/Mappers/DataverseRecordToRecordMapperTests.cs +++ b/src/Dataverse.ConfigurationMigrationTool/Console.Tests/Features/Export/Mappers/DataverseRecordToRecordMapperTests.cs @@ -5,9 +5,10 @@ namespace Dataverse.ConfigurationMigrationTool.Console.Tests.Features.Export.Mappers; public class DataverseRecordToRecordMapperTests { - private readonly DataverseRecordToRecordMapper _mapper = new DataverseRecordToRecordMapper(); + private readonly DataverseRecordToRecordMapper _mapperWithNoEmptyFields = new DataverseRecordToRecordMapper(false); + private readonly DataverseRecordToRecordMapper _mapperWithEmptyFields = new DataverseRecordToRecordMapper(true); [Fact] - public void GivenAnEntity_WhenItIsMappedToARecord_ThenItTheRecordShouldBeProperplyCreated() + public void GivenAnEntity_WhenItIsMappedToARecordWithNoEmptyFields_ThenItTheRecordShouldBeProperplyCreated() { //Arrange var Schema = FakeSchemas.Account; @@ -18,7 +19,7 @@ public void GivenAnEntity_WhenItIsMappedToARecord_ThenItTheRecordShouldBeProperp ["primarycontactid"] = new EntityReference("contact", Guid.NewGuid()) }; //Act - var record = _mapper.Map((Schema, Entity)); + var record = _mapperWithNoEmptyFields.Map((Schema, Entity)); //Assert record.Id.ShouldBe(Entity.Id); record.Field.ForEach(field => @@ -28,8 +29,33 @@ public void GivenAnEntity_WhenItIsMappedToARecord_ThenItTheRecordShouldBeProperp }); record.Field.First(f => f.Name == "name").Value.ShouldBe("Test Account"); var lookupField = record.Field.First(f => f.Name == "primarycontactid"); + record.Field.FirstOrDefault(f => f.Name == "revenue").ShouldBeNull(); lookupField.Value.ShouldBe(Entity.GetAttributeValue("primarycontactid").Id.ToString()); lookupField.Lookupentity.ShouldBe("contact"); lookupField.Lookupentityname.ShouldBeNull(); } + [Fact] + public void GivenAnEntity_WhenItIsMappedToARecordWithEmptyFields_ThenItTheRecordShouldBeProperplyCreated() + { + //Arrange + var Schema = FakeSchemas.Account; + var Entity = new Entity("account") + { + Id = Guid.NewGuid(), + ["name"] = "Test Account", + ["primarycontactid"] = new EntityReference("contact", Guid.NewGuid()) + }; + //Act + var record = _mapperWithEmptyFields.Map((Schema, Entity)); + //Assert + record.Id.ShouldBe(Entity.Id); + record.Field.First(f => f.Name == "name").Value.ShouldBe("Test Account"); + var lookupField = record.Field.First(f => f.Name == "primarycontactid"); + lookupField.Value.ShouldBe(Entity.GetAttributeValue("primarycontactid").Id.ToString()); + lookupField.Lookupentity.ShouldBe("contact"); + lookupField.Lookupentityname.ShouldBeNull(); + var revenue = record.Field.First(f => f.Name == "revenue"); + revenue.Value.ShouldBeNull(); + revenue.IsNull.ShouldBeTrue(); + } } diff --git a/src/Dataverse.ConfigurationMigrationTool/Console.Tests/Features/Export/Mappers/EntityFieldValueToFieldMapperTests.cs b/src/Dataverse.ConfigurationMigrationTool/Console.Tests/Features/Export/Mappers/EntityFieldValueToFieldMapperTests.cs index 2e6dac2..578092f 100644 --- a/src/Dataverse.ConfigurationMigrationTool/Console.Tests/Features/Export/Mappers/EntityFieldValueToFieldMapperTests.cs +++ b/src/Dataverse.ConfigurationMigrationTool/Console.Tests/Features/Export/Mappers/EntityFieldValueToFieldMapperTests.cs @@ -15,7 +15,7 @@ public class EntityFieldValueToFieldMapperTests { new FieldSchema { Name = "testField" }, null, - null + new Field { Name = "testField",IsNull=true } }, { new FieldSchema { Name = "testLookupField" }, diff --git a/src/Dataverse.ConfigurationMigrationTool/Console.Tests/Services/Dataverse/DataverseDomainServiceTests.cs b/src/Dataverse.ConfigurationMigrationTool/Console.Tests/Services/Dataverse/DataverseDomainServiceTests.cs index 3cc6d3c..b6ef0af 100644 --- a/src/Dataverse.ConfigurationMigrationTool/Console.Tests/Services/Dataverse/DataverseDomainServiceTests.cs +++ b/src/Dataverse.ConfigurationMigrationTool/Console.Tests/Services/Dataverse/DataverseDomainServiceTests.cs @@ -1,7 +1,9 @@ ๏ปฟusing Dataverse.ConfigurationMigrationTool.Console.Features.Shared.Domain; using Dataverse.ConfigurationMigrationTool.Console.Services.Dataverse; +using Dataverse.ConfigurationMigrationTool.Console.Services.Dataverse.Configuration; using Dataverse.ConfigurationMigrationTool.Console.Tests.Extensions; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Microsoft.PowerPlatform.Dataverse.Client; using Microsoft.Xrm.Sdk; using Microsoft.Xrm.Sdk.Query; @@ -14,11 +16,15 @@ public class DataverseDomainServiceTests private readonly IOrganizationServiceAsync2 _orgService; private readonly ILogger _logger; private readonly DataverseDomainService _domainService; + private readonly IOptions _options = Options.Create(new DataverseDomainServiceOptions() + { + AllowEmptyFields = false + }); public DataverseDomainServiceTests() { _orgService = Substitute.For(); _logger = Substitute.For>(); - _domainService = new DataverseDomainService(_orgService, _logger); + _domainService = new DataverseDomainService(_orgService, _options, _logger); } [Fact] public async Task GivenADomainService_WhenItExportsWithEntitySchema_ThenItShouldCallDataverseProperly() diff --git a/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/ConfigurationProviders/CustomCommandLineConfigurationExtensions.cs b/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/ConfigurationProviders/CustomCommandLineConfigurationExtensions.cs new file mode 100644 index 0000000..19cc416 --- /dev/null +++ b/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/ConfigurationProviders/CustomCommandLineConfigurationExtensions.cs @@ -0,0 +1,160 @@ +๏ปฟusing Microsoft.Extensions.Configuration; + +namespace Dataverse.ConfigurationMigrationTool.Console.ConfigurationProviders; +public static class CustomCommandLineConfigurationExtensions +{ + /// + /// Adds a + /// that reads configuration values from the command line. + /// + /// The to add to. + /// The command line args. + /// The . + /// + /// + /// The values passed on the command line, in the args string array, should be a set + /// of keys prefixed with two dashes ("--") and then values, separate by either the + /// equals sign ("=") or a space (" "). + /// + /// + /// A forward slash ("/") can be used as an alternative prefix, with either equals or space, and when using + /// an equals sign the prefix can be left out altogether. + /// + /// + /// There are five basic alternative formats for arguments: + /// key1=value1 --key2=value2 /key3=value3 --key4 value4 /key5 value5. + /// + /// + /// + /// A simple console application that has five values. + /// + /// // dotnet run key1=value1 --key2=value2 /key3=value3 --key4 value4 /key5 value5 + /// + /// using Microsoft.Extensions.Configuration; + /// using System; + /// + /// namespace CommandLineSample + /// { + /// public class Program + /// { + /// public static void Main(string[] args) + /// { + /// var builder = new ConfigurationBuilder(); + /// builder.AddCommandLine(args); + /// + /// var config = builder.Build(); + /// + /// Console.WriteLine($"Key1: '{config["Key1"]}'"); + /// Console.WriteLine($"Key2: '{config["Key2"]}'"); + /// Console.WriteLine($"Key3: '{config["Key3"]}'"); + /// Console.WriteLine($"Key4: '{config["Key4"]}'"); + /// Console.WriteLine($"Key5: '{config["Key5"]}'"); + /// } + /// } + /// } + /// + /// + public static IConfigurationBuilder AddCustomCommandline(this IConfigurationBuilder configurationBuilder, string[] args) + { + return configurationBuilder.AddCustomCommandline(args, switchMappings: null, flagMappings: null); + } + + /// + /// Adds a that reads + /// configuration values from the command line using the specified switch mappings. + /// + /// The to add to. + /// The command line args. + /// + /// The switch mappings. A dictionary of short (with prefix "-") and + /// alias keys (with prefix "--"), mapped to the configuration key (no prefix). + /// + /// The . + /// + /// + /// The switchMappings allows additional formats for alternative short and alias keys + /// to be used from the command line. Also see the basic version of AddCommandLine for + /// the standard formats supported. + /// + /// + /// Short keys start with a single dash ("-") and are mapped to the main key name (without + /// prefix), and can be used with either equals or space. The single dash mappings are intended + /// to be used for shorter alternative switches. + /// + /// + /// Note that a single dash switch cannot be accessed directly, but must have a switch mapping + /// defined and accessed using the full key. Passing an undefined single dash argument will + /// cause as FormatException. + /// + /// + /// There are two formats for short arguments: + /// -k1=value1 -k2 value2. + /// + /// + /// Alias key definitions start with two dashes ("--") and are mapped to the main key name (without + /// prefix), and can be used in place of the normal key. They also work when a forward slash prefix + /// is used in the command line (but not with the no prefix equals format). + /// + /// + /// There are only four formats for aliased arguments: + /// --alt3=value3 /alt4=value4 --alt5 value5 /alt6 value6. + /// + /// + /// + /// A simple console application that has two short and four alias switch mappings defined. + /// + /// // dotnet run -k1=value1 -k2 value2 --alt3=value2 /alt4=value3 --alt5 value5 /alt6 value6 + /// + /// using Microsoft.Extensions.Configuration; + /// using System; + /// using System.Collections.Generic; + /// + /// namespace CommandLineSample + /// { + /// public class Program + /// { + /// public static void Main(string[] args) + /// { + /// var switchMappings = new Dictionary<string, string>() + /// { + /// { "-k1", "key1" }, + /// { "-k2", "key2" }, + /// { "--alt3", "key3" }, + /// { "--alt4", "key4" }, + /// { "--alt5", "key5" }, + /// { "--alt6", "key6" }, + /// }; + /// var builder = new ConfigurationBuilder(); + /// builder.AddCommandLine(args, switchMappings); + /// + /// var config = builder.Build(); + /// + /// Console.WriteLine($"Key1: '{config["Key1"]}'"); + /// Console.WriteLine($"Key2: '{config["Key2"]}'"); + /// Console.WriteLine($"Key3: '{config["Key3"]}'"); + /// Console.WriteLine($"Key4: '{config["Key4"]}'"); + /// Console.WriteLine($"Key5: '{config["Key5"]}'"); + /// Console.WriteLine($"Key6: '{config["Key6"]}'"); + /// } + /// } + /// } + /// + /// + public static IConfigurationBuilder AddCustomCommandline( + this IConfigurationBuilder configurationBuilder, + string[] args, + IDictionary? switchMappings, IEnumerable? flagMappings) + { + configurationBuilder.Add(new CustomCommandLineConfigurationSource { Args = args, SwitchMappings = switchMappings, FlagMappings = flagMappings }); + return configurationBuilder; + } + + /// + /// Adds an that reads configuration values from the command line. + /// + /// The to add to. + /// Configures the source. + /// The . + public static IConfigurationBuilder AddCustomCommandline(this IConfigurationBuilder builder, Action? configureSource) + => builder.Add(configureSource); +} diff --git a/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/ConfigurationProviders/CustomCommandLineConfigurationProvider.cs b/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/ConfigurationProviders/CustomCommandLineConfigurationProvider.cs new file mode 100644 index 0000000..58e5b1b --- /dev/null +++ b/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/ConfigurationProviders/CustomCommandLineConfigurationProvider.cs @@ -0,0 +1,159 @@ +๏ปฟusing Microsoft.Extensions.Configuration; + +namespace Dataverse.ConfigurationMigrationTool.Console.ConfigurationProviders; +public class CustomCommandLineConfigurationProvider : ConfigurationProvider +{ + private readonly Dictionary? _switchMappings; + + /// + /// Initializes a new instance. + /// + /// The command line args. + /// The switch mappings. + public CustomCommandLineConfigurationProvider(IEnumerable args, IDictionary? switchMappings = null, IEnumerable? flagMappings = null) + { + ArgumentNullException.ThrowIfNull(args); + + Args = args; + FlagMappings = flagMappings ?? Array.Empty(); + + if (switchMappings != null) + { + _switchMappings = GetValidatedSwitchMappingsCopy(switchMappings); + } + } + + /// + /// Gets the command-line arguments. + /// + protected IEnumerable Args { get; } + protected IEnumerable FlagMappings { get; } + + /// + /// Loads the configuration data from the command-line arguments. + /// + public override void Load() + { + var data = new Dictionary(StringComparer.OrdinalIgnoreCase); + string key, value; + + using (IEnumerator enumerator = Args.GetEnumerator()) + { + while (enumerator.MoveNext()) + { + var currentflag = FlagMappings.FirstOrDefault(f => f.Equals(enumerator.Current, StringComparison.OrdinalIgnoreCase)); + string currentArg = enumerator.Current; + int keyStartIndex = 0; + + if (currentArg.StartsWith("--")) + { + keyStartIndex = 2; + } + else if (currentArg.StartsWith("-")) + { + keyStartIndex = 1; + } + else if (currentArg.StartsWith("/")) + { + // "/SomeSwitch" is equivalent to "--SomeSwitch" when interpreting switch mappings + // So we do a conversion to simplify later processing + currentArg = $"--{currentArg.Substring(1)}"; + keyStartIndex = 2; + } + + int separator = currentArg.IndexOf('='); + + if (separator < 0) + { + // If there is neither equal sign nor prefix in current argument, it is an invalid format + if (keyStartIndex == 0) + { + // Ignore invalid formats + continue; + } + + // If the switch is a key in given switch mappings, interpret it + if (_switchMappings != null && _switchMappings.TryGetValue(currentArg, out string? mappedKey)) + { + key = mappedKey; + } + // If the switch starts with a single "-" and it isn't in given mappings , it is an invalid usage so ignore it + else if (keyStartIndex == 1) + { + continue; + } + // Otherwise, use the switch name directly as a key + else + { + key = currentArg.Substring(keyStartIndex); + } + + if (string.IsNullOrEmpty(currentflag) && !enumerator.MoveNext()) + { + // ignore missing values + continue; + } + + value = string.IsNullOrEmpty(currentflag) ? enumerator.Current : true.ToString(); + } + else + { + string keySegment = currentArg.Substring(0, separator); + + // If the switch is a key in given switch mappings, interpret it + if (_switchMappings != null && _switchMappings.TryGetValue(keySegment, out string? mappedKeySegment)) + { + key = mappedKeySegment; + } + // If the switch starts with a single "-" and it isn't in given mappings , it is an invalid usage + else if (keyStartIndex == 1) + { + throw new FormatException($"{currentArg} is used as a short parameter but is not used into the switch mappings"); + } + // Otherwise, use the switch name directly as a key + else + { + key = currentArg.Substring(keyStartIndex, separator - keyStartIndex); + } + + value = currentArg.Substring(separator + 1); + } + + // Override value when key is duplicated. So we always have the last argument win. + data[key] = value; + } + } + + Data = data; + } + + private static Dictionary GetValidatedSwitchMappingsCopy(IDictionary switchMappings) + { + // The dictionary passed in might be constructed with a case-sensitive comparer + // However, the keys in configuration providers are all case-insensitive + // So we check whether the given switch mappings contain duplicated keys with case-insensitive comparer + var switchMappingsCopy = new Dictionary(switchMappings.Count, StringComparer.OrdinalIgnoreCase); + foreach (KeyValuePair mapping in switchMappings) + { + // Only keys start with "--" or "-" are acceptable + if (!mapping.Key.StartsWith("-") && !mapping.Key.StartsWith("--")) + { + throw new ArgumentException( + $"{mapping.Key} is not valid. Only keys start with \"--\" or \"-\" are acceptable", + nameof(switchMappings)); + } + + if (switchMappingsCopy.ContainsKey(mapping.Key)) + { + throw new ArgumentException( + $"{mapping.Key} is not valid. Duplicated switch keys are not supported.", + nameof(switchMappings)); + } + + switchMappingsCopy.Add(mapping.Key, mapping.Value); + } + + return switchMappingsCopy; + } +} + diff --git a/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/ConfigurationProviders/CustomCommandLineConfigurationSource.cs b/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/ConfigurationProviders/CustomCommandLineConfigurationSource.cs new file mode 100644 index 0000000..12386fb --- /dev/null +++ b/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/ConfigurationProviders/CustomCommandLineConfigurationSource.cs @@ -0,0 +1,23 @@ +๏ปฟusing Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Configuration.CommandLine; + +namespace Dataverse.ConfigurationMigrationTool.Console.ConfigurationProviders; +public class CustomCommandLineConfigurationSource : IConfigurationSource +{ + public IDictionary? SwitchMappings { get; set; } + public IEnumerable? FlagMappings { get; set; } + /// + /// Gets or sets the command line arguments. + /// + public IEnumerable Args { get; set; } = Array.Empty(); + + /// + /// Builds the for this source. + /// + /// The . + /// A . + public IConfigurationProvider Build(IConfigurationBuilder builder) + { + return new CustomCommandLineConfigurationProvider(Args, SwitchMappings, FlagMappings); + } +} diff --git a/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Features/Export/Mappers/DataverseRecordToRecordMapper.cs b/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Features/Export/Mappers/DataverseRecordToRecordMapper.cs index 9de8902..f2ce966 100644 --- a/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Features/Export/Mappers/DataverseRecordToRecordMapper.cs +++ b/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Features/Export/Mappers/DataverseRecordToRecordMapper.cs @@ -5,6 +5,11 @@ namespace Dataverse.ConfigurationMigrationTool.Console.Features.Export.Mappers; public class DataverseRecordToRecordMapper : IMapper<(EntitySchema, Entity), Record> { + private bool AllowEmptyFields { get; set; } + public DataverseRecordToRecordMapper(bool allowEmptyFields) + { + AllowEmptyFields = allowEmptyFields; + } private readonly static IMapper<(FieldSchema, object), Field> _fieldMapper = new EntityFieldValueToFieldMapper(); public Record Map((EntitySchema, Entity) source) { @@ -17,16 +22,16 @@ public Record Map((EntitySchema, Entity) source) foreach (var fieldSchema in entitySchema.Fields.Field) { - if (entity.Contains(fieldSchema.Name)) + + var fieldValue = entity.GetAttributeValue(fieldSchema.Name); + if (!AllowEmptyFields && fieldValue == null) { - var fieldValue = entity[fieldSchema.Name]; - var field = _fieldMapper.Map((fieldSchema, fieldValue)); - if (field == null) - { - continue; - } - record.Field.Add(field); + continue; } + var field = _fieldMapper.Map((fieldSchema, fieldValue)); + + record.Field.Add(field); + } return record; } diff --git a/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Features/Export/Mappers/EntityFieldValueToFieldMapper.cs b/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Features/Export/Mappers/EntityFieldValueToFieldMapper.cs index dc08e32..a9d4ba6 100644 --- a/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Features/Export/Mappers/EntityFieldValueToFieldMapper.cs +++ b/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Features/Export/Mappers/EntityFieldValueToFieldMapper.cs @@ -16,7 +16,8 @@ public Field Map((FieldSchema, object) source) }; if (value == null) { - return null; + fieldResult.IsNull = true; + return fieldResult; } if (value is EntityReference reference) { diff --git a/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Features/Shared/Domain/EntityImport.cs b/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Features/Shared/Domain/EntityImport.cs index 2d43a2b..a6b69bf 100644 --- a/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Features/Shared/Domain/EntityImport.cs +++ b/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Features/Shared/Domain/EntityImport.cs @@ -18,12 +18,15 @@ public class Field [XmlAttribute(AttributeName = "value")] public string Value { get; set; } + [XmlAttribute(AttributeName = "isnull")] + public bool IsNull { get; set; } [XmlAttribute(AttributeName = "lookupentity")] public string Lookupentity { get; set; } [XmlAttribute(AttributeName = "lookupentityname")] public string Lookupentityname { get; set; } + public bool ShouldSerializeIsNull() => IsNull; } [XmlRoot(ElementName = "record")] diff --git a/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Program.cs b/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Program.cs index 2b29b9f..de987a1 100644 --- a/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Program.cs +++ b/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Program.cs @@ -1,4 +1,5 @@ -๏ปฟusing Dataverse.ConfigurationMigrationTool.Console.Features; +๏ปฟusing Dataverse.ConfigurationMigrationTool.Console.ConfigurationProviders; +using Dataverse.ConfigurationMigrationTool.Console.Features; using Dataverse.ConfigurationMigrationTool.Console.Features.Export; using Dataverse.ConfigurationMigrationTool.Console.Features.Import; using Dataverse.ConfigurationMigrationTool.Console.Features.Shared; @@ -11,6 +12,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Microsoft.PowerPlatform.Dataverse.Client; using System.Reflection; var builder = Host.CreateDefaultBuilder(args); @@ -35,7 +37,13 @@ //or you can configure another Configuration Provider to provide the secrets like AzureKeyvault or Hashicorp Vault. config.AddUserSecrets(Assembly.GetExecutingAssembly()); } - config.AddCommandLine(args); + var switchMappings = new Dictionary() + { + { "-il", "Dataverse:InteractiveLogin" }, + { "--il", "Dataverse:InteractiveLogin" }, + { "--enable-empty-fields", "AllowEmptyFields" }, + }; + config.AddCustomCommandline(args, switchMappings, ["--enable-empty-fields", "--AllowEmptyFields", "-il", "--il"]); Console.WriteLine($"Using configuration file: appsettings.{context.HostingEnvironment.EnvironmentName}.json"); }); builder.ConfigureServices((context, services) => @@ -43,8 +51,13 @@ services .AddLogging(lb => lb.AddConsole()) + .Configure(context.Configuration) .AddScoped() .Configure(context.Configuration.GetSection("Dataverse")) + //.PostConfigure((options) => + //{ + // options.InteractiveLogin = args.Contains("--il"); + //}) .Configure(context.Configuration.GetSection("Dataverse")) .AddScoped() .AddSingleton() @@ -63,4 +76,10 @@ }); var app = builder.Build(); +var dataverseOptions = app.Services.GetRequiredService>(); +if (dataverseOptions.Value.InteractiveLogin) +{ + Console.WriteLine("Using Interactive Login for Dataverse authentication"); + Console.WriteLine("***For import command, you will be prompted twice for login because it instantiates two connection types for performance purposes"); +} await app.RunAsync(); diff --git a/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Properties/launchSettings.json b/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Properties/launchSettings.json index aef2781..4e1aa6d 100644 --- a/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Properties/launchSettings.json +++ b/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Properties/launchSettings.json @@ -7,9 +7,30 @@ "DOTNET_ENVIRONMENT": "Development" } }, + "import-il": { + "commandName": "Project", + "commandLineArgs": "import --schema ../../../TestAssets/data_schema.xml --data ../../../TestAssets/data.xml --il", + "environmentVariables": { + "DOTNET_ENVIRONMENT": "Development" + } + }, + "import-null": { + "commandName": "Project", + "commandLineArgs": "import --schema ../../../TestAssets/data_schema.xml --data ../../../TestAssets/exporteddata_withnull.xml", + "environmentVariables": { + "DOTNET_ENVIRONMENT": "Development" + } + }, + "export-data-il": { + "commandName": "Project", + "commandLineArgs": "export-data --enable-empty-fields --schema ../../../TestAssets/data_schema.xml --output ../../../TestAssets/exporteddata.xml --il", + "environmentVariables": { + "DOTNET_ENVIRONMENT": "Development" + } + }, "export-data": { "commandName": "Project", - "commandLineArgs": "export-data --schema ../../../TestAssets/data_schema.xml --output ../../../TestAssets/exporteddata.xml", + "commandLineArgs": "export-data --schema ../../../TestAssets/data_schema.xml --output ../../../TestAssets/exporteddata.xml --AllowEmptyFields", "environmentVariables": { "DOTNET_ENVIRONMENT": "Development" } diff --git a/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Services/Dataverse/Configuration/DataverseDomainServiceOptions.cs b/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Services/Dataverse/Configuration/DataverseDomainServiceOptions.cs new file mode 100644 index 0000000..009fb5d --- /dev/null +++ b/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Services/Dataverse/Configuration/DataverseDomainServiceOptions.cs @@ -0,0 +1,5 @@ +๏ปฟnamespace Dataverse.ConfigurationMigrationTool.Console.Services.Dataverse.Configuration; +public class DataverseDomainServiceOptions +{ + public bool AllowEmptyFields { get; set; } +} diff --git a/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Services/Dataverse/Configuration/SdkDataverseServiceFactoryOptions.cs b/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Services/Dataverse/Configuration/SdkDataverseServiceFactoryOptions.cs index e3d62b5..df997af 100644 --- a/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Services/Dataverse/Configuration/SdkDataverseServiceFactoryOptions.cs +++ b/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Services/Dataverse/Configuration/SdkDataverseServiceFactoryOptions.cs @@ -5,5 +5,6 @@ public class SdkDataverseServiceFactoryOptions public Guid ClientId { get; set; } public string ClientSecret { get; set; } public string Url { get; set; } + public bool InteractiveLogin { get; set; } } diff --git a/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Services/Dataverse/Connection/SdkDataverseServiceFactory.cs b/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Services/Dataverse/Connection/SdkDataverseServiceFactory.cs index 86c39eb..f109c22 100644 --- a/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Services/Dataverse/Connection/SdkDataverseServiceFactory.cs +++ b/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Services/Dataverse/Connection/SdkDataverseServiceFactory.cs @@ -9,6 +9,12 @@ public class SdkDataverseServiceFactory : IDataverseClientFactory { private readonly SdkDataverseServiceFactoryOptions _options; private ILogger _logger; + /// + /// The default Microsoft App ID used for interactive login if no other client ID is provided. + /// Source: https://github.com/microsoft/PowerApps-Samples/tree/master/dataverse/orgsvc/CSharp-NETCore/ServiceClient + /// + const string DefaultMicrosoftAppId = "51f81489-12ee-4a9e-aaae-a2591f45987d"; + const string DefaultMicrosoftRedirectUri = "http://localhost"; public SdkDataverseServiceFactory(IOptions options, ILogger logger) @@ -19,6 +25,12 @@ public SdkDataverseServiceFactory(IOptions op public IOrganizationServiceAsync2 Create() { _logger.LogWarning("Creating a new ServiceClient with Url: {url}", _options.Url); + if (_options.InteractiveLogin) + { + var cstring = $"AuthType=OAuth;Url={_options.Url};RedirectUri={DefaultMicrosoftRedirectUri};AppId={DefaultMicrosoftAppId};LoginPrompt=Auto"; + return new ServiceClient(dataverseConnectionString: cstring, + logger: _logger); + } var serviceClient = new ServiceClient( new Uri(_options.Url), _options.ClientId.ToString(), diff --git a/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Services/Dataverse/DataverseDomainService.cs b/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Services/Dataverse/DataverseDomainService.cs index 4be8b19..1190fa1 100644 --- a/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Services/Dataverse/DataverseDomainService.cs +++ b/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Services/Dataverse/DataverseDomainService.cs @@ -3,7 +3,9 @@ using Dataverse.ConfigurationMigrationTool.Console.Features.Export.Mappers; using Dataverse.ConfigurationMigrationTool.Console.Features.Shared; using Dataverse.ConfigurationMigrationTool.Console.Features.Shared.Domain; +using Dataverse.ConfigurationMigrationTool.Console.Services.Dataverse.Configuration; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Microsoft.PowerPlatform.Dataverse.Client; using Microsoft.Xrm.Sdk; using Microsoft.Xrm.Sdk.Metadata; @@ -14,11 +16,14 @@ public class DataverseDomainService : IDomainService { private readonly IOrganizationServiceAsync2 _orgService; private readonly ILogger _logger; - private static readonly IMapper<(EntitySchema, Entity), Record> _recordMapper = new DataverseRecordToRecordMapper(); - public DataverseDomainService(IOrganizationServiceAsync2 orgService, ILogger logger) + private readonly DataverseDomainServiceOptions _options; + private readonly IMapper<(EntitySchema, Entity), Record> _recordMapper; + public DataverseDomainService(IOrganizationServiceAsync2 orgService, IOptions options, ILogger logger) { + this._options = options.Value; _orgService = orgService; _logger = logger; + _recordMapper = new DataverseRecordToRecordMapper(this._options.AllowEmptyFields); } public async Task> GetRecords(EntitySchema Schema) diff --git a/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.XrmToolBox/ConfigurationMigrationControl.cs b/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.XrmToolBox/ConfigurationMigrationControl.cs index 8ad3e50..ee81b0a 100644 --- a/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.XrmToolBox/ConfigurationMigrationControl.cs +++ b/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.XrmToolBox/ConfigurationMigrationControl.cs @@ -47,10 +47,9 @@ private void InitializeInformativeText() var info = @"{\rtf1\ansi\deff0 {\fonttbl{\f0\fnil\fcharset0 Calibri;}} \par {\b **Information**} - \par Currently , only schema file generation is available with this tool. \par + \par After reflection, Data import and data export will only be handled by the CLI tool from the project repo (see link below) \par - \par Data Export and Data Import are planned to be available soon. - \par In the mean time, the generated schema file can be used to import and export data using the cli tool in the project repo. + \par The generated schema file can be used to import and export data using the cli tool in the project repo. \par A service principal that have permissions to your dataverse environment will be needed.\par \par {\b\i Project repo:} From f1f77feb63d227e562239976d21c3130a46931e3 Mon Sep 17 00:00:00 2001 From: dotnetprog <24593889+dotnetprog@users.noreply.github.com> Date: Fri, 19 Sep 2025 10:53:59 -0400 Subject: [PATCH 2/5] layout docs fixed --- Guide-XrmToolBox.md | 6 +++--- README.md | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Guide-XrmToolBox.md b/Guide-XrmToolBox.md index ac2d722..55a38f0 100644 --- a/Guide-XrmToolBox.md +++ b/Guide-XrmToolBox.md @@ -10,12 +10,12 @@ This tool was created to support more data types that the official tool does. ### Upcoming features ๐Ÿ”œ ๐Ÿ”œ Supports for multiselect optionset \ -๐Ÿ”œ ~~Configuration Data Importation~~ This is no longer planned.\ +๐Ÿ”œ ~~Configuration Data Importation~~ $${\color{red}This is no longer planned.}$$ \ ๐Ÿ”œ ~~Configuration Data Exportation~~ This is no longer planned. > [!IMPORTANT] -> Data import/export features will only be available through the cli tool. -> Since the cli tool is made with .Net Core and XTB Plugins is in .Net Framework, It's really hard to keep a sharable codebase and retricts the usage of some modern libraries. +> Data import/export features will only be available through the cli tool. \ +> Since the cli tool is made with .Net Core and XTB Plugins is in .Net Framework, It's really hard to keep a sharable codebase and retricts the usage of some modern libraries. \ > CLI tool documentation [here](https://github.com/dotnetprog/dataverse-configuration-migration-tool) diff --git a/README.md b/README.md index 383c84d..a7726d1 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ This repository contains a custom .NET CLI tool designed to export and import co Get latest version of the tool built on this [release](https://github.com/dotnetprog/dataverse-configuration-migration-tool/releases/latest) > [!NOTE] > Interactive Login is now available with the CLI tool. The use of service principal is no more mandatory but still recommanded for automation scenarios. \ -> To automate the use of the tool, let's say within a pipeline (e.g Github Action,Azure Devops Pipeline),`appsettings.Production.json` will need to be setup manually with your azure service principal credentials. +> To automate the use of the tool, let's say within a pipeline (e.g Github Action,Azure Devops Pipeline),`appsettings.Production.json` will need to be setup manually with your azure service principal credentials. \ > [Quick Guide](https://recursion.no/blogs/dataverse-setup-service-principal-access-for-environment/) to create an azure service principal ## Why โ“ @@ -43,7 +43,7 @@ This new tool enables you to: > [!IMPORTANT] -> Data import/export features for XrmToolBox is no longer planned. +> Data import/export features for XrmToolBox is no longer planned. \ > Since the cli tool is made with .Net Core and XTB Plugins is in .Net Framework, It's really hard to keep a sharable codebase and retricts the usage of some modern libraries. @@ -131,10 +131,10 @@ Dataverse.ConfigurationMigrationTool.Console.exe export-data --schema "C:\temp\d ``` > [!TIP] -> To use service principal credentials, the interactive login flag `--il` or `-il` must be omitted. -> If you use the executable directly, you must edit the `appsettings.Production.json` file with your service principal credentials -> If you use `dotnet run`, you can set the secrets using `dotnet user-secrets` as shown above or edit the `appsettings.Development.json` file. -> you can also set environment variables `Dataverse__ClientId`, `Dataverse__ClientSecret` and `Dataverse__Url` to override the settings in the json files. +> To use service principal credentials, the interactive login flag `--il` or `-il` must be omitted. \ +> If you use the executable directly, you must edit the `appsettings.Production.json` file with your service principal credentials \ +> If you use `dotnet run`, you can set the secrets using `dotnet user-secrets` as shown above or edit the `appsettings.Development.json` file. \ +> you can also set environment variables `Dataverse__ClientId`, `Dataverse__ClientSecret` and `Dataverse__Url` to override the settings in the json files. \ > you can also use these commandline arguments `--Dataverse:ClientId`, `--Dataverse:ClientSecret` and `--Dataverse:Url` to override the settings in the json files. ## ๐Ÿค Contributing ๐Ÿค From 960002a1cb5c3dfdb49fcbcae5717ecf96f6859c Mon Sep 17 00:00:00 2001 From: dotnetprog <24593889+dotnetprog@users.noreply.github.com> Date: Fri, 19 Sep 2025 10:55:05 -0400 Subject: [PATCH 3/5] layout docs fixed --- Guide-XrmToolBox.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Guide-XrmToolBox.md b/Guide-XrmToolBox.md index 55a38f0..cc98223 100644 --- a/Guide-XrmToolBox.md +++ b/Guide-XrmToolBox.md @@ -10,7 +10,7 @@ This tool was created to support more data types that the official tool does. ### Upcoming features ๐Ÿ”œ ๐Ÿ”œ Supports for multiselect optionset \ -๐Ÿ”œ ~~Configuration Data Importation~~ $${\color{red}This is no longer planned.}$$ \ +๐Ÿ”œ ~~Configuration Data Importation~~ $${\color{red}This \space is \space no \space longer \space planned.}$$ \ ๐Ÿ”œ ~~Configuration Data Exportation~~ This is no longer planned. > [!IMPORTANT] From e5f2698a207bbb0a1ccb1cfd2593d1e2814f86f3 Mon Sep 17 00:00:00 2001 From: dotnetprog <24593889+dotnetprog@users.noreply.github.com> Date: Fri, 19 Sep 2025 10:57:00 -0400 Subject: [PATCH 4/5] layout docs fixed --- Guide-XrmToolBox.md | 8 ++++---- README.md | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Guide-XrmToolBox.md b/Guide-XrmToolBox.md index cc98223..59923ed 100644 --- a/Guide-XrmToolBox.md +++ b/Guide-XrmToolBox.md @@ -11,12 +11,12 @@ This tool was created to support more data types that the official tool does. ### Upcoming features ๐Ÿ”œ ๐Ÿ”œ Supports for multiselect optionset \ ๐Ÿ”œ ~~Configuration Data Importation~~ $${\color{red}This \space is \space no \space longer \space planned.}$$ \ -๐Ÿ”œ ~~Configuration Data Exportation~~ This is no longer planned. +๐Ÿ”œ ~~Configuration Data Exportation~~ $${\color{red}This \space is \space no \space longer \space planned.}$$ > [!IMPORTANT] -> Data import/export features will only be available through the cli tool. \ -> Since the cli tool is made with .Net Core and XTB Plugins is in .Net Framework, It's really hard to keep a sharable codebase and retricts the usage of some modern libraries. \ -> CLI tool documentation [here](https://github.com/dotnetprog/dataverse-configuration-migration-tool) +> - Data import/export features will only be available through the cli tool. \ +> - Since the cli tool is made with .Net Core and XTB Plugins is in .Net Framework, It's really hard to keep a sharable codebase and retricts the usage of some modern libraries. \ +> - CLI tool documentation [here](https://github.com/dotnetprog/dataverse-configuration-migration-tool) ## What's a schema definition file exactly ๐Ÿค”โ“ diff --git a/README.md b/README.md index a7726d1..1cc3810 100644 --- a/README.md +++ b/README.md @@ -5,9 +5,9 @@ This repository contains a custom .NET CLI tool designed to export and import co ### Download latest release Get latest version of the tool built on this [release](https://github.com/dotnetprog/dataverse-configuration-migration-tool/releases/latest) > [!NOTE] -> Interactive Login is now available with the CLI tool. The use of service principal is no more mandatory but still recommanded for automation scenarios. \ -> To automate the use of the tool, let's say within a pipeline (e.g Github Action,Azure Devops Pipeline),`appsettings.Production.json` will need to be setup manually with your azure service principal credentials. \ -> [Quick Guide](https://recursion.no/blogs/dataverse-setup-service-principal-access-for-environment/) to create an azure service principal +> - Interactive Login is now available with the CLI tool. The use of service principal is no more mandatory but still recommanded for automation scenarios. \ +> - To automate the use of the tool, let's say within a pipeline (e.g Github Action,Azure Devops Pipeline),`appsettings.Production.json` will need to be setup manually with your azure service principal credentials. \ +> - [Quick Guide](https://recursion.no/blogs/dataverse-setup-service-principal-access-for-environment/) to create an azure service principal ## Why โ“ Configuration Migration Tool and the PowerPlatform Cli Tool (pac data import verb) seem to have it's limitations when automating in ci/cd. Also, these two only works on a windows machine. \ @@ -131,11 +131,11 @@ Dataverse.ConfigurationMigrationTool.Console.exe export-data --schema "C:\temp\d ``` > [!TIP] -> To use service principal credentials, the interactive login flag `--il` or `-il` must be omitted. \ -> If you use the executable directly, you must edit the `appsettings.Production.json` file with your service principal credentials \ -> If you use `dotnet run`, you can set the secrets using `dotnet user-secrets` as shown above or edit the `appsettings.Development.json` file. \ -> you can also set environment variables `Dataverse__ClientId`, `Dataverse__ClientSecret` and `Dataverse__Url` to override the settings in the json files. \ -> you can also use these commandline arguments `--Dataverse:ClientId`, `--Dataverse:ClientSecret` and `--Dataverse:Url` to override the settings in the json files. +> - To use service principal credentials, the interactive login flag `--il` or `-il` must be omitted. \ +> - If you use the executable directly, you must edit the `appsettings.Production.json` file with your service principal credentials \ +> - If you use `dotnet run`, you can set the secrets using `dotnet user-secrets` as shown above or edit the `appsettings.Development.json` file. \ +> - you can also set environment variables `Dataverse__ClientId`, `Dataverse__ClientSecret` and `Dataverse__Url` to override the settings in the json files. \ +> - you can also use these commandline arguments `--Dataverse:ClientId`, `--Dataverse:ClientSecret` and `--Dataverse:Url` to override the settings in the json files. ## ๐Ÿค Contributing ๐Ÿค From af3a90c30ff7cd21326dd4c9a2092011ef4e3677 Mon Sep 17 00:00:00 2001 From: dotnetprog <24593889+dotnetprog@users.noreply.github.com> Date: Fri, 19 Sep 2025 11:00:56 -0400 Subject: [PATCH 5/5] layout docs fixed --- Guide-XrmToolBox.md | 4 ++-- README.md | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Guide-XrmToolBox.md b/Guide-XrmToolBox.md index 59923ed..c0d85a4 100644 --- a/Guide-XrmToolBox.md +++ b/Guide-XrmToolBox.md @@ -14,8 +14,8 @@ This tool was created to support more data types that the official tool does. ๐Ÿ”œ ~~Configuration Data Exportation~~ $${\color{red}This \space is \space no \space longer \space planned.}$$ > [!IMPORTANT] -> - Data import/export features will only be available through the cli tool. \ -> - Since the cli tool is made with .Net Core and XTB Plugins is in .Net Framework, It's really hard to keep a sharable codebase and retricts the usage of some modern libraries. \ +> - Data import/export features will only be available through the cli tool. +> - Since the cli tool is made with .Net Core and XTB Plugins is in .Net Framework, It's really hard to keep a sharable codebase and retricts the usage of some modern libraries. > - CLI tool documentation [here](https://github.com/dotnetprog/dataverse-configuration-migration-tool) diff --git a/README.md b/README.md index 1cc3810..6bf90ad 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,8 @@ This repository contains a custom .NET CLI tool designed to export and import co ### Download latest release Get latest version of the tool built on this [release](https://github.com/dotnetprog/dataverse-configuration-migration-tool/releases/latest) > [!NOTE] -> - Interactive Login is now available with the CLI tool. The use of service principal is no more mandatory but still recommanded for automation scenarios. \ -> - To automate the use of the tool, let's say within a pipeline (e.g Github Action,Azure Devops Pipeline),`appsettings.Production.json` will need to be setup manually with your azure service principal credentials. \ +> - Interactive Login is now available with the CLI tool. The use of service principal is no more mandatory but still recommanded for automation scenarios. +> - To automate the use of the tool, let's say within a pipeline (e.g Github Action,Azure Devops Pipeline),`appsettings.Production.json` will need to be setup manually with your azure service principal credentials. > - [Quick Guide](https://recursion.no/blogs/dataverse-setup-service-principal-access-for-environment/) to create an azure service principal ## Why โ“ @@ -43,8 +43,8 @@ This new tool enables you to: > [!IMPORTANT] -> Data import/export features for XrmToolBox is no longer planned. \ -> Since the cli tool is made with .Net Core and XTB Plugins is in .Net Framework, It's really hard to keep a sharable codebase and retricts the usage of some modern libraries. +> - Data import/export features for XrmToolBox is no longer planned. +> - Since the cli tool is made with .Net Core and XTB Plugins is in .Net Framework, It's really hard to keep a sharable codebase and retricts the usage of some modern libraries. @@ -131,10 +131,10 @@ Dataverse.ConfigurationMigrationTool.Console.exe export-data --schema "C:\temp\d ``` > [!TIP] -> - To use service principal credentials, the interactive login flag `--il` or `-il` must be omitted. \ -> - If you use the executable directly, you must edit the `appsettings.Production.json` file with your service principal credentials \ -> - If you use `dotnet run`, you can set the secrets using `dotnet user-secrets` as shown above or edit the `appsettings.Development.json` file. \ -> - you can also set environment variables `Dataverse__ClientId`, `Dataverse__ClientSecret` and `Dataverse__Url` to override the settings in the json files. \ +> - To use service principal credentials, the interactive login flag `--il` or `-il` must be omitted. +> - If you use the executable directly, you must edit the `appsettings.Production.json` file with your service principal credentials +> - If you use `dotnet run`, you can set the secrets using `dotnet user-secrets` as shown above or edit the `appsettings.Development.json` file. +> - you can also set environment variables `Dataverse__ClientId`, `Dataverse__ClientSecret` and `Dataverse__Url` to override the settings in the json files. > - you can also use these commandline arguments `--Dataverse:ClientId`, `--Dataverse:ClientSecret` and `--Dataverse:Url` to override the settings in the json files. ## ๐Ÿค Contributing ๐Ÿค