diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ca37f3faba..704f420789 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -59,6 +59,12 @@ jobs: mysql-version: '8.0' root-password: 'YourStrong!Passw0rd' auto-start: true + - name: Create MySql Logging Db + run: dotnet run -c Release --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver LiveLoggingServer_ID "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_logging2" --dir ~/rdmp/rdmp-yaml/ + - name: Create MySql DQE Db + run: dotnet run -c Release --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver DQE "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_dqe" --dir ~/rdmp/rdmp-yaml/ + - name: Create MySql Cohort Building Query Caching Db + run: dotnet run -c Release --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver CohortIdentificationQueryCachingServer_ID "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_cache" --dir ~/rdmp/rdmp-yaml/ - name: Build run: dotnet build --configuration Release --verbosity minimal - name: Initialise RDMP diff --git a/CHANGELOG.md b/CHANGELOG.md index c47a20b537..e8a2145232 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ... -### Changed -- Removed restriction preventing [Lookup] requiring all foreign key columns being from the same table [#1331](https://github.com/HicServices/RDMP/issues/1307) -- If there are multiple IsPrimaryExtractionTable involved in a query then the one with the IsExtractionIdentifier column (if any) will be picked (previously QueryBuildingException was thrown) [#1365](https://github.com/HicServices/RDMP/issues/1365) - ### Added - Added 'Set Description' command to [AggregateConfiguration] context menu - Template cohort builder aggregates can be dragged onto extraction datasets to import the container tree [#1307](https://github.com/HicServices/RDMP/issues/1307) @@ -20,13 +16,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added ability to pop out tooltips/problems into modal popup [#1334](https://github.com/HicServices/RDMP/issues/1334) ### Changed - +- The 'Core' folder in extraction execution user interface is no longer disabled when empty [#1377](https://github.com/HicServices/RDMP/issues/1377) - Datasets in extraction UI are no longer expanded by default (i.e. to show Supporting Documents/Sql) [#1264](https://github.com/HicServices/RDMP/issues/1264) +- Removed restriction preventing [Lookup] requiring all foreign key columns being from the same table [#1331](https://github.com/HicServices/RDMP/issues/1307) +- If there are multiple IsPrimaryExtractionTable involved in a query then the one with the IsExtractionIdentifier column (if any) will be picked (previously QueryBuildingException was thrown) [#1365](https://github.com/HicServices/RDMP/issues/1365) ### Fixed - Running RDMP cli without supplying repository connection details (and after deleting `Databases.yaml`) now results in a specific error message instead of null reference [#1346]https://github.com/HicServices/RDMP/issues/1346 - Fixed Pipeline components who run in threaded but call UI methods resulting in unstable UI components [#1357](https://github.com/HicServices/RDMP/issues/1357) +- YamlRepository now saves LoadModuleAssembly binary content as a `.nupkg` file instead of string yaml [#1351](https://github.com/HicServices/RDMP/issues/1351) +- Fixed Console Gui activator 'Select File' dialog having a confusing title of "Directory" [#1282](https://github.com/HicServices/RDMP/issues/1282) ## [7.0.17] - 2022-08-01 diff --git a/Documentation/CodeTutorials/Packages.md b/Documentation/CodeTutorials/Packages.md index 3bf8fefb03..aee1f0cdfa 100644 --- a/Documentation/CodeTutorials/Packages.md +++ b/Documentation/CodeTutorials/Packages.md @@ -21,7 +21,7 @@ | NPOI | [GitHub](https://github.com/tonyqus/npoi) | [2.5.5](https://www.nuget.org/packages/NPOI/2.5.5) | Apache 2.0 | Enables reading/writing Microsoft Excel files | | ExcelNumberFormat | [GitHub](https://github.com/andersnm/ExcelNumberFormat) | [1.1.0](https://www.nuget.org/packages/ExcelNumberFormat/1.1.0) |[MIT](https://opensource.org/licenses/MIT) | Handles translating number formats from Excel formats into usable values | | | [NLog](https://nlog-project.org/) | [GitHub](https://github.com/NLog/NLog) | [5.0.2](https://www.nuget.org/packages/NLog/5.0.2) | [BSD 3-Clause](https://github.com/NLog/NLog/blob/dev/LICENSE.txt) | Flexible user configurable logging | | -| HIC.FAnsiSql |[GitHub](https://github.com/HicServices/FAnsiSql) | [2.0.5](https://www.nuget.org/packages/HIC.FansiSql/2.0.5) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | [DBMS] abstraction layer | +| HIC.FAnsiSql |[GitHub](https://github.com/HicServices/FAnsiSql) | [3.0.0](https://www.nuget.org/packages/HIC.FansiSql/3.0.0) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | [DBMS] abstraction layer | | HIC.BadMedicine | [GitHub](https://github.com/HicServices/BadMedicine) | [1.1.0](https://www.nuget.org/packages/HIC.BadMedicine/1.1.0) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Generate Test Datasets for tests/exericses | | SSH.NET | [GitHub](https://github.com/sshnet/SSH.NET) | [2020.0.2](https://www.nuget.org/packages/SSH.NET/2020.0.2) | [MIT](https://github.com/sshnet/SSH.NET/blob/develop/LICENSE) | Enables fetching files from SFTP servers | | Moq 4 | [GitHub](https://github.com/moq/moq4) | [4.18.2](https://www.nuget.org/packages/Moq/4.18.2) |[BSD 3](https://github.com/moq/moq4/blob/master/License.txt) | Mock objects during unit testing | @@ -45,7 +45,7 @@ | System.Security.Permissions |[GitHub](https://github.com/dotnet/corefx) | [6.0.0](https://www.nuget.org/packages/System.Security.Permissions/6.0.0) |[MIT](https://opensource.org/licenses/MIT) | Provides common types for Xml doc reading in UI code | | | [AutoComplete Console](https://www.codeproject.com/Articles/1182358/Using-Autocomplete-in-Windows-Console-Applications) by Jasper Lammers | Embedded | 4.0 | [CPOL](https://www.codeproject.com/info/cpol10.aspx) | Provides interactive autocomplete in console input | | | System.Resources.Extensions | [GitHub](https://github.com/dotnet/corefx) | [4.6.0](https://www.nuget.org/packages/System.Resources.Extensions/4.6.0) | [MIT](https://opensource.org/licenses/MIT) | Allows [publishing with dotnet publish on machines with netcoreapp3.0 SDK installed](https://github.com/microsoft/msbuild/issues/4704#issuecomment-530034240) | | -| ReadLine | [GitHub](https://github.com/tonerdo/readline) | [2.0.1](https://www.nuget.org/packages/ReadLine/2.0.1) | [MIT](https://opensource.org/licenses/MIT) | Allows autocomplete on command line | | +| Spectre.Console | [GitHub](https://github.com/spectreconsole/spectre.console) | [0.44.0](https://www.nuget.org/packages/Spectre.Console/0.44.0) | [MIT](https://opensource.org/licenses/MIT) | Allows richer command line interactions| | | HIC.System.Windows.Forms.DataVisualization | [GitHub](https://github.com/HicServices/winforms-datavisualization) | [1.0.1](https://www.nuget.org/packages/HIC.System.Windows.Forms.DataVisualization/1.0.1) |[MIT](https://opensource.org/licenses/MIT) | Dotnet core support for DQE charts | | | System.DirectoryServices.Protocols | [GitHub](https://github.com/dotnet/runtime) | [6.0.1](https://www.nuget.org/packages/System.DirectoryServices.Protocols/6.0.1) | MIT | Required dependency of Oracle when using LDAP auth | | Autoupdater.NET | [GitHub](https://github.com/ravibpatel/AutoUpdater.NET) | [1.7.0](https://github.com/ravibpatel/AutoUpdater.NET) | MIT | Manages updating of the RDMP windows client directly from the RDMP GitHub Releases| diff --git a/Plugins/Plugin.Test/Plugin.Test.nuspec b/Plugins/Plugin.Test/Plugin.Test.nuspec index 450358c048..f0c3c27967 100644 --- a/Plugins/Plugin.Test/Plugin.Test.nuspec +++ b/Plugins/Plugin.Test/Plugin.Test.nuspec @@ -14,10 +14,11 @@ Copyright 2018-2019 - + + @@ -37,8 +38,7 @@ - - + diff --git a/Plugins/Plugin.UI/Plugin.UI.nuspec b/Plugins/Plugin.UI/Plugin.UI.nuspec index 34ebd6de23..80ba49f04d 100644 --- a/Plugins/Plugin.UI/Plugin.UI.nuspec +++ b/Plugins/Plugin.UI/Plugin.UI.nuspec @@ -14,7 +14,7 @@ Copyright 2018-2019 - + @@ -22,6 +22,7 @@ + @@ -40,8 +41,7 @@ - - + diff --git a/Plugins/Plugin/Plugin.nuspec b/Plugins/Plugin/Plugin.nuspec index d9381ce919..47b3f249cb 100644 --- a/Plugins/Plugin/Plugin.nuspec +++ b/Plugins/Plugin/Plugin.nuspec @@ -16,8 +16,9 @@ + - + @@ -33,8 +34,7 @@ - - + diff --git a/Rdmp.Core.Tests/Curation/Integration/QueryBuildingTests/AggregateBuilderTests/MySqlAggregateBuilderTests.cs b/Rdmp.Core.Tests/Curation/Integration/QueryBuildingTests/AggregateBuilderTests/MySqlAggregateBuilderTests.cs index be33557d78..5dcda80250 100644 --- a/Rdmp.Core.Tests/Curation/Integration/QueryBuildingTests/AggregateBuilderTests/MySqlAggregateBuilderTests.cs +++ b/Rdmp.Core.Tests/Curation/Integration/QueryBuildingTests/AggregateBuilderTests/MySqlAggregateBuilderTests.cs @@ -8,6 +8,7 @@ using NUnit.Framework; using Rdmp.Core.Curation.Data; using Rdmp.Core.QueryBuilding; +using ReusableLibraryCode.Settings; namespace Rdmp.Core.Tests.Curation.Integration.QueryBuildingTests.AggregateBuilderTests { @@ -44,21 +45,25 @@ Col1 desc topx.DeleteInDatabase(); } - [Test] - public void Test_AggregateBuilder_MySql_Top31OrderByCountAsc() + [TestCase(true)] + [TestCase(false)] + public void Test_AggregateBuilder_MySql_Top31OrderByCountAsc(bool useAliasForGroupBy) { _ti.DatabaseType = DatabaseType.MySql; _ti.SaveToDatabase(); + UserSettings.UseAliasInsteadOfTransformInGroupByAggregateGraphs = useAliasForGroupBy; + var builder = new AggregateBuilder(null, "count(*)", null); builder.AddColumn(_dimension1); var topx = new AggregateTopX(CatalogueRepository, _configuration, 31); topx.OrderByDirection = AggregateTopXOrderByDirection.Ascending; builder.AggregateTopX = topx; - - Assert.AreEqual(CollapseWhitespace(@"/**/ + if (useAliasForGroupBy) + { + Assert.AreEqual(CollapseWhitespace(@"/**/ SELECT Col1, count(*) AS MyCount @@ -69,9 +74,27 @@ group by order by MyCount asc LIMIT 31"), CollapseWhitespace(builder.SQL)); + } + else + { + Assert.AreEqual(CollapseWhitespace(@"/**/ +SELECT +Col1, +count(*) AS MyCount +FROM +T1 +group by +Col1 +order by +count(*) asc +LIMIT 31"), CollapseWhitespace(builder.SQL)); + } + topx.DeleteInDatabase(); + + UserSettings.UseAliasInsteadOfTransformInGroupByAggregateGraphs = false; } } } diff --git a/Rdmp.Core.Tests/Curation/YamlRepositoryTests.cs b/Rdmp.Core.Tests/Curation/YamlRepositoryTests.cs index 7d94b8b553..32825fe2b4 100644 --- a/Rdmp.Core.Tests/Curation/YamlRepositoryTests.cs +++ b/Rdmp.Core.Tests/Curation/YamlRepositoryTests.cs @@ -282,15 +282,17 @@ public void YamlRepository_LoadSavePluginClass() var lma2 = UnitTests.WhenIHaveA(repo1); - lma1.Plugin.Name = "MyPlugin"; + lma1.Plugin.Name = "MyPlugin1.1.1.1.nupkg"; lma1.Plugin.RdmpVersion = new Version(version); //the version of Rdmp.Core targetted lma1.Plugin.PluginVersion = new Version(1, 1, 1, 1); //the version of the plugin lma1.Plugin.SaveToDatabase(); + lma1.SaveToDatabase(); - lma2.Plugin.Name = "MyPlugin"; + lma2.Plugin.Name = "MyPlugin1.1.1.2.nupkg"; lma2.Plugin.RdmpVersion = new Version(version);//the version of Rdmp.Core targetted (same as above) lma2.Plugin.PluginVersion = new Version(1, 1, 1, 2);//the version of the plugin (higher) lma2.Plugin.SaveToDatabase(); + lma2.SaveToDatabase(); var plugins = repo1.PluginManager.GetCompatiblePlugins(); Assert.That(plugins, Has.Length.EqualTo(1)); diff --git a/Rdmp.Core.Tests/Logging/DataLoadTaskHelper.cs b/Rdmp.Core.Tests/Logging/DataLoadTaskHelper.cs index 1a1eaab1ec..c6ddcc0163 100644 --- a/Rdmp.Core.Tests/Logging/DataLoadTaskHelper.cs +++ b/Rdmp.Core.Tests/Logging/DataLoadTaskHelper.cs @@ -4,6 +4,7 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . +using System; using System.Collections.Generic; using System.Linq; using FAnsi.Discovery; @@ -41,9 +42,11 @@ public void CreateDataLoadTask(string taskName) var taskCmd = _loggingServer.GetCommand( - "INSERT INTO DataLoadTask VALUES (100, '" + taskName + "', '" + taskName + "', GETDATE(), '" + datasetName + "', 1, 1, '" + datasetName + "')", + "INSERT INTO DataLoadTask VALUES (100, '" + taskName + "', '" + taskName + "',@date, '" + datasetName + "', 1, 1, '" + datasetName + "')", con); + _loggingServer.AddParameterWithValueToCommand("@date", taskCmd, DateTime.Now); + taskCmd.ExecuteNonQuery(); _sqlToCleanUp.Push("DELETE FROM DataLoadTask WHERE dataSetID = '" + datasetName + "'"); } diff --git a/Rdmp.Core.Tests/Logging/LogManagerTest.cs b/Rdmp.Core.Tests/Logging/LogManagerTest.cs index 7de633a10b..92c5526c51 100644 --- a/Rdmp.Core.Tests/Logging/LogManagerTest.cs +++ b/Rdmp.Core.Tests/Logging/LogManagerTest.cs @@ -238,6 +238,9 @@ public void LoggingDatabase_TestActuallyCreatingIt(DatabaseType type) Assert.AreEqual("bad.cs", archival.Errors.Single().Source); Assert.AreEqual("Wrote some records", archival.Progress.Single().Description); + + var fatal = archival.Errors.Single(); + lm.ResolveFatalErrors(new[] { fatal.ID }, DataLoadInfo.FatalErrorStates.Resolved, "problem resolved by building more towers"); } } } diff --git a/Rdmp.Core/CommandExecution/BasicActivateItems.cs b/Rdmp.Core/CommandExecution/BasicActivateItems.cs index bf34c4948a..6ac6db85d1 100644 --- a/Rdmp.Core/CommandExecution/BasicActivateItems.cs +++ b/Rdmp.Core/CommandExecution/BasicActivateItems.cs @@ -779,7 +779,7 @@ public virtual ExternalDatabaseServer CreateNewPlatformDatabase(ICatalogueReposi throw new ArgumentException($"Database must be picked before calling {nameof(CreateNewPlatformDatabase)} when using {nameof(BasicActivateItems)}",nameof(db)); MasterDatabaseScriptExecutor executor = new MasterDatabaseScriptExecutor(db); - executor.CreateAndPatchDatabase(patcher,new AcceptAllCheckNotifier()); + executor.CreateAndPatchDatabase(patcher, new AcceptAllCheckNotifier() { WriteToConsole = true}); var eds = new ExternalDatabaseServer(catalogueRepository,"New " + (defaultToSet == PermissableDefaults.None ? "" : defaultToSet.ToString()) + "Server",patcher); eds.SetProperties(db); diff --git a/Rdmp.Core/CommandLine/Interactive/AutoComplete.cs b/Rdmp.Core/CommandLine/Interactive/AutoComplete.cs index e6511ccc27..5c2eb4aa56 100644 --- a/Rdmp.Core/CommandLine/Interactive/AutoComplete.cs +++ b/Rdmp.Core/CommandLine/Interactive/AutoComplete.cs @@ -9,7 +9,7 @@ namespace Rdmp.Core.CommandLine.Interactive { - class AutoComplete : IAutoCompleteHandler + class AutoComplete { private readonly string[] autocompletes; diff --git a/Rdmp.Core/CommandLine/Interactive/ConsoleInputManager.cs b/Rdmp.Core/CommandLine/Interactive/ConsoleInputManager.cs index 30b4a22d69..96dea5c880 100644 --- a/Rdmp.Core/CommandLine/Interactive/ConsoleInputManager.cs +++ b/Rdmp.Core/CommandLine/Interactive/ConsoleInputManager.cs @@ -7,26 +7,24 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using System.Globalization; using System.IO; using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; using FAnsi.Discovery; using MapsDirectlyToDatabaseTable; -using Rdmp.Core.CohortCommitting.Pipeline; using Rdmp.Core.CommandExecution; using Rdmp.Core.CommandLine.Interactive.Picking; -using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.Aggregation; using Rdmp.Core.Curation.Data.DataLoad; -using Rdmp.Core.DataExport.Data; using Rdmp.Core.DataExport.DataExtraction; using Rdmp.Core.DataViewing; -using Rdmp.Core.Logging; using Rdmp.Core.Repositories; -using Rdmp.Core.Startup; using ReusableLibraryCode; using ReusableLibraryCode.Checks; using ReusableLibraryCode.DataAccess; +using Spectre.Console; namespace Rdmp.Core.CommandLine.Interactive { @@ -59,9 +57,23 @@ public override void Show(string title,string message) public override bool TypeText(DialogArgs args, int maxLength, string initialText, out string text, bool requireSaneHeaderText) { - WritePromptFor(args); - text = ReadLineWithAuto(); - return !string.IsNullOrWhiteSpace(text); + text = AnsiConsole.Prompt( + new TextPrompt(GetPromptFor(args)) + .AllowEmpty() + ); + + if(string.Equals(text , "Cancel",StringComparison.CurrentCultureIgnoreCase)) + { + // user does not want to type any text + return false; + } + + // user typed "null" or some spaces or something + if (IsBasicallyNull(text)) + text = null; + + // thats still an affirmative choice + return true; } public override DiscoveredDatabase SelectDatabase(bool allowDatabaseCreation, string taskDescription) @@ -69,8 +81,7 @@ public override DiscoveredDatabase SelectDatabase(bool allowDatabaseCreation, st if (DisallowInput) throw new InputDisallowedException($"Value required for '{taskDescription}'"); - Console.WriteLine(taskDescription); - var value = ReadLineWithAuto(new PickDatabase()); + var value = ReadLineWithAuto(new DialogArgs { WindowTitle = taskDescription}, new PickDatabase()); return value.Database; } @@ -79,8 +90,7 @@ public override DiscoveredTable SelectTable(bool allowDatabaseCreation, string t if (DisallowInput) throw new InputDisallowedException($"Value required for '{taskDescription}'"); - Console.WriteLine(taskDescription); - var value = ReadLineWithAuto(new PickTable()); + var value = ReadLineWithAuto(new DialogArgs { WindowTitle = taskDescription },new PickTable()); return value.Table; } @@ -133,11 +143,8 @@ public override bool SelectType(DialogArgs args, Type[] available,out Type chose public override IMapsDirectlyToDatabaseTable[] SelectMany(DialogArgs args, Type arrayElementType, IMapsDirectlyToDatabaseTable[] availableObjects) { - WritePromptFor(args); - - var value = ReadLineWithAuto(new PickObjectBase[] - {new PickObjectByID(this), new PickObjectByName(this)}, - availableObjects.Select(t=>t.GetType().Name).Distinct()); + var value = ReadLineWithAuto(args,new PickObjectBase[] + {new PickObjectByID(this), new PickObjectByName(this)}); var unavailable = value.DatabaseEntities.Except(availableObjects).ToArray(); @@ -152,28 +159,57 @@ public override bool SelectType(DialogArgs args, Type[] available,out Type chose /// /// /// + /// /// Thrown if is true - private void WritePromptFor(DialogArgs args, bool entryLabel = true) + private string GetPromptFor(DialogArgs args, bool entryLabel = true, params PickObjectBase[] pickers) { if (DisallowInput) throw new InputDisallowedException($"Value required for '{args}'"); + var sb = new StringBuilder(); + if (!string.IsNullOrWhiteSpace(args.WindowTitle)) { - Console.WriteLine(args.WindowTitle); + sb.Append(Markup.Escape(args.WindowTitle)); + + if(entryLabel && !string.IsNullOrWhiteSpace(args.EntryLabel)) + { + sb.Append(" - "); + } } - + + if (entryLabel && !string.IsNullOrWhiteSpace(args.EntryLabel)) + { + sb.Append($"[green]{Markup.Escape(args.EntryLabel)}[/]"); + } + if (!string.IsNullOrWhiteSpace(args.TaskDescription)) { - Console.WriteLine(args.TaskDescription); + sb.AppendLine(); + sb.Append($"[grey]{Markup.Escape(args.TaskDescription)}[/]"); } - - if (entryLabel && !string.IsNullOrWhiteSpace(args.EntryLabel)) + foreach(var picker in pickers) { - Console.Write(args.EntryLabel); + sb.AppendLine(); + sb.Append($"Format:[grey]{Markup.Escape(picker.Format)}[/]"); + + if(picker.Examples.Any()) + { + + sb.AppendLine(); + sb.Append($"Examples:"); + foreach (var example in picker.Examples) + { + sb.AppendLine(); + sb.Append($"[grey]{Markup.Escape(example)}[/]"); + } + } + sb.AppendLine(); + sb.Append(":"); } - + + return sb.ToString(); } public override IMapsDirectlyToDatabaseTable SelectOne(DialogArgs args, IMapsDirectlyToDatabaseTable[] availableObjects) @@ -192,9 +228,8 @@ public override IMapsDirectlyToDatabaseTable SelectOne(DialogArgs args, IMapsDir Console.Write(args.EntryLabel); - var value = ReadLineWithAuto(new PickObjectBase[] - {new PickObjectByID(this), new PickObjectByName(this)}, - availableObjects.Select(t=>t.GetType().Name).Distinct()); + var value = ReadLineWithAuto(args, new PickObjectBase[] + {new PickObjectByID(this), new PickObjectByName(this)}); var chosen = value.DatabaseEntities?.SingleOrDefault(); @@ -231,34 +266,18 @@ public override bool SelectObject(DialogArgs args, T[] available, out T selec return false; } - private string ReadLineWithAuto(IEnumerable autoComplete = null) + private CommandLineObjectPickerArgumentValue ReadLineWithAuto(DialogArgs args, params PickObjectBase[] pickers) { if (DisallowInput) throw new InputDisallowedException("Value required"); - ReadLine.AutoCompletionHandler = new AutoComplete(autoComplete); + var line = AnsiConsole.Prompt( + new TextPrompt( + GetPromptFor(args,true, pickers).Trim())); - return ReadLine.Read(); - } - - private CommandLineObjectPickerArgumentValue ReadLineWithAuto(PickObjectBase picker) - { - if (DisallowInput) - throw new InputDisallowedException("Value required"); - - string line = ReadLineWithAuto(picker.GetAutoCompleteIfAny()); - return picker.Parse(line, 0); - } - private CommandLineObjectPickerArgumentValue ReadLineWithAuto(PickObjectBase[] pickers,IEnumerable autoComplete) - { - if (DisallowInput) - throw new InputDisallowedException("Value required"); - - string line = ReadLineWithAuto(autoComplete); - - var picker = new CommandLineObjectPicker(new[]{line},RepositoryLocator,pickers); - return picker[0]; + var cli = new CommandLineObjectPicker(new[] { line }, RepositoryLocator, pickers); + return cli[0]; } public override DirectoryInfo SelectDirectory(string prompt) @@ -266,8 +285,19 @@ public override DirectoryInfo SelectDirectory(string prompt) if (DisallowInput) throw new InputDisallowedException($"Value required for '{prompt}'"); - Console.WriteLine(prompt); - return new DirectoryInfo(Console.ReadLine()); + var result = AnsiConsole.Prompt( + new TextPrompt( + GetPromptFor(new DialogArgs + { + WindowTitle = "Select Directory", + EntryLabel = prompt + })) + .AllowEmpty()); + + if (IsBasicallyNull(result)) + return null; + + return new DirectoryInfo(result); } public override FileInfo SelectFile(string prompt) @@ -283,31 +313,52 @@ public override FileInfo SelectFile(string prompt, string patternDescription, st if (DisallowInput) throw new InputDisallowedException($"Value required for '{prompt}'"); - Console.WriteLine(prompt); - var file = Console.ReadLine(); + var result = AnsiConsole.Prompt( + new TextPrompt( + GetPromptFor(new DialogArgs + { + WindowTitle = "Select File", + EntryLabel = prompt + })) + .AllowEmpty()); - // if user types the literal string null then return null (typically interpretted as - 'I don't want to pick a file') - // but not the same as task cancellation - if (string.Equals(file,"null", StringComparison.CurrentCultureIgnoreCase)) + if (IsBasicallyNull(result)) return null; - if (file != null) - return new FileInfo(file); + return new FileInfo(result); + } - return null; + private bool IsBasicallyNull(string result) + { + if (string.IsNullOrWhiteSpace(result)) + return true; + + // if user types the literal string null then return null (typically interpretted as - 'I don't want to pick one') + // but not the same as task cancellation + if (string.Equals(result, "null", StringComparison.CurrentCultureIgnoreCase)) + return true; + + return false; } - + public override FileInfo[] SelectFiles(string prompt, string patternDescription, string pattern) { if (DisallowInput) throw new InputDisallowedException($"Value required for '{prompt}'"); - Console.WriteLine(prompt); - Console.WriteLine(@"Enter path with optional wildcards (e.g. c:\*.csv):"); + var file = AnsiConsole.Prompt( + new TextPrompt( + GetPromptFor(new DialogArgs + { + WindowTitle = "Select File(s)", + TaskDescription = patternDescription, + EntryLabel = prompt + })) + .AllowEmpty()); - var file = Console.ReadLine(); + if (IsBasicallyNull(file)) + return null; - if (file == null) return null; var asteriskIdx = file.IndexOf('*'); if(asteriskIdx != -1) @@ -337,32 +388,34 @@ public override FileInfo[] SelectFiles(string prompt, string patternDescription, protected override bool SelectValueTypeImpl(DialogArgs args, Type paramType, object initialValue,out object chosen) { - WritePromptFor(args); - - chosen = UsefulStuff.ChangeType(ReadLineWithAuto(), paramType); - + chosen = UsefulStuff.ChangeType(AnsiConsole.Ask(GetPromptFor(args)), paramType); return true; } public override bool YesNo(DialogArgs args, out bool chosen) { - WritePromptFor(args, false); + var result = GetString(args, new List { "Yes","No","Cancel"}); - Console.WriteLine(args.EntryLabel + "(Y/n)"); - //if user picks no then it's false otherwise true - chosen = !string.Equals(Console.ReadLine()?.Trim(), "n", StringComparison.CurrentCultureIgnoreCase); - - //user made a conscious decision - return true; + if (result == "Yes") + chosen = true; + else + chosen = false; + + //user made a noncancel decision? + return result != "Cancel" && !string.IsNullOrWhiteSpace(result); } public string GetString(DialogArgs args, List options) { - WritePromptFor(args); + var chosen = AnsiConsole.Prompt( + new SelectionPrompt() + .PageSize(10) + .Title(GetPromptFor(args)) + .AddChoices(options) + ); - ReadLine.AutoCompletionHandler = new AutoComplete(options); - return ReadLine.Read(); + return chosen; } public override void ShowData(IViewSQLAndResultsCollection collection) @@ -452,5 +505,14 @@ public override void LaunchSubprocess(ProcessStartInfo startInfo) { throw new NotSupportedException(); } + + public override void Wait(string title, Task task, CancellationTokenSource cts) + { + AnsiConsole.Status() + .Spinner(Spinner.Known.Star) + .Start(title, ctx => + base.Wait(title, task, cts) + ); + } } } diff --git a/Rdmp.Core/CommandLine/Runners/ExecuteCommandRunner.cs b/Rdmp.Core/CommandLine/Runners/ExecuteCommandRunner.cs index ee0aa06aaf..fabddb28f3 100644 --- a/Rdmp.Core/CommandLine/Runners/ExecuteCommandRunner.cs +++ b/Rdmp.Core/CommandLine/Runners/ExecuteCommandRunner.cs @@ -20,6 +20,7 @@ using Rdmp.Core.Repositories; using ReusableLibraryCode.Checks; using ReusableLibraryCode.Progress; +using Spectre.Console; namespace Rdmp.Core.CommandLine.Runners { @@ -138,8 +139,7 @@ private void RunCommandExecutionLoop(IRDMPPlatformRepositoryServiceLocator repos while (true) { - Console.WriteLine("Enter Command (or 'exit')"); - var command = _input.GetString(new DialogArgs { WindowTitle = "Command" }, _commands.Keys.ToList()); + var command = _input.GetString(new DialogArgs { WindowTitle = "Enter Command (or Ctrl+C)" }, _commands.Keys.ToList()); try { command = GetCommandAndPickerFromLine(command, out _picker,repositoryLocator); @@ -151,7 +151,7 @@ private void RunCommandExecutionLoop(IRDMPPlatformRepositoryServiceLocator repos } catch (Exception ex) { - Console.WriteLine(ex.Message); + AnsiConsole.WriteException(ex, ExceptionFormats.ShortenEverything); } diff --git a/Rdmp.Core/Curation/Data/Defaults/PermissableDefaultsExtensions.cs b/Rdmp.Core/Curation/Data/Defaults/PermissableDefaultsExtensions.cs index 2caaaf93f9..ff3a5b24f9 100644 --- a/Rdmp.Core/Curation/Data/Defaults/PermissableDefaultsExtensions.cs +++ b/Rdmp.Core/Curation/Data/Defaults/PermissableDefaultsExtensions.cs @@ -27,7 +27,7 @@ public static IPatcher ToTier2DatabaseType(this PermissableDefaults permissableD case PermissableDefaults.IdentifierDumpServer_ID: return new IdentifierDumpDatabasePatcher(); case PermissableDefaults.DQE: - return new IdentifierDumpDatabasePatcher(); + return new DataQualityEnginePatcher(); case PermissableDefaults.WebServiceQueryCachingServer_ID: return new QueryCachingPatcher(); case PermissableDefaults.CohortIdentificationQueryCachingServer_ID: diff --git a/Rdmp.Core/Curation/Data/LoadModuleAssembly.cs b/Rdmp.Core/Curation/Data/LoadModuleAssembly.cs index 7756253f3f..82c88a8945 100644 --- a/Rdmp.Core/Curation/Data/LoadModuleAssembly.cs +++ b/Rdmp.Core/Curation/Data/LoadModuleAssembly.cs @@ -16,6 +16,7 @@ using Rdmp.Core.Curation.Data.ImportExport; using Rdmp.Core.Curation.Data.Serialization; using Rdmp.Core.Repositories; +using YamlDotNet.Serialization; namespace Rdmp.Core.Curation.Data { @@ -36,6 +37,7 @@ public class LoadModuleAssembly : DatabaseEntity, IInjectKnown /// /// The assembly (dll) file as a Byte[], use File.WriteAllBytes to write it to disk /// + [YamlIgnore] public Byte[] Bin { get { return _bin;} diff --git a/Rdmp.Core/DataQualityEngine/Data/ColumnState.cs b/Rdmp.Core/DataQualityEngine/Data/ColumnState.cs index 1449d90973..810abe41da 100644 --- a/Rdmp.Core/DataQualityEngine/Data/ColumnState.cs +++ b/Rdmp.Core/DataQualityEngine/Data/ColumnState.cs @@ -133,7 +133,7 @@ public void Commit(Evaluation evaluation,string pivotCategory, DbConnection con, throw new NotSupportedException("ColumnState was already committed"); var sql = string.Format( - "INSERT INTO [dbo].[ColumnState]([TargetProperty],[DataLoadRunID],[Evaluation_ID],[CountCorrect],[CountDBNull],[ItemValidatorXML],[CountMissing],[CountWrong],[CountInvalidatesRow],[PivotCategory])VALUES({0},{1},{2},{3},{4},{5},{6},{7},{8},{9})", + "INSERT INTO ColumnState(TargetProperty,DataLoadRunID,Evaluation_ID,CountCorrect,CountDBNull,ItemValidatorXML,CountMissing,CountWrong,CountInvalidatesRow,PivotCategory)VALUES({0},{1},{2},{3},{4},{5},{6},{7},{8},{9})", "@TargetProperty", DataLoadRunID ,evaluation.ID diff --git a/Rdmp.Core/DataQualityEngine/Data/PeriodicityState.cs b/Rdmp.Core/DataQualityEngine/Data/PeriodicityState.cs index 11b50654c2..048ca45d47 100644 --- a/Rdmp.Core/DataQualityEngine/Data/PeriodicityState.cs +++ b/Rdmp.Core/DataQualityEngine/Data/PeriodicityState.cs @@ -7,6 +7,7 @@ using System; using System.Collections.Generic; using System.Data; +using MapsDirectlyToDatabaseTable; using Rdmp.Core.Validation.Constraints; using ReusableLibraryCode; @@ -56,17 +57,19 @@ public PeriodicityState(int year, int month, Consequence? consequence ) var calc = new DatasetTimespanCalculator(); var result = calc.GetMachineReadableTimespanIfKnownOf(evaluation, discardOutliers); + var t = evaluation.DQERepository; + using (var con = evaluation.DQERepository.GetConnection()) { var sql = - @"SELECT - [Year] - ,[Month] + @$"SELECT + {t.Wrap("Year")} + ,{t.Wrap("Month")} ,RowEvaluation ,CountOfRecords FROM [PeriodicityState] where - Evaluation_ID = " + evaluation.ID + " and PivotCategory = 'ALL' ORDER BY [Year],[Month]"; + Evaluation_ID = ${evaluation.ID} and PivotCategory = 'ALL' ORDER BY {t.Wrap("Year")},{t.Wrap("Month")}"; using(var cmd = DatabaseCommandHelper.GetCommand(sql, con.Connection, con.Transaction)) { @@ -179,15 +182,10 @@ public void Commit(Evaluation evaluation, string pivotCategory) if (IsCommitted) throw new NotSupportedException("PeriodicityState was already committed"); + var t = evaluation.DQERepository; + string sql = - string.Format( - "INSERT INTO [dbo].[PeriodicityState]([Evaluation_ID],[Year],[Month],[CountOfRecords],[RowEvaluation],[PivotCategory])VALUES({0},{1},{2},{3},{4},{5})" - ,evaluation.ID - ,Year - ,Month - ,CountOfRecords - , "@RowEvaluation", - "@PivotCategory"); + $"INSERT INTO PeriodicityState(Evaluation_ID,{t.Wrap("Year")},{t.Wrap("Month")},CountOfRecords,RowEvaluation,PivotCategory)VALUES({evaluation.ID},{Year},{Month},{CountOfRecords},@RowEvaluation,@PivotCategory)"; using (var cmd = DatabaseCommandHelper.GetCommand(sql, con.Connection, con.Transaction)) { diff --git a/Rdmp.Core/DataQualityEngine/Data/RowState.cs b/Rdmp.Core/DataQualityEngine/Data/RowState.cs index 9225bee892..64d61ea59e 100644 --- a/Rdmp.Core/DataQualityEngine/Data/RowState.cs +++ b/Rdmp.Core/DataQualityEngine/Data/RowState.cs @@ -43,7 +43,7 @@ public RowState(Evaluation evaluation, int dataLoadRunID, int correct, int missi { var sql = string.Format( - "INSERT INTO [dbo].[RowState]([Evaluation_ID],[Correct],[Missing],[Wrong],[Invalid],[DataLoadRunID],[ValidatorXML],[PivotCategory])VALUES({0},{1},{2},{3},{4},{5},@validatorXML,{6})", + "INSERT INTO RowState(Evaluation_ID,Correct,Missing,Wrong,Invalid,DataLoadRunID,ValidatorXML,PivotCategory)VALUES({0},{1},{2},{3},{4},{5},@validatorXML,{6})", evaluation.ID, correct, missing, diff --git a/Rdmp.Core/Databases/DataQualityEnginePatcher.cs b/Rdmp.Core/Databases/DataQualityEnginePatcher.cs index 8114147057..00ea4dc4b1 100644 --- a/Rdmp.Core/Databases/DataQualityEnginePatcher.cs +++ b/Rdmp.Core/Databases/DataQualityEnginePatcher.cs @@ -23,7 +23,7 @@ public DataQualityEnginePatcher() : base(2, "Databases.DataQualityEngineDatabase } public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) { - var header = GetHeader(InitialScriptName, new Version(1, 0, 0)); + var header = GetHeader(db.Server.DatabaseType,InitialScriptName, new Version(1, 0, 0)); var sql = new StringBuilder(); @@ -32,7 +32,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) new DatabaseColumnRequest("DateOfEvaluation",new DatabaseTypeRequest(typeof(DateTime))), new DatabaseColumnRequest("CatalogueID",new DatabaseTypeRequest(typeof(int))){AllowNulls = false}, new DatabaseColumnRequest("ID",new DatabaseTypeRequest(typeof (int))){IsAutoIncrement = true, IsPrimaryKey = true} - },null,false,null)); + },null,false,null).TrimEnd() + ";"); // foreign keys var evaluationId = new DiscoveredColumn(db.ExpectTable("Evaluation"), "ID", false); @@ -57,7 +57,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) }, new Dictionary { {columnState_Evaluation_ID ,evaluationId } - },true , null)); + },true , null).TrimEnd() + ";"); sql.AppendLine(db.Helper.GetCreateTableSql(db, "RowState", new[] { @@ -72,7 +72,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) }, new Dictionary { {rowState_Evaluation_ID ,evaluationId } - }, true, null)); + }, true, null).TrimEnd() + ";"); sql.AppendLine(db.Helper.GetCreateTableSql(db, "PeriodicityState", new[] @@ -86,7 +86,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) }, new Dictionary { {periodicityState_Evaluation_ID ,evaluationId } - }, true, null)); + }, true, null).TrimEnd() + ";"); sql.AppendLine(db.Helper.GetCreateTableSql(db, "DQEGraphAnnotation", new[] { @@ -104,7 +104,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) }, new Dictionary { {annotation_Evaluation_ID ,evaluationId } - }, true, null)); + }, true, null).TrimEnd() + ";"); return new Patch(InitialScriptName, header + sql); } diff --git a/Rdmp.Core/Databases/LoggingDatabasePatcher.cs b/Rdmp.Core/Databases/LoggingDatabasePatcher.cs index f846e2ccd1..7bb789416a 100644 --- a/Rdmp.Core/Databases/LoggingDatabasePatcher.cs +++ b/Rdmp.Core/Databases/LoggingDatabasePatcher.cs @@ -24,14 +24,13 @@ public LoggingDatabasePatcher():base(2,"Databases.LoggingDatabase") public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) { - var header = GetHeader(InitialScriptName, new Version(1, 0, 0)); + var header = GetHeader(db.Server.DatabaseType, InitialScriptName, new Version(1, 0, 0)); - var sql = new StringBuilder(); sql.AppendLine(db.Helper.GetCreateTableSql(db, "DataSet", new[] { - new DatabaseColumnRequest("dataSetID",new DatabaseTypeRequest(typeof(string),450){Unicode = true}){IsPrimaryKey = true}, + new DatabaseColumnRequest("dataSetID",new DatabaseTypeRequest(typeof(string),150){Unicode = true}){IsPrimaryKey = true}, new DatabaseColumnRequest("name",new DatabaseTypeRequest(typeof(string),2000){Unicode = true}){AllowNulls = true}, new DatabaseColumnRequest("description",new DatabaseTypeRequest(typeof(string),int.MaxValue){Unicode = true}){AllowNulls = true}, new DatabaseColumnRequest("time_period",new DatabaseTypeRequest(typeof(string),64){Unicode = true}){AllowNulls = true}, @@ -46,7 +45,8 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) new DatabaseColumnRequest("contact_email",new DatabaseTypeRequest(typeof(string),64){Unicode = true}){AllowNulls = true}, new DatabaseColumnRequest("frequency",new DatabaseTypeRequest(typeof(string),32){Unicode = true}){AllowNulls = true}, new DatabaseColumnRequest("method",new DatabaseTypeRequest(typeof(string),16){Unicode = true}){AllowNulls = true} - }, null, false, null)); + }, null, false, null).TrimEnd() + ";"); + // foreign keys var datasetId = new DiscoveredColumn(db.ExpectTable("DataSet"), "dataSetID", false); @@ -73,11 +73,11 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) new DatabaseColumnRequest("userAccount",new DatabaseTypeRequest(typeof(string),500){Unicode = true}), new DatabaseColumnRequest("statusID", new DatabaseTypeRequest(typeof(int))), new DatabaseColumnRequest("isTest", new DatabaseTypeRequest(typeof(bool))), - dataLoadTask_datasetID = new DatabaseColumnRequest("dataSetID", new DatabaseTypeRequest(typeof(string), 450) { Unicode = true }), + dataLoadTask_datasetID = new DatabaseColumnRequest("dataSetID", new DatabaseTypeRequest(typeof(string), 150) { Unicode = true }), }, new Dictionary { {dataLoadTask_datasetID ,datasetId } - }, true, null)); + }, true, null).TrimEnd() + ";"); @@ -97,7 +97,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) }, new Dictionary { {dataLoadRun_dataLoadTaskID ,dataLoadTask_ID } - }, true, null)); + }, true, null).TrimEnd() + ";"); sql.AppendLine(db.Helper.GetCreateTableSql(db, "TableLoadRun", new[] { @@ -118,7 +118,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) }, new Dictionary { {tableLoadRun_dataLoadRunID ,dataLoadRun_ID } - }, true, null)); + }, true, null).TrimEnd() + ";"); sql.AppendLine(db.Helper.GetCreateTableSql(db, "DataSource", new[] { @@ -134,7 +134,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) }, new Dictionary { {dataSource_tableLoadRunID ,tableLoadRun_ID } - }, true, null)); + }, true, null).TrimEnd() + ";"); @@ -152,7 +152,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) }, new Dictionary { {fatalError_dataLoadRunID ,dataLoadRun_ID } - }, true, null)); + }, true, null).TrimEnd() + ";"); sql.AppendLine(db.Helper.GetCreateTableSql(db, "ProgressLog", new[] @@ -167,7 +167,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) }, new Dictionary { {progressLog_dataLoadRunID ,dataLoadRun_ID } - }, true, null)); + }, true, null).TrimEnd() + ";"); @@ -184,7 +184,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) }, new Dictionary { {rowError_tableLoadRunID ,tableLoadRun_ID } - }, true, null)); + }, true, null).TrimEnd() + ";"); @@ -194,44 +194,44 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) new DatabaseColumnRequest("status",new DatabaseTypeRequest(typeof(string),50){Unicode = true }){AllowNulls = true}, new DatabaseColumnRequest("description",new DatabaseTypeRequest(typeof(string),int.MaxValue)){AllowNulls = true}, - },null, true)); + },null, true).TrimEnd() + ";"); sql.AppendLine(db.Helper.GetCreateTableSql(db, "z_FatalErrorStatus", new[] { new DatabaseColumnRequest("ID",new DatabaseTypeRequest(typeof(int))){AllowNulls = false, IsPrimaryKey = true}, new DatabaseColumnRequest("status",new DatabaseTypeRequest(typeof(string),20){Unicode = true }), - }, null, true)); + }, null, true).TrimEnd() + ";"); sql.AppendLine(db.Helper.GetCreateTableSql(db, "z_RowErrorType", new[] { new DatabaseColumnRequest("ID",new DatabaseTypeRequest(typeof(int))){AllowNulls = false, IsPrimaryKey = true}, new DatabaseColumnRequest("type",new DatabaseTypeRequest(typeof(string),20){Unicode = true }), - }, null, true)); + }, null, true).TrimEnd() + ";"); sql.AppendLine(@" -INSERT INTO z_DataLoadTaskStatus(ID, status, description) VALUES(1, 'Open', NULL) -INSERT INTO z_DataLoadTaskStatus (ID, status, description) VALUES(2, 'Ready', NULL) -INSERT INTO z_DataLoadTaskStatus (ID, status, description) VALUES(3, 'Commited', NULL) -INSERT INTO z_FatalErrorStatus(ID, status) VALUES(1, 'Outstanding') -INSERT INTO z_FatalErrorStatus (ID, status) VALUES(2, 'Resolved') -INSERT INTO z_FatalErrorStatus (ID, status) VALUES(3, 'Blocked') -INSERT INTO z_RowErrorType(ID, type) VALUES(1, 'LoadRow') -INSERT INTO z_RowErrorType (ID, type) VALUES(2, 'Duplication') -INSERT INTO z_RowErrorType (ID, type) VALUES(3, 'Validation') -INSERT INTO z_RowErrorType (ID, type) VALUES(4, 'DatabaseOperation') -INSERT INTO z_RowErrorType (ID, type) VALUES(5, 'Unknown') - ---create datasets -INSERT INTO DataSet (dataSetID, name, description, time_period, SLA_required, supplier_name, supplier_tel_no, supplier_email, contact_name, contact_position, currentContactInstitutions, contact_tel_no, contact_email, frequency, method) VALUES(N'DataExtraction', 'DataExtraction', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) -INSERT INTO DataSet (dataSetID, name, description, time_period, SLA_required, supplier_name, supplier_tel_no, supplier_email, contact_name, contact_position, currentContactInstitutions, contact_tel_no, contact_email, frequency, method) VALUES(N'Internal', 'Internal', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) - ---create tasks -INSERT INTO DataLoadTask(ID, description, name, createTime, userAccount, statusID, isTest, dataSetID) VALUES(1, 'Internal', 'Internal', GETDATE(), 'Thomas', 1, 0, 'Internal') -INSERT INTO DataLoadTask (ID, description, name, createTime, userAccount, statusID, isTest, dataSetID) VALUES(2, 'DataExtraction', 'DataExtraction', GETDATE(), 'Thomas', 1, 0, 'DataExtraction') +INSERT INTO z_DataLoadTaskStatus(ID, status, description) VALUES(1, 'Open', NULL); +INSERT INTO z_DataLoadTaskStatus (ID, status, description) VALUES(2, 'Ready', NULL); +INSERT INTO z_DataLoadTaskStatus (ID, status, description) VALUES(3, 'Commited', NULL); +INSERT INTO z_FatalErrorStatus(ID, status) VALUES(1, 'Outstanding'); +INSERT INTO z_FatalErrorStatus (ID, status) VALUES(2, 'Resolved'); +INSERT INTO z_FatalErrorStatus (ID, status) VALUES(3, 'Blocked'); +INSERT INTO z_RowErrorType(ID, type) VALUES(1, 'LoadRow'); +INSERT INTO z_RowErrorType (ID, type) VALUES(2, 'Duplication'); +INSERT INTO z_RowErrorType (ID, type) VALUES(3, 'Validation'); +INSERT INTO z_RowErrorType (ID, type) VALUES(4, 'DatabaseOperation'); +INSERT INTO z_RowErrorType (ID, type) VALUES(5, 'Unknown'); + +/*create datasets*/ +INSERT INTO DataSet (dataSetID, name, description, time_period, SLA_required, supplier_name, supplier_tel_no, supplier_email, contact_name, contact_position, currentContactInstitutions, contact_tel_no, contact_email, frequency, method) VALUES(N'DataExtraction', 'DataExtraction', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); +INSERT INTO DataSet (dataSetID, name, description, time_period, SLA_required, supplier_name, supplier_tel_no, supplier_email, contact_name, contact_position, currentContactInstitutions, contact_tel_no, contact_email, frequency, method) VALUES(N'Internal', 'Internal', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); + +/*create tasks*/ +INSERT INTO DataLoadTask(ID, description, name, userAccount, statusID, isTest, dataSetID) VALUES(1, 'Internal', 'Internal', 'Thomas', 1, 0, 'Internal'); +INSERT INTO DataLoadTask (ID, description, name, userAccount, statusID, isTest, dataSetID) VALUES(2, 'DataExtraction', 'DataExtraction', 'Thomas', 1, 0, 'DataExtraction'); "); diff --git a/Rdmp.Core/Databases/QueryCachingPatcher.cs b/Rdmp.Core/Databases/QueryCachingPatcher.cs index e5e8421e0a..9d68ca0739 100644 --- a/Rdmp.Core/Databases/QueryCachingPatcher.cs +++ b/Rdmp.Core/Databases/QueryCachingPatcher.cs @@ -25,7 +25,7 @@ public QueryCachingPatcher():base(2,"Databases.QueryCachingDatabase") public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) { - var header = GetHeader(InitialScriptName, new Version(1,0,0)); + var header = GetHeader(db.Server.DatabaseType, InitialScriptName, new Version(1,0,0)); var body = db.Helper.GetCreateTableSql(db, "CachedAggregateConfigurationResults", new[] { diff --git a/Rdmp.Core/Logging/DataLoadInfo.cs b/Rdmp.Core/Logging/DataLoadInfo.cs index b585dd1e4d..76df9951c8 100644 --- a/Rdmp.Core/Logging/DataLoadInfo.cs +++ b/Rdmp.Core/Logging/DataLoadInfo.cs @@ -6,9 +6,7 @@ using System; using System.Collections.Generic; -using System.Data; using System.Data.Common; -using Microsoft.Data.SqlClient; using FAnsi.Discovery; namespace Rdmp.Core.Logging @@ -160,15 +158,12 @@ public DataLoadInfo(string dataLoadTaskName, string packageName, string descript private void RecordNewDataLoadInDatabase(string dataLoadTaskName) { - int parentTaskID = -1; - - using (var con = (SqlConnection)_server.GetConnection()) + using (var con = _server.GetConnection()) { con.Open(); - SqlCommand cmd = new SqlCommand("SELECT ID FROM DataLoadTask WHERE name=@name", con); - cmd.Parameters.Add("@name", SqlDbType.VarChar, 255); - cmd.Parameters["@name"].Value = dataLoadTaskName; + var cmd = _server.GetCommand("SELECT ID FROM DataLoadTask WHERE name=@name", con); + _server.AddParameterWithValueToCommand("@name",cmd, dataLoadTaskName); var result = cmd.ExecuteScalar(); @@ -177,28 +172,20 @@ private void RecordNewDataLoadInDatabase(string dataLoadTaskName) throw new Exception("Could not find data load task named:" + dataLoadTaskName); //ID can come back as a decimal or an Int32 or an Int64 so whatever, just turn it into a string and then parse it - parentTaskID = int.Parse(result.ToString()); + var parentTaskID = int.Parse(result.ToString()); + + cmd = _server.GetCommand( + @"INSERT INTO DataLoadRun (description,startTime,dataLoadTaskID,isTest,packageName,userAccount,suggestedRollbackCommand) VALUES (@description,@startTime,@dataLoadTaskID,@isTest,@packageName,@userAccount,@suggestedRollbackCommand); +SELECT @@IDENTITY;", con); + _server.AddParameterWithValueToCommand("@description", cmd, _description); + _server.AddParameterWithValueToCommand("@startTime", cmd, _startTime); + _server.AddParameterWithValueToCommand("@dataLoadTaskID", cmd, parentTaskID); + _server.AddParameterWithValueToCommand("@isTest",cmd, _isTest); + _server.AddParameterWithValueToCommand("@packageName", cmd, _packageName); + _server.AddParameterWithValueToCommand("@userAccount", cmd, _userAccount); + _server.AddParameterWithValueToCommand("@suggestedRollbackCommand", cmd, _suggestedRollbackCommand ?? string.Empty); - cmd = new SqlCommand( - @"INSERT INTO DataLoadRun (description,startTime,dataLoadTaskID,isTest,packageName,userAccount,suggestedRollbackCommand) VALUES (@description,@startTime,@dataLoadTaskID,@isTest,@packageName,@userAccount,@suggestedRollbackCommand); -SELECT SCOPE_IDENTITY();", con); - - cmd.Parameters.Add("@description", SqlDbType.VarChar, -1); - cmd.Parameters.Add("@startTime", SqlDbType.DateTime); - cmd.Parameters.Add("@dataLoadTaskID", SqlDbType.Int); - cmd.Parameters.Add("@isTest", SqlDbType.Bit); - cmd.Parameters.Add("@packageName", SqlDbType.VarChar, 100); - cmd.Parameters.Add("@userAccount", SqlDbType.VarChar, 50); - cmd.Parameters.Add("@suggestedRollbackCommand", SqlDbType.VarChar, -1); - - cmd.Parameters["@description"].Value = _description; - cmd.Parameters["@startTime"].Value = _startTime; - cmd.Parameters["@dataLoadTaskID"].Value = parentTaskID; - cmd.Parameters["@isTest"].Value = _isTest; - cmd.Parameters["@packageName"].Value = _packageName; - cmd.Parameters["@userAccount"].Value = _userAccount; - cmd.Parameters["@suggestedRollbackCommand"].Value = _suggestedRollbackCommand ?? string.Empty; //ID can come back as a decimal or an Int32 or an Int64 so whatever, just turn it into a string and then parse it _id = int.Parse(cmd.ExecuteScalar().ToString()); @@ -312,25 +299,19 @@ public void LogFatalError(string errorSource, string errorDescription) //look up the fatal error ID (get hte name of the Enum so that we can refactor if nessesary without breaking the code looking for a constant string) string initialErrorStatus = Enum.GetName(typeof(FatalErrorStates), FatalErrorStates.Outstanding); - SqlCommand cmdLookupStatusID = new SqlCommand("SELECT ID from z_FatalErrorStatus WHERE status=@status", (SqlConnection)con); - cmdLookupStatusID.Parameters.Add("@status", SqlDbType.NChar, 20); - cmdLookupStatusID.Parameters["@status"].Value = initialErrorStatus; + + var cmdLookupStatusID = _server.GetCommand("SELECT ID from z_FatalErrorStatus WHERE status=@status", con); + _server.AddParameterWithValueToCommand("@status",cmdLookupStatusID, initialErrorStatus); int statusID = int.Parse(cmdLookupStatusID.ExecuteScalar().ToString()); - SqlCommand cmdRecordFatalError = new SqlCommand( - @"INSERT INTO FatalError (time,source,description,statusID,dataLoadRunID) VALUES (@time,@source,@description,@statusID,@dataLoadRunID);", (SqlConnection)con); - cmdRecordFatalError.Parameters.Add("@time", SqlDbType.DateTime); - cmdRecordFatalError.Parameters.Add("@source", SqlDbType.VarChar, 50); - cmdRecordFatalError.Parameters.Add("@description", SqlDbType.VarChar, -1); - cmdRecordFatalError.Parameters.Add("@statusID", SqlDbType.Int); - cmdRecordFatalError.Parameters.Add("@dataLoadRunID", SqlDbType.Int); - - cmdRecordFatalError.Parameters["@time"].Value = DateTime.Now; - cmdRecordFatalError.Parameters["@source"].Value = errorSource; - cmdRecordFatalError.Parameters["@description"].Value = errorDescription; - cmdRecordFatalError.Parameters["@statusID"].Value = statusID; - cmdRecordFatalError.Parameters["@dataLoadRunID"].Value = ID; + var cmdRecordFatalError = _server.GetCommand( + @"INSERT INTO FatalError (time,source,description,statusID,dataLoadRunID) VALUES (@time,@source,@description,@statusID,@dataLoadRunID);", con); + _server.AddParameterWithValueToCommand("@time", cmdRecordFatalError, DateTime.Now); + _server.AddParameterWithValueToCommand("@source", cmdRecordFatalError, errorSource); + _server.AddParameterWithValueToCommand("@description", cmdRecordFatalError, errorDescription); + _server.AddParameterWithValueToCommand("@statusID", cmdRecordFatalError, statusID); + _server.AddParameterWithValueToCommand("@dataLoadRunID", cmdRecordFatalError, ID); cmdRecordFatalError.ExecuteNonQuery(); @@ -352,24 +333,18 @@ public enum ProgressEventType public void LogProgress(ProgressEventType pevent, string Source, string Description) { - using (var con = (SqlConnection)DatabaseSettings.GetConnection()) - using (var cmdRecordProgress = new SqlCommand("INSERT INTO ProgressLog " + + using (var con = DatabaseSettings.GetConnection()) + using (var cmdRecordProgress = _server.GetCommand("INSERT INTO ProgressLog " + "(dataLoadRunID,eventType,source,description,time) " + "VALUES (@dataLoadRunID,@eventType,@source,@description,@time);", con)) { con.Open(); - cmdRecordProgress.Parameters.Add("@dataLoadRunID", SqlDbType.Int); - cmdRecordProgress.Parameters.Add("@eventType", SqlDbType.VarChar, 50); - cmdRecordProgress.Parameters.Add("@source", SqlDbType.VarChar, 100); - cmdRecordProgress.Parameters.Add("@description", SqlDbType.VarChar, 8000); - cmdRecordProgress.Parameters.Add("@time", SqlDbType.DateTime); - - cmdRecordProgress.Parameters["@dataLoadRunID"].Value = ID; - cmdRecordProgress.Parameters["@eventType"].Value = pevent.ToString(); - cmdRecordProgress.Parameters["@source"].Value = Source; - cmdRecordProgress.Parameters["@description"].Value = Description; - cmdRecordProgress.Parameters["@time"].Value = DateTime.Now; + _server.AddParameterWithValueToCommand("@dataLoadRunID",cmdRecordProgress, ID); + _server.AddParameterWithValueToCommand("@eventType", cmdRecordProgress, pevent.ToString()); + _server.AddParameterWithValueToCommand("@source", cmdRecordProgress, Source); + _server.AddParameterWithValueToCommand("@description", cmdRecordProgress, Description); + _server.AddParameterWithValueToCommand("@time", cmdRecordProgress, DateTime.Now); cmdRecordProgress.ExecuteNonQuery(); } diff --git a/Rdmp.Core/Logging/LogManager.cs b/Rdmp.Core/Logging/LogManager.cs index 9ee1bba3c4..13fa6859c5 100644 --- a/Rdmp.Core/Logging/LogManager.cs +++ b/Rdmp.Core/Logging/LogManager.cs @@ -9,9 +9,11 @@ using System.Data; using System.Data.Common; using System.Linq; +using System.Text; using System.Threading; using System.Threading.Tasks; using FAnsi.Discovery; +using FAnsi.Discovery.QuerySyntax; using Rdmp.Core.Logging.PastEvents; using ReusableLibraryCode; using ReusableLibraryCode.DataAccess; @@ -153,13 +155,9 @@ public IEnumerable GetArchivalDataLoadInfos(string dataTas var dataTaskId = GetDataTaskId(dataTask,Server, con); string where = ""; - string top = ""; using (var cmd = Server.GetCommand("", con)) { - if (topX != null) - top = "TOP " + topX.Value; - if (specificDataLoadRunIDOnly != null) where = "WHERE ID=" + specificDataLoadRunIDOnly.Value; else @@ -171,9 +169,32 @@ public IEnumerable GetArchivalDataLoadInfos(string dataTas cmd.Parameters.Add(p); } - string sql = "SELECT " + top + " *, (select top 1 1 from FatalError where dataLoadRunID = DataLoadRun.ID) hasErrors FROM " + run.GetFullyQualifiedName() +" " + where + " ORDER BY ID desc"; + TopXResponse top = null; + + if (topX.HasValue) + top = Server.GetQuerySyntaxHelper().HowDoWeAchieveTopX(topX.Value); + + StringBuilder sb = new StringBuilder(); + + + sb.Append("SELECT "); + + if(top?.Location == QueryComponent.SELECT) + { + sb.AppendLine(top.SQL); + } + + sb.Append(" *"); + + + sb.AppendLine($" FROM {run.GetFullyQualifiedName()} {where} ORDER BY ID desc"); + + if(top?.Location == QueryComponent.Postfix) + { + sb.AppendLine(top.SQL); + } - cmd.CommandText = sql; + cmd.CommandText = sb.ToString(); DbDataReader r; if (token == null) @@ -246,10 +267,11 @@ public void CreateNewLoggingTask(int id, string dataSetID) var sql = "INSERT INTO DataLoadTask (ID, description, name, createTime, userAccount, statusID, isTest, dataSetID) " + "VALUES " + - "(" + id + ", @dataSetID, @dataSetID, GetDate(), @username, 1, 0, @dataSetID)"; + "(" + id + ", @dataSetID, @dataSetID, @date, @username, 1, 0, @dataSetID)"; using (var cmd = Server.GetCommand(sql, conn)) { + Server.AddParameterWithValueToCommand("@date", cmd,DateTime.Now); Server.AddParameterWithValueToCommand("@dataSetID",cmd,dataSetID); Server.AddParameterWithValueToCommand("@username",cmd,Environment.UserName); @@ -318,7 +340,7 @@ public void ResolveFatalErrors(int[] ids, DataLoadInfo.FatalErrorStates newState conn.Open(); { var sql = - "UPDATE [FatalError] SET explanation =@explanation, statusID=@statusID where ID in (" + string.Join(",", ids) + ")"; + "UPDATE FatalError SET explanation =@explanation, statusID=@statusID where ID in (" + string.Join(",", ids) + ")"; int affectedRows; diff --git a/Rdmp.Core/Logging/PastEvents/ArchivalDataLoadInfo.cs b/Rdmp.Core/Logging/PastEvents/ArchivalDataLoadInfo.cs index 7f527568f5..f484a16bfc 100644 --- a/Rdmp.Core/Logging/PastEvents/ArchivalDataLoadInfo.cs +++ b/Rdmp.Core/Logging/PastEvents/ArchivalDataLoadInfo.cs @@ -7,6 +7,7 @@ using System; using System.Collections.Generic; using System.Data.Common; +using System.Linq; using FAnsi.Discovery; using ReusableLibraryCode.Settings; @@ -28,7 +29,7 @@ public class ArchivalDataLoadInfo : IArchivalLoggingRecordOfPastEvent, IComparab public DateTime StartTime { get; internal set; } public DateTime? EndTime { get; internal set; } - public bool HasErrors { get; private set; } + public bool HasErrors => _knownErrors.Value.Any(); public string ToShortString() { @@ -94,8 +95,6 @@ internal ArchivalDataLoadInfo(DbDataReader r,DiscoveredDatabase loggingDatabase) Description = r["description"] as string; - HasErrors = r["hasErrors"] != DBNull.Value; - _knownTableInfos = new Lazy>(GetTableInfos); _knownErrors = new Lazy>(GetErrors); _knownProgress = new Lazy>(GetProgress); diff --git a/Rdmp.Core/Logging/TableLoadInfo.cs b/Rdmp.Core/Logging/TableLoadInfo.cs index 4490d7ba3f..82d00e3c25 100644 --- a/Rdmp.Core/Logging/TableLoadInfo.cs +++ b/Rdmp.Core/Logging/TableLoadInfo.cs @@ -6,7 +6,7 @@ using System; using System.Data; -using Microsoft.Data.SqlClient; +using FAnsi.Connections; using FAnsi.Discovery; namespace Rdmp.Core.Logging @@ -78,23 +78,18 @@ public TableLoadInfo(DataLoadInfo parent,string suggestedRollbackCommand,string private void RecordNewTableLoadInDatabase(DataLoadInfo parent,string destinationTable, DataSource[] sources, int expectedInserts) { - using (var con = (SqlConnection)_databaseSettings.GetConnection()) - using (var cmd = new SqlCommand("INSERT INTO TableLoadRun (startTime,dataLoadRunID,targetTable,expectedInserts,suggestedRollbackCommand) " + + using (var con = _databaseSettings.GetConnection()) + using (var cmd = _databaseSettings.GetCommand("INSERT INTO TableLoadRun (startTime,dataLoadRunID,targetTable,expectedInserts,suggestedRollbackCommand) " + "VALUES (@startTime,@dataLoadRunID,@targetTable,@expectedInserts,@suggestedRollbackCommand); " + - "SELECT SCOPE_IDENTITY();", con)) + "SELECT @@IDENTITY;", con)) { con.Open(); - cmd.Parameters.Add("@startTime", SqlDbType.DateTime); - cmd.Parameters.Add("@dataLoadRunID", SqlDbType.Int); - cmd.Parameters.Add("@targetTable", SqlDbType.VarChar, 200); - cmd.Parameters.Add("@expectedInserts", SqlDbType.BigInt); - cmd.Parameters.Add("@suggestedRollbackCommand", SqlDbType.VarChar, -1); - - cmd.Parameters["@startTime"].Value = DateTime.Now; - cmd.Parameters["@dataLoadRunID"].Value = parent.ID; - cmd.Parameters["@targetTable"].Value = destinationTable; - cmd.Parameters["@expectedInserts"].Value = expectedInserts; - cmd.Parameters["@suggestedRollbackCommand"].Value = _suggestedRollbackCommand; + + _databaseSettings.AddParameterWithValueToCommand("@startTime",cmd, DateTime.Now); + _databaseSettings.AddParameterWithValueToCommand("@dataLoadRunID", cmd, parent.ID); + _databaseSettings.AddParameterWithValueToCommand("@targetTable", cmd, destinationTable); + _databaseSettings.AddParameterWithValueToCommand("@expectedInserts", cmd, expectedInserts); + _databaseSettings.AddParameterWithValueToCommand("@suggestedRollbackCommand", cmd, _suggestedRollbackCommand); //get the ID, can come back as a decimal or an Int32 or an Int64 so whatever, just turn it into a string and then parse it _id = int.Parse(cmd.ExecuteScalar().ToString()); @@ -105,27 +100,14 @@ private void RecordNewTableLoadInDatabase(DataLoadInfo parent,string destination //for each of the sources, create them in the DataSource table foreach (DataSource s in DataSources) { - using (var cmdInsertDs = new SqlCommand("INSERT INTO DataSource (source,tableLoadRunID,originDate,MD5) " + - "VALUES (@source,@tableLoadRunID,@originDate,@MD5); SELECT SCOPE_IDENTITY();", con)) + using (var cmdInsertDs = _databaseSettings.GetCommand("INSERT INTO DataSource (source,tableLoadRunID,originDate,MD5) " + + "VALUES (@source,@tableLoadRunID,@originDate,@MD5); SELECT @@IDENTITY;", con)) { - cmdInsertDs.Parameters.Add("@source", SqlDbType.VarChar, -1); - cmdInsertDs.Parameters.Add("@tableLoadRunID", SqlDbType.Int); - cmdInsertDs.Parameters.Add("@originDate", SqlDbType.Date); - cmdInsertDs.Parameters.Add("@MD5", SqlDbType.Binary, 128); - - - cmdInsertDs.Parameters["@source"].Value = s.Source; - cmdInsertDs.Parameters["@tableLoadRunID"].Value = _id; - - if (s.UnknownOriginDate) - cmdInsertDs.Parameters["@originDate"].Value = DBNull.Value; - else - cmdInsertDs.Parameters["@originDate"].Value = s.OriginDate; - if (s.MD5 != null) - cmdInsertDs.Parameters["@MD5"].Value = s.MD5; - else - cmdInsertDs.Parameters["@MD5"].Value = DBNull.Value; + _databaseSettings.AddParameterWithValueToCommand("@source", cmdInsertDs, s.Source); + _databaseSettings.AddParameterWithValueToCommand("@tableLoadRunID", cmdInsertDs, _id); + _databaseSettings.AddParameterWithValueToCommand("@originDate", cmdInsertDs, s.UnknownOriginDate ? DBNull.Value : s.OriginDate); + _databaseSettings.AddParameterWithValueToCommand("@MD5", cmdInsertDs, s.MD5 != null ? s.MD5:DBNull.Value); s.ID = int.Parse(cmdInsertDs.ExecuteScalar().ToString()); } @@ -220,36 +202,20 @@ public string Notes public void CloseAndArchive() { - using (var con = (SqlConnection)_databaseSettings.GetConnection()) + using (var con = _databaseSettings.BeginNewTransactedConnection()) { - con.Open(); - using (SqlTransaction transaction = con.BeginTransaction()) - using (var cmdCloseRecord = new SqlCommand("UPDATE TableLoadRun SET endTime=@endTime,inserts=@inserts,updates=@updates,deletes=@deletes,errorRows=@errorRows,duplicates=@duplicates, notes=@notes WHERE ID=@ID", con, transaction)) + using (var cmdCloseRecord = _databaseSettings.GetCommand("UPDATE TableLoadRun SET endTime=@endTime,inserts=@inserts,updates=@updates,deletes=@deletes,errorRows=@errorRows,duplicates=@duplicates, notes=@notes WHERE ID=@ID", con.Connection, con.ManagedTransaction)) { try { - cmdCloseRecord.Parameters.Add("@endTime", SqlDbType.DateTime); - cmdCloseRecord.Parameters.Add("@inserts", SqlDbType.BigInt); - cmdCloseRecord.Parameters.Add("@updates", SqlDbType.BigInt); - cmdCloseRecord.Parameters.Add("@deletes", SqlDbType.BigInt); - cmdCloseRecord.Parameters.Add("@errorRows", SqlDbType.BigInt); - cmdCloseRecord.Parameters.Add("@duplicates", SqlDbType.BigInt); - cmdCloseRecord.Parameters.Add("@notes", SqlDbType.VarChar); - cmdCloseRecord.Parameters.Add("@ID", SqlDbType.Int); - - cmdCloseRecord.Parameters["@endTime"].Value = DateTime.Now; - cmdCloseRecord.Parameters["@inserts"].Value = this.Inserts; - cmdCloseRecord.Parameters["@updates"].Value = this.Updates; - cmdCloseRecord.Parameters["@deletes"].Value = this.Deletes; - cmdCloseRecord.Parameters["@errorRows"].Value = this.ErrorRows; - cmdCloseRecord.Parameters["@duplicates"].Value = this.DiscardedDuplicates; - - if (string.IsNullOrWhiteSpace(this.Notes)) - cmdCloseRecord.Parameters["@notes"].Value = DBNull.Value; - else - cmdCloseRecord.Parameters["@notes"].Value = this.Notes; - - cmdCloseRecord.Parameters["@ID"].Value = this.ID; + _databaseSettings.AddParameterWithValueToCommand("@endTime",cmdCloseRecord, DateTime.Now); + _databaseSettings.AddParameterWithValueToCommand("@inserts", cmdCloseRecord, this.Inserts); + _databaseSettings.AddParameterWithValueToCommand("@updates", cmdCloseRecord, this.Updates); + _databaseSettings.AddParameterWithValueToCommand("@deletes", cmdCloseRecord, this.Deletes); + _databaseSettings.AddParameterWithValueToCommand("@errorRows", cmdCloseRecord, this.ErrorRows); + _databaseSettings.AddParameterWithValueToCommand("@duplicates", cmdCloseRecord, this.DiscardedDuplicates); + _databaseSettings.AddParameterWithValueToCommand("@notes", cmdCloseRecord, string.IsNullOrWhiteSpace(this.Notes) ? DBNull.Value : this.Notes); + _databaseSettings.AddParameterWithValueToCommand("@ID", cmdCloseRecord, this.ID); int affectedRows = cmdCloseRecord.ExecuteNonQuery(); @@ -257,9 +223,9 @@ public void CloseAndArchive() throw new Exception("Error closing TableLoadInfo in database, the UPDATE command affected " + affectedRows + " when we expected 1 (will attempt to rollback transaction)"); foreach (DataSource s in DataSources) - MarkDataSourceAsArchived(s, con, transaction); + MarkDataSourceAsArchived(s, con); - transaction.Commit(); + con.ManagedTransaction.CommitAndCloseConnection(); _endTime = DateTime.Now; _isClosed = true; @@ -267,27 +233,24 @@ public void CloseAndArchive() catch (Exception) { //if something goes wrong with the update, roll it back - transaction.Rollback(); + con.ManagedTransaction.AbandonAndCloseConnection(); throw; } } } } - private void MarkDataSourceAsArchived(DataSource ds, SqlConnection con, SqlTransaction transaction) + private void MarkDataSourceAsArchived(DataSource ds, IManagedConnection con) { if (string.IsNullOrEmpty(ds.Archive)) return; - using (var cmdSetArchived = new SqlCommand("UPDATE DataSource SET archive=@archive, source = @source WHERE ID=@ID", con, transaction)) + using (var cmdSetArchived = _databaseSettings.GetCommand("UPDATE DataSource SET archive=@archive, source = @source WHERE ID=@ID", con.Connection, con.ManagedTransaction)) { - cmdSetArchived.Parameters.Add("@archive", SqlDbType.VarChar, -1); - cmdSetArchived.Parameters.Add("@source", SqlDbType.VarChar, -1); - cmdSetArchived.Parameters.Add("@ID", SqlDbType.Int); - cmdSetArchived.Parameters["@archive"].Value = ds.Archive; - cmdSetArchived.Parameters["@source"].Value = ds.Source; - cmdSetArchived.Parameters["@ID"].Value = ds.ID; + _databaseSettings.AddParameterWithValueToCommand("@archive", cmdSetArchived,ds.Archive); + _databaseSettings.AddParameterWithValueToCommand("@source", cmdSetArchived, ds.Source); + _databaseSettings.AddParameterWithValueToCommand("@ID", cmdSetArchived, ds.ID); cmdSetArchived.ExecuteNonQuery(); } diff --git a/Rdmp.Core/QueryBuilding/AggregateBuilder.cs b/Rdmp.Core/QueryBuilding/AggregateBuilder.cs index 7a1266fdf9..7f78175136 100644 --- a/Rdmp.Core/QueryBuilding/AggregateBuilder.cs +++ b/Rdmp.Core/QueryBuilding/AggregateBuilder.cs @@ -13,6 +13,7 @@ using Rdmp.Core.Curation.Data.Aggregation; using Rdmp.Core.QueryBuilding.Options; using Rdmp.Core.QueryBuilding.Parameters; +using ReusableLibraryCode.Settings; namespace Rdmp.Core.QueryBuilding { @@ -564,7 +565,7 @@ private string GetGroupOrOrderByCustomLineBasedOn(QueryTimeColumn col) private string GetGroupOrOrderByCustomLineBasedOn(string select, string alias) { - if (QuerySyntaxHelper.DatabaseType == FAnsi.DatabaseType.MySql) + if (UserSettings.UseAliasInsteadOfTransformInGroupByAggregateGraphs) { return !string.IsNullOrWhiteSpace(alias) ? alias : // for MySql prefer using the alias if it has one diff --git a/Rdmp.Core/Rdmp.Core.csproj b/Rdmp.Core/Rdmp.Core.csproj index 0cc9a1c817..2c7047ccd9 100644 --- a/Rdmp.Core/Rdmp.Core.csproj +++ b/Rdmp.Core/Rdmp.Core.csproj @@ -267,7 +267,7 @@ - + diff --git a/Rdmp.Core/Repositories/RowVerCache.cs b/Rdmp.Core/Repositories/RowVerCache.cs index 7a578b725e..b8383d2ee4 100644 --- a/Rdmp.Core/Repositories/RowVerCache.cs +++ b/Rdmp.Core/Repositories/RowVerCache.cs @@ -6,11 +6,11 @@ using System; using System.Collections.Generic; -using Microsoft.Data.SqlClient; using System.Linq; using System.Text; using System.Threading; using MapsDirectlyToDatabaseTable; +using Microsoft.Data.SqlClient; namespace Rdmp.Core.Repositories { diff --git a/Rdmp.Core/Repositories/YamlRepository.cs b/Rdmp.Core/Repositories/YamlRepository.cs index 8150464eb2..6cedf60d3f 100644 --- a/Rdmp.Core/Repositories/YamlRepository.cs +++ b/Rdmp.Core/Repositories/YamlRepository.cs @@ -81,7 +81,7 @@ private void LoadObjects() var deserializer = builder.Build(); - foreach (var t in GetCompatibleTypes()) + foreach (var t in GetCompatibleTypes().OrderBy(ObjectDependencyOrder)) { // find the directory that contains all the YAML files e.g. MyDir/Catalogue/ var typeDir = subdirs.FirstOrDefault(d => d.Name.Equals(t.Name)); @@ -120,6 +120,18 @@ private void LoadObjects() LoadWhereSubContainers(); } + private int ObjectDependencyOrder(Type arg) + { + // Load Plugin objects before dependent children + if (arg == typeof(Rdmp.Core.Curation.Data.Plugin)) + return 1; + + if (arg == typeof(LoadModuleAssembly)) + return 2; + + return 3; + } + /// /// Sets on . @@ -138,6 +150,12 @@ protected virtual void SetRepositoryOnObject(IMapsDirectlyToDatabaseTable obj) case ConcreteContainer container: container.SetManager(this); break; + case LoadModuleAssembly lma: + lock(lockFs) + { + lma.Bin = File.ReadAllBytes(GetNupkgPath(lma)); + break; + } } } @@ -150,7 +168,15 @@ public override void InsertAndHydrate(T toCreate, Dictionary { SaveToDatabase(toCreate); } - + } + + private string GetNupkgPath(LoadModuleAssembly lma) + { + //somedir/LoadModuleAssembly/ + var path = Path.GetDirectoryName(GetPath(lma)); + + //somedir/LoadModuleAssembly/MyPlugin1.0.0.nupkg + return Path.Combine(path, GetObjectByID(lma.Plugin_ID).Name); } public override void DeleteFromDatabase(IMapsDirectlyToDatabaseTable oTableWrapperObject) @@ -159,6 +185,12 @@ public override void DeleteFromDatabase(IMapsDirectlyToDatabaseTable oTableWrapp { base.DeleteFromDatabase(oTableWrapperObject); File.Delete(GetPath(oTableWrapperObject)); + + // if deleting a LoadModuleAssembly also delete its binary content file (the plugin dlls in nupkg) + if (oTableWrapperObject is LoadModuleAssembly lma) + { + File.Delete(GetNupkgPath(lma)); + } } } @@ -171,6 +203,14 @@ public override void SaveToDatabase(IMapsDirectlyToDatabaseTable o) lock (lockFs) { File.WriteAllText(GetPath(o), yaml); + + // Do not write plugin binary content into yaml that results in + // a massive blob of binary yaml (not useful and slow to load) + if (o is LoadModuleAssembly lma) + { + // write the nupkg as a binary file instead to the same folder + File.WriteAllBytes(GetNupkgPath(lma), lma.Bin); + } } } diff --git a/Rdmp.UI.Tests/DesignPatternTests/ClassFileEvaluation/DocumentationCrossExaminationTest.cs b/Rdmp.UI.Tests/DesignPatternTests/ClassFileEvaluation/DocumentationCrossExaminationTest.cs index 6f3289db68..bfea41917d 100644 --- a/Rdmp.UI.Tests/DesignPatternTests/ClassFileEvaluation/DocumentationCrossExaminationTest.cs +++ b/Rdmp.UI.Tests/DesignPatternTests/ClassFileEvaluation/DocumentationCrossExaminationTest.cs @@ -261,7 +261,8 @@ class DocumentationCrossExaminationTest "NuGet", "MyPluginClass", "SubContainer", - "DescribeCommand" // this class has now been removed from RDMP codebase, don't complain if you see it in docs (e.g. CHANGELOG.md). + "DescribeCommand", // this class has now been removed from RDMP codebase, don't complain if you see it in docs (e.g. CHANGELOG.md). + "GetDate" }; #endregion public DocumentationCrossExaminationTest(DirectoryInfo slndir) diff --git a/Rdmp.UI/ProjectUI/ExecuteExtractionUI.cs b/Rdmp.UI/ProjectUI/ExecuteExtractionUI.cs index d9f0277ecb..59a9383740 100644 --- a/Rdmp.UI/ProjectUI/ExecuteExtractionUI.cs +++ b/Rdmp.UI/ProjectUI/ExecuteExtractionUI.cs @@ -286,11 +286,7 @@ public override void SetDatabaseObject(IActivateItems activator, ExtractionConfi //if there are no project specific datasets if (_datasets.All(sds => sds.ExtractableDataSet.Project_ID == null)) tlvDatasets.DisableObject(_projectSpecificDatasetsFolder); //disable this option - - //if all the datasets are project specific - if (_datasets.All(sds => sds.ExtractableDataSet.Project_ID != null)) - tlvDatasets.DisableObject(_coreDatasetsFolder); - + //don't accept refresh while executing if (checkAndExecuteUI1.IsExecuting) return; diff --git a/Reusable/MapsDirectlyToDatabaseTable/TableRepository.cs b/Reusable/MapsDirectlyToDatabaseTable/TableRepository.cs index 88db416c99..65bdff3b7d 100644 --- a/Reusable/MapsDirectlyToDatabaseTable/TableRepository.cs +++ b/Reusable/MapsDirectlyToDatabaseTable/TableRepository.cs @@ -640,7 +640,7 @@ public void TestConnection() throw new InvalidOperationException( "Invalid parameters for " + typeof(T).Name + " INSERT. Do not use @ when specifying parameter names, this is SQL-specific and will be added when required: " + string.Join(", ", parameters.Where(kvp => kvp.Key.StartsWith("@")))); - var columnString = string.Join(", ", parameters.Select(kvp => "[" + kvp.Key + "]")); + var columnString = string.Join(", ", parameters.Select(kvp => Wrap(kvp.Key))); var parameterString = string.Join(", ", parameters.Select(kvp => "@" + kvp.Key)); query += "(" + columnString + ") VALUES (" + parameterString + ")"; } @@ -961,5 +961,10 @@ public void EndTransaction(bool commit) { EndTransactedConnection(commit); } + + public string Wrap(string name) + { + return DiscoveredServer.GetQuerySyntaxHelper().EnsureWrapped(name); + } } } diff --git a/Reusable/MapsDirectlyToDatabaseTable/Versioning/MasterDatabaseScriptExecutor.cs b/Reusable/MapsDirectlyToDatabaseTable/Versioning/MasterDatabaseScriptExecutor.cs index 4b32f35171..bea803e3ad 100644 --- a/Reusable/MapsDirectlyToDatabaseTable/Versioning/MasterDatabaseScriptExecutor.cs +++ b/Reusable/MapsDirectlyToDatabaseTable/Versioning/MasterDatabaseScriptExecutor.cs @@ -426,6 +426,9 @@ public Patch[] GetPatchesRun() public void CreateAndPatchDatabase(IPatcher patcher, ICheckNotifier notifier) { var initialPatch = patcher.GetInitialCreateScriptContents(Database); + notifier.OnCheckPerformed( + new CheckEventArgs($"About to run:{Environment.NewLine}{initialPatch.EntireScript}", CheckResult.Success)); + CreateDatabase(initialPatch, notifier); //get everything in the /up/ folder that is .sql diff --git a/Reusable/MapsDirectlyToDatabaseTable/Versioning/Patch.cs b/Reusable/MapsDirectlyToDatabaseTable/Versioning/Patch.cs index 4e3c0cc307..96fbf86de9 100644 --- a/Reusable/MapsDirectlyToDatabaseTable/Versioning/Patch.cs +++ b/Reusable/MapsDirectlyToDatabaseTable/Versioning/Patch.cs @@ -56,10 +56,12 @@ private void ExtractDescriptionAndVersionFromScriptContents() { var lines = EntireScript.Split(new []{'\r', '\n'},StringSplitOptions.RemoveEmptyEntries); - if(!lines[0].StartsWith(VersionKey)) + var idx = lines[0].IndexOf(VersionKey); + + if (idx == -1) throw new InvalidPatchException(locationInAssembly,"Script does not start with " + VersionKey); - string versionNumber = lines[0].Substring(VersionKey.Length).Trim(':',' ','\n','\r'); + string versionNumber = lines[0].Substring(idx + VersionKey.Length).Trim(':',' ','\n','\r','/','*'); try { @@ -72,10 +74,12 @@ private void ExtractDescriptionAndVersionFromScriptContents() if(lines.Length >=2) { - if(!lines[1].StartsWith(DescriptionKey)) - throw new InvalidPatchException(locationInAssembly,"Second line of patch scripts must start with " + DescriptionKey); + idx = lines[1].IndexOf(DescriptionKey); + + if (idx == -1 ) + throw new InvalidPatchException(locationInAssembly,"Second line of patch scripts must start with " + DescriptionKey); - string description = lines[1].Substring(DescriptionKey.Length); + string description = lines[1].Substring(idx + DescriptionKey.Length).Trim(':', ' ', '\n', '\r', '/', '*'); Description = description; } } diff --git a/Reusable/MapsDirectlyToDatabaseTable/Versioning/Patcher.cs b/Reusable/MapsDirectlyToDatabaseTable/Versioning/Patcher.cs index c48f447df8..f29c895635 100644 --- a/Reusable/MapsDirectlyToDatabaseTable/Versioning/Patcher.cs +++ b/Reusable/MapsDirectlyToDatabaseTable/Versioning/Patcher.cs @@ -43,20 +43,31 @@ protected Patcher(int tier,string resourceSubdirectory) Tier = tier; ResourceSubdirectory = resourceSubdirectory; } - + /// /// Generates a properly formatted header for creation when you only know the SQL you want to execute /// + /// /// /// /// - protected string GetHeader(string description,Version version) + protected string GetHeader(DatabaseType dbType, string description,Version version) { - string header = Patch.VersionKey + version.ToString() + Environment.NewLine; - header += Patch.DescriptionKey + description + Environment.NewLine; + string header = CommentFor(dbType,Patch.VersionKey + version.ToString()) + Environment.NewLine; + header += CommentFor(dbType,Patch.DescriptionKey + description) + Environment.NewLine; return header; } + + private string CommentFor(DatabaseType dbType, string sql) + { + if (dbType == DatabaseType.MicrosoftSQLServer) + return sql; + + // some DBMS don't like the -- notation so we need to wrap with C style comments + return $"/*{sql}*/"; + } + public virtual Patch GetInitialCreateScriptContents(DiscoveredDatabase db) { var assembly = GetDbAssembly(); @@ -76,8 +87,8 @@ public virtual Patch GetInitialCreateScriptContents(DiscoveredDatabase db) string sql = sr.ReadToEnd(); - if (!sql.StartsWith(Patch.VersionKey)) - sql = GetHeader(InitialScriptName, new Version(1, 0, 0)) + sql; + if (!sql.Contains(Patch.VersionKey)) + sql = GetHeader(db.Server.DatabaseType, InitialScriptName, new Version(1, 0, 0)) + sql; return new Patch(InitialScriptName, sql); diff --git a/Reusable/ReusableLibraryCode/Checks/AcceptAllCheckNotifier.cs b/Reusable/ReusableLibraryCode/Checks/AcceptAllCheckNotifier.cs index 4a8e71d623..533978baf4 100644 --- a/Reusable/ReusableLibraryCode/Checks/AcceptAllCheckNotifier.cs +++ b/Reusable/ReusableLibraryCode/Checks/AcceptAllCheckNotifier.cs @@ -13,6 +13,11 @@ namespace ReusableLibraryCode.Checks /// public class AcceptAllCheckNotifier : ICheckNotifier { + /// + /// True to write out all messages seen directly to the console + /// + public bool WriteToConsole { get; set; } + /// /// Check handler that throws on Failures but otherwise returns true /// @@ -20,6 +25,9 @@ public class AcceptAllCheckNotifier : ICheckNotifier /// public virtual bool OnCheckPerformed(CheckEventArgs args) { + if (WriteToConsole) + Console.WriteLine($"{args.Result}:{args.Message}"); + //if there is a proposed fix then accept it regardless of whether it was a Fail. if (!string.IsNullOrWhiteSpace(args.ProposedFix)) return true; diff --git a/Reusable/ReusableLibraryCode/ReusableLibraryCode.csproj b/Reusable/ReusableLibraryCode/ReusableLibraryCode.csproj index c96291d737..a417a65ef5 100644 --- a/Reusable/ReusableLibraryCode/ReusableLibraryCode.csproj +++ b/Reusable/ReusableLibraryCode/ReusableLibraryCode.csproj @@ -57,7 +57,7 @@ - + diff --git a/Reusable/ReusableLibraryCode/Settings/UserSettings.cs b/Reusable/ReusableLibraryCode/Settings/UserSettings.cs index 844e0529b1..aa3a9df2c2 100644 --- a/Reusable/ReusableLibraryCode/Settings/UserSettings.cs +++ b/Reusable/ReusableLibraryCode/Settings/UserSettings.cs @@ -397,6 +397,20 @@ public static bool ShowProjectSpecificColumns set { AppSettings.AddOrUpdateValue("ShowProjectSpecificColumns", value); } } + /// + /// When generating an aggregate graph, use the column alias instead of the select sql. For example + /// when you have the select column 'SELECT YEAR(dt) as myYear' then the GROUP BY will default to + /// 'GROUP BY YEAR(dt)'. Setting this property to true will instead use 'GROUP BY myYear'. Typically + /// this only works in MySql but it is not universally supported by all MySql versions and server settings + /// + /// Defaults to false. + /// + public static bool UseAliasInsteadOfTransformInGroupByAggregateGraphs + { + get { return AppSettings.GetValueOrDefault("ShowProjectSpecificColumns", false); } + set { AppSettings.AddOrUpdateValue("ShowProjectSpecificColumns", value); } + } + #endregion diff --git a/Tools/rdmp/CommandLine/Gui/ConsoleGuiActivator.cs b/Tools/rdmp/CommandLine/Gui/ConsoleGuiActivator.cs index aa3d63e6be..0e081e3b3e 100644 --- a/Tools/rdmp/CommandLine/Gui/ConsoleGuiActivator.cs +++ b/Tools/rdmp/CommandLine/Gui/ConsoleGuiActivator.cs @@ -235,7 +235,7 @@ public override FileInfo SelectFile(string prompt) public override FileInfo SelectFile(string prompt, string patternDescription, string pattern) { - var openDir = new OpenDialog(prompt,"Directory") + var openDir = new OpenDialog(prompt,"File") { AllowsMultipleSelection = false, AllowedFileTypes = pattern == null ? null : new []{pattern.TrimStart('*')} diff --git a/Tools/rdmp/Program.cs b/Tools/rdmp/Program.cs index b94cbf90b0..2bffbd1838 100644 --- a/Tools/rdmp/Program.cs +++ b/Tools/rdmp/Program.cs @@ -31,6 +31,12 @@ namespace Rdmp.Core { class Program { + /// + /// True if the user passed the -q switch at startup to suppress any helpful messages we might + /// show (e.g. maybe they want to pipe the results somewhere) + /// + public static bool Quiet { get; private set; } + static int Main(string[] args) { try @@ -51,6 +57,7 @@ static int Main(string[] args) if(args.Any(a=>a.Equals("-q")) || args.Any(a=>a.Equals("--quiet",StringComparison.CurrentCultureIgnoreCase))) { + Quiet = true; foreach(var t in LogManager.Configuration.AllTargets.ToArray()) { if(t.GetType().Name.Contains("Console",StringComparison.CurrentCultureIgnoreCase))