Merge branch 'develop' into hotfix/mysql-ci

HicServices · Aug 30, 2022 · 26534e9 · 26534e9
2 parents 3a7445a + f366ba8
commit 26534e9
Show file tree

Hide file tree

Showing 41 changed files with 476 additions and 328 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -59,6 +59,12 @@ jobs:
           mysql-version: '8.0'
           root-password: 'YourStrong!Passw0rd'
           auto-start: true
+      - name: Create MySql Logging Db
+        run: dotnet run -c Release --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver LiveLoggingServer_ID "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_logging2" --dir ~/rdmp/rdmp-yaml/
+      - name: Create MySql DQE Db
+        run: dotnet run -c Release --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver DQE "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_dqe" --dir ~/rdmp/rdmp-yaml/
+      - name: Create MySql Cohort Building Query Caching Db
+        run: dotnet run -c Release --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver CohortIdentificationQueryCachingServer_ID "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_cache" --dir ~/rdmp/rdmp-yaml/
       - name: Build
         run: dotnet build --configuration Release --verbosity minimal
       - name: Initialise RDMP

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,10 +8,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ...
 
-### Changed
-- Removed restriction preventing [Lookup] requiring all foreign key columns being from the same table [#1331](https://github.com/HicServices/RDMP/issues/1307)
-- If there are multiple IsPrimaryExtractionTable involved in a query then the one with the IsExtractionIdentifier column (if any) will be picked (previously QueryBuildingException was thrown) [#1365](https://github.com/HicServices/RDMP/issues/1365)
-
 ### Added
 - Added 'Set Description' command to [AggregateConfiguration] context menu
 - Template cohort builder aggregates can be dragged onto extraction datasets to import the container tree [#1307](https://github.com/HicServices/RDMP/issues/1307)
@@ -20,13 +16,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added ability to pop out tooltips/problems into modal popup [#1334](https://github.com/HicServices/RDMP/issues/1334)
 
 ### Changed
-
+- The 'Core' folder in extraction execution user interface is no longer disabled when empty [#1377](https://github.com/HicServices/RDMP/issues/1377)
 - Datasets in extraction UI are no longer expanded by default (i.e. to show Supporting Documents/Sql) [#1264](https://github.com/HicServices/RDMP/issues/1264)
+- Removed restriction preventing [Lookup] requiring all foreign key columns being from the same table [#1331](https://github.com/HicServices/RDMP/issues/1307)
+- If there are multiple IsPrimaryExtractionTable involved in a query then the one with the IsExtractionIdentifier column (if any) will be picked (previously QueryBuildingException was thrown) [#1365](https://github.com/HicServices/RDMP/issues/1365)
 
 ### Fixed
 
 - Running RDMP cli without supplying repository connection details (and after deleting `Databases.yaml`) now results in a specific error message instead of null reference [#1346]https://github.com/HicServices/RDMP/issues/1346
 - Fixed Pipeline components who run in threaded but call UI methods resulting in unstable UI components [#1357](https://github.com/HicServices/RDMP/issues/1357)
+- YamlRepository now saves LoadModuleAssembly binary content as a `.nupkg` file instead of string yaml [#1351](https://github.com/HicServices/RDMP/issues/1351)
+- Fixed Console Gui activator 'Select File' dialog having a confusing title of "Directory" [#1282](https://github.com/HicServices/RDMP/issues/1282)
 
 ## [7.0.17] - 2022-08-01
 

diff --git a/Documentation/CodeTutorials/Packages.md b/Documentation/CodeTutorials/Packages.md
@@ -21,7 +21,7 @@
 | NPOI | [GitHub](https://github.com/tonyqus/npoi) | [2.5.5](https://www.nuget.org/packages/NPOI/2.5.5) | Apache 2.0 | Enables reading/writing Microsoft Excel files |
 | ExcelNumberFormat | [GitHub](https://github.com/andersnm/ExcelNumberFormat) | [1.1.0](https://www.nuget.org/packages/ExcelNumberFormat/1.1.0) |[MIT](https://opensource.org/licenses/MIT)  | Handles translating number formats from Excel formats into usable values | |
 | [NLog](https://nlog-project.org/) | [GitHub](https://github.com/NLog/NLog) | [5.0.2](https://www.nuget.org/packages/NLog/5.0.2) | [BSD 3-Clause](https://github.com/NLog/NLog/blob/dev/LICENSE.txt) | Flexible user configurable logging | |
-| HIC.FAnsiSql |[GitHub](https://github.com/HicServices/FAnsiSql) | [2.0.5](https://www.nuget.org/packages/HIC.FansiSql/2.0.5) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | [DBMS] abstraction layer |
+| HIC.FAnsiSql |[GitHub](https://github.com/HicServices/FAnsiSql) | [3.0.0](https://www.nuget.org/packages/HIC.FansiSql/3.0.0) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | [DBMS] abstraction layer |
 | HIC.BadMedicine | [GitHub](https://github.com/HicServices/BadMedicine) | [1.1.0](https://www.nuget.org/packages/HIC.BadMedicine/1.1.0) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Generate Test Datasets for tests/exericses |
 | SSH.NET  | [GitHub](https://github.com/sshnet/SSH.NET)  | [2020.0.2](https://www.nuget.org/packages/SSH.NET/2020.0.2) | [MIT](https://github.com/sshnet/SSH.NET/blob/develop/LICENSE) | Enables fetching files from SFTP servers |
 | Moq 4 | [GitHub](https://github.com/moq/moq4) | [4.18.2](https://www.nuget.org/packages/Moq/4.18.2) |[BSD 3](https://github.com/moq/moq4/blob/master/License.txt)  | Mock objects during unit testing |
@@ -45,7 +45,7 @@
 | System.Security.Permissions |[GitHub](https://github.com/dotnet/corefx) | [6.0.0](https://www.nuget.org/packages/System.Security.Permissions/6.0.0) |[MIT](https://opensource.org/licenses/MIT) | Provides common types for Xml doc reading in UI code |  |
 | [AutoComplete Console](https://www.codeproject.com/Articles/1182358/Using-Autocomplete-in-Windows-Console-Applications) by Jasper Lammers | Embedded | 4.0 | [CPOL](https://www.codeproject.com/info/cpol10.aspx) | Provides interactive autocomplete in console input | |
 | System.Resources.Extensions | [GitHub](https://github.com/dotnet/corefx) | [4.6.0](https://www.nuget.org/packages/System.Resources.Extensions/4.6.0) | [MIT](https://opensource.org/licenses/MIT) | Allows [publishing with dotnet publish on machines with netcoreapp3.0 SDK installed](https://github.com/microsoft/msbuild/issues/4704#issuecomment-530034240) | |
-| ReadLine | [GitHub](https://github.com/tonerdo/readline) | [2.0.1](https://www.nuget.org/packages/ReadLine/2.0.1) | [MIT](https://opensource.org/licenses/MIT) | Allows autocomplete on command line | |
+| Spectre.Console | [GitHub](https://github.com/spectreconsole/spectre.console) | [0.44.0](https://www.nuget.org/packages/Spectre.Console/0.44.0) | [MIT](https://opensource.org/licenses/MIT) | Allows richer command line interactions| |
 | HIC.System.Windows.Forms.DataVisualization | [GitHub](https://github.com/HicServices/winforms-datavisualization) | [1.0.1](https://www.nuget.org/packages/HIC.System.Windows.Forms.DataVisualization/1.0.1) |[MIT](https://opensource.org/licenses/MIT) | Dotnet core support for DQE charts |  |
 | System.DirectoryServices.Protocols | [GitHub](https://github.com/dotnet/runtime) | [6.0.1](https://www.nuget.org/packages/System.DirectoryServices.Protocols/6.0.1) | MIT | Required dependency of Oracle when using LDAP auth |
 | Autoupdater.NET | [GitHub](https://github.com/ravibpatel/AutoUpdater.NET) | [1.7.0](https://github.com/ravibpatel/AutoUpdater.NET) | MIT | Manages updating of the RDMP windows client directly from the RDMP GitHub Releases|

diff --git a/Plugins/Plugin.Test/Plugin.Test.nuspec b/Plugins/Plugin.Test/Plugin.Test.nuspec
@@ -14,10 +14,11 @@
         <copyright>Copyright 2018-2019</copyright>
         <dependencies>
 			      <dependency id="HIC.BadMedicine" version="1.1.0" />
-            <dependency id="HIC.FAnsiSql" version="2.0.5" />
+            <dependency id="HIC.FAnsiSql" version="3.0.0" />
             <dependency id="CsvHelper" version="28.0.1" />
             <dependency id="Newtonsoft.Json" version="13.0.1"/>
             <dependency id="CommandLineParser" version="2.9.1" />
+			<dependency id="Spectre.Console" version="0.44.0"/>
             <dependency id="SSH.NET" version="2020.0.2" />
             <dependency id="NUnit" version="3.13.3" />
             <dependency id="Moq" version="4.18.2" />
@@ -37,8 +38,7 @@
             <dependency id="System.Globalization" version="4.3.0" />
             <dependency id="System.Net.NameResolution" version="4.3.0" />
             <dependency id="System.Net.Primitives" version="4.3.1" /> 
-            <dependency id="System.Resources.Extensions" version="4.6.0" />
-            <dependency id="ReadLine" version="2.0.1" />
+            <dependency id="System.Resources.Extensions" version="4.6.0" />            
             <dependency id="System.DirectoryServices.Protocols" version="6.0.1" />
         </dependencies>
     </metadata>

diff --git a/Plugins/Plugin.UI/Plugin.UI.nuspec b/Plugins/Plugin.UI/Plugin.UI.nuspec
@@ -14,14 +14,15 @@
         <copyright>Copyright 2018-2019</copyright>
         <dependencies>
 			<dependency id="HIC.BadMedicine" version="1.1.0" />
-            <dependency id="HIC.FAnsiSql" version="2.0.5" />
+            <dependency id="HIC.FAnsiSql" version="3.0.0" />
             <dependency id="DockPanelSuite.ThemeVS2015" version="2.13.0" />
             <dependency id="CsvHelper" version="28.0.1" />
 			<dependency id="NHunspell" version="1.2.5554.16953" />
 			<dependency id="Newtonsoft.Json" version="13.0.1"/>
             <dependency id="fernandreu.ScintillaNET" version="4.2.0" />
             <dependency id="ObjectListView.Official" version="2.9.1" />
             <dependency id="CommandLineParser" version="2.9.1" />
+			<dependency id="Spectre.Console" version="0.44.0"/>
             <dependency id="SSH.NET" version="2020.0.2" />
             <dependency id="NLog" version="5.0.2" />
             <dependency id="ExcelNumberFormat" version="1.1.0" />
@@ -40,8 +41,7 @@
             <dependency id="System.Net.Primitives" version="4.3.1" />
             <dependency id="System.Security.Permissions" version="6.0.0" />
             <dependency id="System.Resources.Extensions" version="4.6.0" />
-            <dependency id="HIC.System.Windows.Forms.DataVisualization" version="1.0.1" />
-            <dependency id="ReadLine" version="2.0.1" />
+            <dependency id="HIC.System.Windows.Forms.DataVisualization" version="1.0.1" />            
             <dependency id="System.DirectoryServices.Protocols" version="6.0.1" />
             <dependency id="YamlDotNet" version="12.0.0" />
         </dependencies>

diff --git a/Plugins/Plugin/Plugin.nuspec b/Plugins/Plugin/Plugin.nuspec
@@ -16,8 +16,9 @@
             <dependency id="CsvHelper" version="28.0.1" />
             <dependency id="Newtonsoft.Json" version="13.0.1"/>
             <dependency id="CommandLineParser" version="2.9.1" />
+			<dependency id="Spectre.Console" version="0.44.0"/>
             <dependency id="SSH.NET" version="2020.0.2" />
-            <dependency id="HIC.FAnsiSql" version="2.0.5" />
+			<dependency id="HIC.FAnsiSql" version="3.0.0" />
             <dependency id="NLog" version="5.0.2" />
             <dependency id="HIC.BadMedicine" version="1.1.0" />
             <dependency id="ExcelNumberFormat" version="1.1.0" />
@@ -33,8 +34,7 @@
             <dependency id="System.Globalization" version="4.3.0" />
             <dependency id="System.Net.NameResolution" version="4.3.0" />
             <dependency id="System.Net.Primitives" version="4.3.1" /> 
-            <dependency id="System.Resources.Extensions" version="4.6.0" />
-            <dependency id="ReadLine" version="2.0.1" />
+            <dependency id="System.Resources.Extensions" version="4.6.0" />            
             <dependency id="System.DirectoryServices.Protocols" version="6.0.1" />
             <dependency id="YamlDotNet" version="12.0.0" />
         </dependencies>

diff --git a/...ration/Integration/QueryBuildingTests/AggregateBuilderTests/MySqlAggregateBuilderTests.cs b/...ration/Integration/QueryBuildingTests/AggregateBuilderTests/MySqlAggregateBuilderTests.cs
@@ -8,6 +8,7 @@
 using NUnit.Framework;
 using Rdmp.Core.Curation.Data;
 using Rdmp.Core.QueryBuilding;
+using ReusableLibraryCode.Settings;
 
 namespace Rdmp.Core.Tests.Curation.Integration.QueryBuildingTests.AggregateBuilderTests
 {
@@ -44,21 +45,25 @@ Col1 desc
 
             topx.DeleteInDatabase();
         }
-        [Test]
-        public void Test_AggregateBuilder_MySql_Top31OrderByCountAsc()
+        [TestCase(true)]
+        [TestCase(false)]
+        public void Test_AggregateBuilder_MySql_Top31OrderByCountAsc(bool useAliasForGroupBy)
         {
             _ti.DatabaseType = DatabaseType.MySql;
             _ti.SaveToDatabase();
 
+            UserSettings.UseAliasInsteadOfTransformInGroupByAggregateGraphs = useAliasForGroupBy;
+
             var builder = new AggregateBuilder(null, "count(*)", null);
             builder.AddColumn(_dimension1);
 
             var topx = new AggregateTopX(CatalogueRepository, _configuration, 31);
             topx.OrderByDirection = AggregateTopXOrderByDirection.Ascending;
             builder.AggregateTopX = topx;
-
 
-            Assert.AreEqual(CollapseWhitespace(@"/**/
+            if (useAliasForGroupBy)
+            {
+                Assert.AreEqual(CollapseWhitespace(@"/**/
 SELECT 
 Col1,
 count(*) AS MyCount
@@ -69,9 +74,27 @@ group by
 order by 
 MyCount asc
 LIMIT 31"), CollapseWhitespace(builder.SQL));
+            }
+            else
+            {
+                Assert.AreEqual(CollapseWhitespace(@"/**/
+SELECT 
+Col1,
+count(*) AS MyCount
+FROM 
+T1
+group by 
+Col1
+order by 
+count(*) asc
+LIMIT 31"), CollapseWhitespace(builder.SQL));
+            }
+
 
 
             topx.DeleteInDatabase();
+
+            UserSettings.UseAliasInsteadOfTransformInGroupByAggregateGraphs = false;
         }
     }
 }
diff --git a/Rdmp.Core.Tests/Curation/YamlRepositoryTests.cs b/Rdmp.Core.Tests/Curation/YamlRepositoryTests.cs
@@ -282,15 +282,17 @@ public void YamlRepository_LoadSavePluginClass()
             var lma2 = UnitTests.WhenIHaveA<LoadModuleAssembly>(repo1);
 
 
-            lma1.Plugin.Name = "MyPlugin";
+            lma1.Plugin.Name = "MyPlugin1.1.1.1.nupkg";
             lma1.Plugin.RdmpVersion = new Version(version); //the version of Rdmp.Core targetted
             lma1.Plugin.PluginVersion = new Version(1, 1, 1, 1); //the version of the plugin
             lma1.Plugin.SaveToDatabase();
+            lma1.SaveToDatabase();
 
-            lma2.Plugin.Name = "MyPlugin";
+            lma2.Plugin.Name = "MyPlugin1.1.1.2.nupkg";
             lma2.Plugin.RdmpVersion = new Version(version);//the version of Rdmp.Core targetted (same as above)
             lma2.Plugin.PluginVersion = new Version(1, 1, 1, 2);//the version of the plugin (higher)
             lma2.Plugin.SaveToDatabase();
+            lma2.SaveToDatabase();
 
             var plugins = repo1.PluginManager.GetCompatiblePlugins();
             Assert.That(plugins, Has.Length.EqualTo(1));

diff --git a/Rdmp.Core.Tests/Logging/DataLoadTaskHelper.cs b/Rdmp.Core.Tests/Logging/DataLoadTaskHelper.cs
@@ -4,6 +4,7 @@
 // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
 // You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
 
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using FAnsi.Discovery;
@@ -41,9 +42,11 @@ public void CreateDataLoadTask(string taskName)
 
                 var taskCmd =
                     _loggingServer.GetCommand(
-                        "INSERT INTO DataLoadTask VALUES (100, '" + taskName + "', '" + taskName + "', GETDATE(), '" + datasetName + "', 1, 1, '" + datasetName + "')",
+                        "INSERT INTO DataLoadTask VALUES (100, '" + taskName + "', '" + taskName + "',@date, '" + datasetName + "', 1, 1, '" + datasetName + "')",
                         con);
 
+                _loggingServer.AddParameterWithValueToCommand("@date", taskCmd, DateTime.Now);
+
                 taskCmd.ExecuteNonQuery();
                 _sqlToCleanUp.Push("DELETE FROM DataLoadTask WHERE dataSetID = '" + datasetName + "'");
             }

diff --git a/Rdmp.Core.Tests/Logging/LogManagerTest.cs b/Rdmp.Core.Tests/Logging/LogManagerTest.cs
@@ -238,6 +238,9 @@ public void LoggingDatabase_TestActuallyCreatingIt(DatabaseType type)
             Assert.AreEqual("bad.cs", archival.Errors.Single().Source);
 
             Assert.AreEqual("Wrote some records", archival.Progress.Single().Description);
+
+            var fatal = archival.Errors.Single();
+            lm.ResolveFatalErrors(new[] { fatal.ID }, DataLoadInfo.FatalErrorStates.Resolved, "problem resolved by building more towers");
         }
     }
 }
diff --git a/Rdmp.Core/CommandExecution/BasicActivateItems.cs b/Rdmp.Core/CommandExecution/BasicActivateItems.cs
@@ -779,7 +779,7 @@ public virtual ExternalDatabaseServer CreateNewPlatformDatabase(ICatalogueReposi
                 throw new ArgumentException($"Database must be picked before calling {nameof(CreateNewPlatformDatabase)} when using {nameof(BasicActivateItems)}",nameof(db));
 
             MasterDatabaseScriptExecutor executor = new MasterDatabaseScriptExecutor(db);
-            executor.CreateAndPatchDatabase(patcher,new AcceptAllCheckNotifier());
+            executor.CreateAndPatchDatabase(patcher, new AcceptAllCheckNotifier() { WriteToConsole = true});
 
             var eds = new ExternalDatabaseServer(catalogueRepository,"New " + (defaultToSet == PermissableDefaults.None ? "" :  defaultToSet.ToString()) + "Server",patcher);
             eds.SetProperties(db);

diff --git a/Rdmp.Core/CommandLine/Interactive/AutoComplete.cs b/Rdmp.Core/CommandLine/Interactive/AutoComplete.cs
@@ -9,7 +9,7 @@
 
 namespace Rdmp.Core.CommandLine.Interactive
 {
-    class AutoComplete : IAutoCompleteHandler
+    class AutoComplete
     {
         private readonly string[] autocompletes;