Merge branch 'release/8.2.0' into task/RDMP-180-rdmp-settings

.github/pull_request_template.md

@@ -0,0 +1,24 @@
+## Proposed Change
+
+Summarise your proposed changes here, including any notes for reviewers.
+
+## Type of change
+
+What types of changes does your code introduce? Tick all that apply.
+
+-   [ ] Bugfix (non-breaking change which fixes an issue)
+-   [ ] New Feature (non-breaking change which adds functionality)
+-   [ ] Breaking Change (fix or feature that would cause existing functionality to not work as expected)
+-   [ ] Documentation-Only Update
+-   [ ] Other  (if none of the other choices apply)
+
+## Checklist
+
+By opening this PR, I confirm that I have:
+
+-   [ ] Ensured that the PR branch is in sync with the target branch (i.e. it is automatically merge-able)
+-   [ ] Created or updated any tests if relevant
+-   [ ] Have validated this change against the [Test Plan](https://github.com/HicServices/RDMP/blob/develop/Documentation/CodeTutorials/TestPlan.md)
+-   [ ] Requested a review by one of the repository maintainers
+-   [ ] Have written new documentation or updated existing documentation to detail any new or updated functionality and how to use it
+-   [ ] Have added an entry into the changelog

CHANGELOG.md

@@ -1,4 +1,6 @@
 
+
+
 # Changelog
 All notable changes to this project will be documented in this file.
 
@@ -11,12 +13,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Add Key-Value store for instance settings
 
-## [8.1.6] - Unreleased
+## [8.1.6] - 2024-05-27
 
 ## Changed
 
-- Add Microsoft.Bcl.AsyncInterfaces 6.0.0 for plugin dependancy tree
-- Add prompt to reanem container when adding a cohort filter
+- Improve error messages for Multi-ExtractionIdentifier extractions
+- Add prompt to rename container when adding a cohort filter
+- Allow for column selection when using the RemoteTableAttacher
+- Fix to remove stack trace button from non error popups
+- Add ability to set Extraction Category as "Not Extractable"
+- Replace BadMedicine v1.2.1 with SynthEHR v2.0.0
+- Fix issue with RDMP being slow to load when having numerous Load Metadatas
+- Fix issue creating logging databases on PostgreSQL servers
 
 ## [8.1.5] - 2024-04-03
 

Directory.Packages.props

@@ -4,8 +4,8 @@
 		<PackageVersion Include="Equ" Version="2.3.0"/>
 		<PackageVersion Include="ExcelNumberFormat" Version="1.1.0"/>
 		<PackageVersion Include="FluentFTP" Version="50.0.1"/>
-		<PackageVersion Include="HIC.BadMedicine" Version="1.2.1"/>
-		<PackageVersion Include="HIC.FAnsiSql" Version="3.2.2"/>
+		<PackageVersion Include="HIC.SynthEHR" Version="2.0.0"/>
+		<PackageVersion Include="HIC.FAnsiSql" Version="3.2.3"/>
 		<PackageVersion Include="LibArchive.Net" Version="0.1.5"/>
 		<PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0">
 			<PrivateAssets>all</PrivateAssets>
@@ -20,9 +20,9 @@
 		<PackageVersion Include="Spectre.Console" Version="0.49.1"/>
 		<PackageVersion Include="SSH.NET" Version="2024.0.0"/>
 		<PackageVersion Include="Terminal.Gui" Version="1.16.0"/>
-		<PackageVersion Include="YamlDotNet" Version="15.1.2"/>
+		<PackageVersion Include="YamlDotNet" Version="15.1.4"/>
 		<PackageVersion Include="ConsoleControl" Version="1.3.0"/>
-		<PackageVersion Include="Autoupdater.NET.Official" Version="1.8.5"/>
+		<PackageVersion Include="Autoupdater.NET.Official" Version="1.8.6"/>
 		<PackageVersion Include="DockPanelSuite.ThemeVS2015" Version="3.1.0"/>
 		<PackageVersion Include="System.Threading.ThreadPool" Version="4.3.0"/>
 		<PackageVersion Include="WeCantSpell.Hunspell" Version="5.0.0"/>
@@ -36,8 +36,7 @@
 		<PackageVersion Include="NUnit3TestAdapter" Version="4.5.0"/>
 		<PackageVersion Include="NUnit.Analyzers" Version="4.2.0"/>
 		<PackageVersion Include="NSubstitute" Version="5.1.0"/>
-		<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.9.0"/>
+		<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.10.0"/>
 		<PackageVersion Include="coverlet.collector" Version="6.0.2"/>
-		<PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="6.0.0" />
 	</ItemGroup>
 </Project>

Documentation/CodeTutorials/Packages.md

@@ -25,7 +25,7 @@
 | LibArchive.Net | [GitHub](https://github.com/jas88/libarchive.net) | [BSD](https://opensource.org/license/bsd-2-clause/) | Access archive formats without the LZMA bugs of SharpCompress | |
 | [NLog](https://nlog-project.org/) | [GitHub](https://github.com/NLog/NLog) | [BSD 3-Clause](https://github.com/NLog/NLog/blob/dev/LICENSE.txt) | Flexible user configurable logging | |
 | HIC.FAnsiSql |[GitHub](https://github.com/HicServices/FAnsiSql) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | [DBMS] abstraction layer |
-| HIC.BadMedicine | [GitHub](https://github.com/HicServices/BadMedicine) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Generate Test Datasets for tests/exericses |
+| HIC.SynthEHR | [GitHub](https://github.com/HicServices/SynthEHR) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Generate Test Datasets for tests/exericses |
 | SSH.NET  | [GitHub](https://github.com/sshnet/SSH.NET)  | [MIT](https://github.com/sshnet/SSH.NET/blob/develop/LICENSE) | Enables fetching files from SFTP servers |
 | Moq 4 | [GitHub](https://github.com/moq/moq4) |[BSD 3](https://github.com/moq/moq4/blob/master/License.txt)  | Mock objects during unit testing |
 | [Newtonsoft.Json](https://www.newtonsoft.com/json) | [GitHub](https://github.com/JamesNK/Newtonsoft.Json) | [MIT](https://opensource.org/licenses/MIT) | Serialization of objects for sharing/transmission |
@@ -40,5 +40,4 @@
 | Autoupdater.NET.Official | [GitHub](https://github.com/ravibpatel/AutoUpdater.NET) | MIT | Manages updating of the RDMP windows client directly from the RDMP GitHub Releases|
 | ConsoleControl | [GitHub](https://github.com/dwmkerr/consolecontrol)  | MIT | Runs RDMP cli subprocesses|
 | Terminal.Gui                            | [GitHub](https://github.com/gui-cs/Terminal.Gui)                           | [MIT](https://opensource.org/licenses/MIT)                                                     | Console user-interface|
-| Microsoft.Bcl.AsyncInterfaces | [GitHub](https://github.com/dotnet/runtime) |[MIT](https://opensource.org/licenses/MIT)  | |
 [DBMS]: ./Glossary.md#DBMS

Documentation/CodeTutorials/PluginWriting.md

@@ -1436,7 +1436,7 @@ When packaging `dotnet publish` results you can exclude dlls that already come w
 ```
 <files>
     <file src="Plugin\windows\bin\$configuration$\net6.0-windows\win-x64\publish\*"
-          exclude="**\BadMedicine.Core.dll;**\FAnsi.*;**\MapsDirectlyToDatabaseTable.dll;**\MySql.Data.dll;**\Oracle.ManagedDataAccess.dll;**\Rdmp.Core.dll;**\NPOI.*;**\Renci.*;**\MathNet.Numerics.dll*;**\Rdmp.UI.dll;**\ScintillaNET.dll;**\ReusableUIComponents.dll;**\ObjectListView.dll;**\WeifenLuo.WinFormsUI.Docking*"
+          exclude="**\SynthEHR.Core.dll;**\FAnsi.*;**\MapsDirectlyToDatabaseTable.dll;**\MySql.Data.dll;**\Oracle.ManagedDataAccess.dll;**\Rdmp.Core.dll;**\NPOI.*;**\Renci.*;**\MathNet.Numerics.dll*;**\Rdmp.UI.dll;**\ScintillaNET.dll;**\ReusableUIComponents.dll;**\ObjectListView.dll;**\WeifenLuo.WinFormsUI.Docking*"
           target="lib\windows" />
 </files>
 ```

Documentation/CodeTutorials/TestPlan.md

@@ -0,0 +1,56 @@
+# RDMP Test Plan
+The RDMP test plan outlines the questions that should be asked of new changes to help ensure the correct changes are being made at the correct time.
+These questions are designed to help highlight any steps in the testing process that may have been missed.
+
+If you are looking to release a new version with this change, please look at the [Release Testing](#release_testing) section below.
+
+## Generating Test Data
+There is functionality to generate example data built into RDMP. This functionality has a 'nightmare' mode, which can generate large amounts of data for testing purposes.
+This mode can be enabled when creating platform databases by adding the 'Nightmare' flag to the example data settings during platform creation.
+Additional data can be used by using the 'Factor' flag to increase the amount of data, this will multiply the number of objects created by your specified Factor.
+
+## Questions To Ask
+### Does this change add a database migration?
+If the change adds a database migration, ask yourself the following questions:
+* Is the change backwards compatable?
+* Have I tested the change with a fresh install of RDMP?
+* Have I tested the change through the upgrade path with populated data that my change affects?
+* Should this change be included in a patch release? or be part of the next minor release? Or even a major release?
+    * Patch releases contain small improvements and should be usable interchangably with other versions within the same minor version e.g v8.1.4 and v8.1.5
+    * Minor released contain new functionality and should be backwards compatable with other versions in the same major version e.g. v8.1.0 and v8.2.0
+    * Major releases contain changes that are not backwards compatable
+
+### Does this change add new functionality?
+If the change adds new functionality, ask yourself the following questions:
+* Is the functionality usable via the GUI? If no ,why not?
+* Is the functionality usable via the CLI? If no, why not?
+* Has the functionalty been covered via unit tests?
+* Has the functionalty been manually tested?
+    * Has the happy path been tested?
+        * This is the expected path, where users are paying attention and are on their best behaviour
+    * Has the sad path been tested?
+        * This is where the user tries to be as obtuse as possible
+* Does this change do any data processing? If so, check the performance questions below
+
+### Does this change have any impact on performance?
+If the change adds or amends functionality that processes data
+* Is the functionality performant?
+    * Can the space/time complexity of the functions be reduced? 
+* Does the functionality handle large datasets (>1GB) efficiently?
+
+
+### What assumptions have been made?
+* Have any assumptions about how this functionality will be used been made?
+* Have any assumptions about the input data been made? Can these assumptions be extracted out into configuration?
+
+## <a name="release_testing"></a>Release Testing
+The release testing process should be completed once the release is feature complete and no more code changes are due to take place.
+This testing is to ensure that all code changes made during this release play nicely together and have no unintended side effcts.
+It may be useful at this point to revisit each piece of functionality and ensure they work as expected and perform some user acceptance testing on them, in light with the questions above.
+### What's changed?
+* Does the changelog accuratly reflect the pull request changes made?
+* If there are database migrations, are they all correctly sequenced and not overlapping?
+* Does the release work as epxected with a fresh install and via the upgrade path?
+* Is all functionality documented?
+* Are all version numbers bumped correctly?
+* Do all of the managed plugins work with the new release without issues or warnings?

Documentation/CodeTutorials/UserManual.md

@@ -129,7 +129,7 @@ _\* Unless using [file system backend](./YamlRepository.md)_
 ## Example Data
 Example data can be setup on install by ticking the 'Example Datasets' checkbox during platform database setup.
 
-New example data can be generated through the `Diagnostics=>Generate Test  Data` menu. Or with the [BadMedicine](https://github.com/HicServices/BadMedicine) command line tool. 
+New example data can be generated through the `Diagnostics=>Generate Test  Data` menu. Or with the [SynthEHR](https://github.com/HicServices/SynthEHR) command line tool. 
 
 ## Importing a flat file as a new dataset
 If you have some CSV files (or Excel/Fixed Width) that you want to load into your database, you can do so with the RDMP bulk import tool.
@@ -384,7 +384,7 @@ drop trigger Prescribing_OnUpdate;
 
 Each data load in RDMP has a single [LoadMetadata] object which acts as the root for the configuration.  A given configuration will load one or more tables (which must have primary keys).  The tables loaded includes all the tables that underly the [Catalogues] which are ­associated with the [LoadMetadata].  For example if a Catalogue 'Biochemistry' includes a join on two tables 'Header' and 'Results' then a [LoadMetadata] which includes 'Biochemistry' would include both the 'Header' and 'Results' tables.
 
-Create a new Biochemistry CSV file with [BadMedicine] and add a primary key to the 'Biochemistry' table in your example data.  The key should be a composite primary key of:
+Create a new Biochemistry CSV file with [SynthEHR] and add a primary key to the 'Biochemistry' table in your example data.  The key should be a composite primary key of:
 
 - chi
 - SampleDate
@@ -458,7 +458,7 @@ Set the following properties:
 
 ### Running the load
 
-Place the Biochemistry.csv file generated by [BadMedicine] into the ForLoading directory of your load (e.g. `C:\temp\biochem\Data\ForLoading`).
+Place the Biochemistry.csv file generated by [SynthEHR] into the ForLoading directory of your load (e.g. `C:\temp\biochem\Data\ForLoading`).
 
 Right click the [LoadMetadata] and select 'Check and Execute'
 
@@ -838,7 +838,7 @@ _The question mark after ExtractionConfiguration means match all where the Proje
 
 
 [AggregateConfiguration]: ./Glossary.md#AggregateConfiguration
-[BadMedicine]: https://github.com/HicServices/BadMedicine
+[SynthEHR]: https://github.com/HicServices/SynthEHR
 [Catalogue]: ./Glossary.md#Catalogue
 [Catalogues]: ./Glossary.md#Catalogue
 [CohortIdentificationConfiguration]: ./Glossary.md#CohortIdentificationConfiguration

Documentation/DataLoadEngine/RemoteAttachers.md

@@ -48,7 +48,8 @@ The full configuration options are
 | Delta Reading Look Forward Days               | If using the Delta Reading fetch duration, this is how many days forward in time you wish to look each time                                                                                                                                                                                                                                                  |
 | Set Delta Reading To Last Seen Date Post Load | Optional overwrite to the Delta Reading fetch option. Will use the most recently seen date in the fetched data rather than the adding the forward look days amount onto the stored minimum date                                                                                                                                                              |
 | Culture                                      | Optionally specify a custom date format                                                                                                                                                                                                                                                                                                                     |
-| Explicit Date Time Format                    | Optionally specify a specific datetime format        
+| Explicit Date Time Format                    | Optionally specify a specific datetime format    
+| Selected Columns                             | Optionally select which columns you wish to pull from the remote server (defaults to "*" )
 
 ## Configuring the Remote Database Attacher
 The Remote Database Attacher has a number of configuration options the required fields are:
@@ -68,4 +69,11 @@ The Remote Database Attacher has a number of configuration options the required
 | Delta Reading Look Forward Days               | If using the Delta Reading fetch duration, this is how many days forward in time you wish to look each time                                                                                                                                                                                                                                                  |
 | Set Delta Reading To Last Seen Date Post Load | Optional overwrite to the Delta Reading fetch option. Will use the most recently seen date in the fetched data rather than the adding the forward look days amount onto the stored minimum date                                                                                                                                                              |
 | Culture                                      | Optionally specify a custom date format                                                                                                                                                                                                                                                                                                                     |
-| Explicit Date Time Format                    | Optionally specify a specific datetime format     
+| Explicit Date Time Format                    | Optionally specify a specific datetime format   
+| Selected Columns                             | Optionally select which columns you wish to pull from the remote server (defaults to "*" )
+
+
+## Using Historical Loading with custom SQL
+By default, historical fetch configuration will be ignored in custome select SQL.
+To enable the use of hostorical fetching, include the string "$RDMPDefinedWhereClause" in the where clause of your query
+e.e. "Select * from Biochemistry Where $RDMPDefinedWhereClause"

Rdmp.Core.Tests/Curation/Unit/ExerciseData/TestBiochemistryCreation.cs

@@ -6,8 +6,8 @@
 
 using System;
 using System.IO;
-using BadMedicine;
-using BadMedicine.Datasets;
+using SynthEHR;
+using SynthEHR.Datasets;
 using NUnit.Framework;
 
 namespace Rdmp.Core.Tests.Curation.Unit.ExerciseData;

Rdmp.Core.Tests/Curation/Unit/ExerciseData/TestDemographyCreation.cs

@@ -6,8 +6,8 @@
 
 using System;
 using System.IO;
-using BadMedicine;
-using BadMedicine.Datasets;
+using SynthEHR;
+using SynthEHR.Datasets;
 using NUnit.Framework;
 
 namespace Rdmp.Core.Tests.Curation.Unit.ExerciseData;

Rdmp.Core.Tests/Curation/Unit/ExerciseData/TestPrescribingCreation.cs

@@ -7,8 +7,8 @@
 using System;
 using System.IO;
 using System.Linq;
-using BadMedicine;
-using BadMedicine.Datasets;
+using SynthEHR;
+using SynthEHR.Datasets;
 using NUnit.Framework;
 
 namespace Rdmp.Core.Tests.Curation.Unit.ExerciseData;

Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationTest.cs

@@ -8,7 +8,7 @@
 using System.Data;
 using System.IO;
 using System.Linq;
-using BadMedicine.Datasets;
+using SynthEHR.Datasets;
 using FAnsi.Discovery;
 using NUnit.Framework;
 using Rdmp.Core.Curation.Data;

Rdmp.Core.Tests/DataLoad/Engine/Integration/DataTableUploadDestinationTests.cs

@@ -1251,7 +1251,9 @@ public void TwoBatch_BooleanResizingTest(DatabaseType dbType, bool giveNullValue
 
         destination.ProcessPipelineData(dt1, toConsole, token);
 
-        Assert.That(db.ExpectTable("DataTableUploadDestinationTests").DiscoverColumn("TestedCol").DataType.SQLType, Is.EqualTo("bit"));
+        // MS SQL is odd and uses "bit" as a boolean, MySQL has a boolean type but aliases it to tinyint(1)
+        Assert.That(db.ExpectTable("DataTableUploadDestinationTests").DiscoverColumn("TestedCol").DataType?.SQLType,
+            Is.EqualTo(dbType == DatabaseType.MicrosoftSQLServer ? "bit" : "tinyint(1)"));
 
         destination.ProcessPipelineData(dt2, toMemory, token);
 

Rdmp.Core.Tests/QueryCaching/QueryCachingCrossServerTests.cs

@@ -7,8 +7,8 @@
 using System;
 using System.Linq;
 using System.Threading;
-using BadMedicine;
-using BadMedicine.Datasets;
+using SynthEHR;
+using SynthEHR.Datasets;
 using FAnsi;
 using FAnsi.Discovery;
 using NUnit.Framework;

Rdmp.Core/CohortCreation/Execution/ExamplePluginCohortCompiler.cs

@@ -9,7 +9,7 @@
 using System.Data;
 using System.Linq;
 using System.Threading;
-using BadMedicine;
+using SynthEHR;
 using Rdmp.Core.Curation.Data;
 using Rdmp.Core.Curation.Data.Aggregation;
 using Rdmp.Core.QueryCaching.Aggregation;

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractionCategory.cs

@@ -69,17 +69,25 @@ public override void Execute()
             Show("Cannot set the Extraction Category to 'Core' for a  Project Specific Catalogue item. It will be saved as 'Project Specific'.");
         }
 
-        if (ExecuteWithCommit(() => ExecuteImpl(c.Value), $"Set ExtractionCategory to '{c}'", _extractionInformations))
+        if (ExecuteWithCommit(() => ExecuteImpl(c.Value), c == ExtractionCategory.NotExtractable ? "Make Not Extractable" : $"Set ExtractionCategory to '{c}'", _extractionInformations))
             //publish the root Catalogue
             Publish(_extractionInformations.First());
     }
 
     private void ExecuteImpl(ExtractionCategory category)
     {
+
         foreach (var ei in _extractionInformations)
         {
-            ei.ExtractionCategory = category;
-            ei.SaveToDatabase();
+            if (category == ExtractionCategory.NotExtractable)
+            {
+                ei.DeleteInDatabase();
+            }
+            else
+            {
+                ei.ExtractionCategory = category;
+                ei.SaveToDatabase();
+            }
         }
     }
 }