Merge pull request #1380 from HicServices/ci-create-mysql-logging

Test creating a MySql logging db from RDMP running on yamlrepository

.github/workflows/build.yml

@@ -59,6 +59,12 @@ jobs:
           mysql-version: '8.0'
           root-password: 'YourStrong!Passw0rd'
           auto-start: true
+      - name: Create MySql Logging Db
+        run: dotnet run -c Release --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver LiveLoggingServer_ID "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_logging2" --dir ~/rdmp/rdmp-yaml/
+      - name: Create MySql DQE Db
+        run: dotnet run -c Release --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver DQE "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_dqe" --dir ~/rdmp/rdmp-yaml/
+      - name: Create MySql Cohort Building Query Caching Db
+        run: dotnet run -c Release --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver CohortIdentificationQueryCachingServer_ID "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_cache" --dir ~/rdmp/rdmp-yaml/
       - name: Build
         run: dotnet build --configuration Release --verbosity minimal
       - name: Initialise RDMP

Rdmp.Core.Tests/Curation/Integration/QueryBuildingTests/AggregateBuilderTests/MySqlAggregateBuilderTests.cs

@@ -8,6 +8,7 @@
 using NUnit.Framework;
 using Rdmp.Core.Curation.Data;
 using Rdmp.Core.QueryBuilding;
+using ReusableLibraryCode.Settings;
 
 namespace Rdmp.Core.Tests.Curation.Integration.QueryBuildingTests.AggregateBuilderTests
 {
@@ -44,21 +45,25 @@ Col1 desc
 
             topx.DeleteInDatabase();
         }
-        [Test]
-        public void Test_AggregateBuilder_MySql_Top31OrderByCountAsc()
+        [TestCase(true)]
+        [TestCase(false)]
+        public void Test_AggregateBuilder_MySql_Top31OrderByCountAsc(bool useAliasForGroupBy)
         {
             _ti.DatabaseType = DatabaseType.MySql;
             _ti.SaveToDatabase();
 
+            UserSettings.UseAliasInsteadOfTransformInGroupByAggregateGraphs = useAliasForGroupBy;
+
             var builder = new AggregateBuilder(null, "count(*)", null);
             builder.AddColumn(_dimension1);
 
             var topx = new AggregateTopX(CatalogueRepository, _configuration, 31);
             topx.OrderByDirection = AggregateTopXOrderByDirection.Ascending;
             builder.AggregateTopX = topx;
-
 
-            Assert.AreEqual(CollapseWhitespace(@"/**/
+            if (useAliasForGroupBy)
+            {
+                Assert.AreEqual(CollapseWhitespace(@"/**/
 SELECT 
 Col1,
 count(*) AS MyCount
@@ -69,9 +74,27 @@ group by
 order by 
 MyCount asc
 LIMIT 31"), CollapseWhitespace(builder.SQL));
+            }
+            else
+            {
+                Assert.AreEqual(CollapseWhitespace(@"/**/
+SELECT 
+Col1,
+count(*) AS MyCount
+FROM 
+T1
+group by 
+Col1
+order by 
+count(*) asc
+LIMIT 31"), CollapseWhitespace(builder.SQL));
+            }
+
 
 
             topx.DeleteInDatabase();
+
+            UserSettings.UseAliasInsteadOfTransformInGroupByAggregateGraphs = false;
         }
     }
 }

Rdmp.Core.Tests/Logging/DataLoadTaskHelper.cs

@@ -4,6 +4,7 @@
 // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
 // You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
 
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using FAnsi.Discovery;
@@ -41,9 +42,11 @@ public void CreateDataLoadTask(string taskName)
 
                 var taskCmd =
                     _loggingServer.GetCommand(
-                        "INSERT INTO DataLoadTask VALUES (100, '" + taskName + "', '" + taskName + "', GETDATE(), '" + datasetName + "', 1, 1, '" + datasetName + "')",
+                        "INSERT INTO DataLoadTask VALUES (100, '" + taskName + "', '" + taskName + "',@date, '" + datasetName + "', 1, 1, '" + datasetName + "')",
                         con);
 
+                _loggingServer.AddParameterWithValueToCommand("@date", taskCmd, DateTime.Now);
+
                 taskCmd.ExecuteNonQuery();
                 _sqlToCleanUp.Push("DELETE FROM DataLoadTask WHERE dataSetID = '" + datasetName + "'");
             }

Rdmp.Core.Tests/Logging/LogManagerTest.cs

@@ -238,6 +238,9 @@ public void LoggingDatabase_TestActuallyCreatingIt(DatabaseType type)
             Assert.AreEqual("bad.cs", archival.Errors.Single().Source);
 
             Assert.AreEqual("Wrote some records", archival.Progress.Single().Description);
+
+            var fatal = archival.Errors.Single();
+            lm.ResolveFatalErrors(new[] { fatal.ID }, DataLoadInfo.FatalErrorStates.Resolved, "problem resolved by building more towers");
         }
     }
 }

Rdmp.Core/CommandExecution/BasicActivateItems.cs

@@ -779,7 +779,7 @@ public virtual ExternalDatabaseServer CreateNewPlatformDatabase(ICatalogueReposi
                 throw new ArgumentException($"Database must be picked before calling {nameof(CreateNewPlatformDatabase)} when using {nameof(BasicActivateItems)}",nameof(db));
 
             MasterDatabaseScriptExecutor executor = new MasterDatabaseScriptExecutor(db);
-            executor.CreateAndPatchDatabase(patcher,new AcceptAllCheckNotifier());
+            executor.CreateAndPatchDatabase(patcher, new AcceptAllCheckNotifier() { WriteToConsole = true});
 
             var eds = new ExternalDatabaseServer(catalogueRepository,"New " + (defaultToSet == PermissableDefaults.None ? "" :  defaultToSet.ToString()) + "Server",patcher);
             eds.SetProperties(db);

Rdmp.Core/Curation/Data/Defaults/PermissableDefaultsExtensions.cs

@@ -27,7 +27,7 @@ public static IPatcher ToTier2DatabaseType(this PermissableDefaults permissableD
                 case PermissableDefaults.IdentifierDumpServer_ID:
                     return new IdentifierDumpDatabasePatcher();
                 case PermissableDefaults.DQE:
-                    return new IdentifierDumpDatabasePatcher();
+                    return new DataQualityEnginePatcher();
                 case PermissableDefaults.WebServiceQueryCachingServer_ID:
                     return new QueryCachingPatcher();
                 case PermissableDefaults.CohortIdentificationQueryCachingServer_ID:

Rdmp.Core/DataQualityEngine/Data/ColumnState.cs

@@ -133,7 +133,7 @@ public void Commit(Evaluation evaluation,string pivotCategory, DbConnection con,
                 throw new NotSupportedException("ColumnState was already committed");
 
             var sql = string.Format(
-               "INSERT INTO [dbo].[ColumnState]([TargetProperty],[DataLoadRunID],[Evaluation_ID],[CountCorrect],[CountDBNull],[ItemValidatorXML],[CountMissing],[CountWrong],[CountInvalidatesRow],[PivotCategory])VALUES({0},{1},{2},{3},{4},{5},{6},{7},{8},{9})",
+               "INSERT INTO ColumnState(TargetProperty,DataLoadRunID,Evaluation_ID,CountCorrect,CountDBNull,ItemValidatorXML,CountMissing,CountWrong,CountInvalidatesRow,PivotCategory)VALUES({0},{1},{2},{3},{4},{5},{6},{7},{8},{9})",
                "@TargetProperty",
                DataLoadRunID
                ,evaluation.ID

Rdmp.Core/DataQualityEngine/Data/PeriodicityState.cs

@@ -7,6 +7,7 @@
 using System;
 using System.Collections.Generic;
 using System.Data;
+using MapsDirectlyToDatabaseTable;
 using Rdmp.Core.Validation.Constraints;
 using ReusableLibraryCode;
 
@@ -56,17 +57,19 @@ public PeriodicityState(int year, int month, Consequence? consequence )
            var calc = new DatasetTimespanCalculator();
            var result = calc.GetMachineReadableTimespanIfKnownOf(evaluation, discardOutliers);
 
+            var t = evaluation.DQERepository;
+
            using (var con = evaluation.DQERepository.GetConnection())
            {
                var sql = 
-               @"SELECT 
-      [Year]
-      ,[Month]
+               @$"SELECT 
+      {t.Wrap("Year")}
+      ,{t.Wrap("Month")}
       ,RowEvaluation
       ,CountOfRecords
   FROM [PeriodicityState]
     where
-  Evaluation_ID = " + evaluation.ID + " and PivotCategory = 'ALL' ORDER BY [Year],[Month]";
+  Evaluation_ID = ${evaluation.ID} and PivotCategory = 'ALL' ORDER BY {t.Wrap("Year")},{t.Wrap("Month")}";
 
                using(var cmd = DatabaseCommandHelper.GetCommand(sql, con.Connection, con.Transaction))
                {
@@ -179,15 +182,10 @@ public void Commit(Evaluation evaluation, string pivotCategory)
                 if (IsCommitted)
                     throw new NotSupportedException("PeriodicityState was already committed");
 
+                var t = evaluation.DQERepository;
+
                 string sql =
-                    string.Format(
-                        "INSERT INTO [dbo].[PeriodicityState]([Evaluation_ID],[Year],[Month],[CountOfRecords],[RowEvaluation],[PivotCategory])VALUES({0},{1},{2},{3},{4},{5})"
-                        ,evaluation.ID
-                        ,Year
-                        ,Month
-                        ,CountOfRecords
-                        , "@RowEvaluation",
-                        "@PivotCategory");
+                        $"INSERT INTO PeriodicityState(Evaluation_ID,{t.Wrap("Year")},{t.Wrap("Month")},CountOfRecords,RowEvaluation,PivotCategory)VALUES({evaluation.ID},{Year},{Month},{CountOfRecords},@RowEvaluation,@PivotCategory)";
 
                 using (var cmd = DatabaseCommandHelper.GetCommand(sql, con.Connection, con.Transaction))
                 {

Rdmp.Core/DataQualityEngine/Data/RowState.cs

@@ -43,7 +43,7 @@ public RowState(Evaluation evaluation, int dataLoadRunID, int correct, int missi
         {
 
             var sql = string.Format(
-                "INSERT INTO [dbo].[RowState]([Evaluation_ID],[Correct],[Missing],[Wrong],[Invalid],[DataLoadRunID],[ValidatorXML],[PivotCategory])VALUES({0},{1},{2},{3},{4},{5},@validatorXML,{6})",
+                "INSERT INTO RowState(Evaluation_ID,Correct,Missing,Wrong,Invalid,DataLoadRunID,ValidatorXML,PivotCategory)VALUES({0},{1},{2},{3},{4},{5},@validatorXML,{6})",
                 evaluation.ID,
                 correct,
                 missing,

Rdmp.Core/Databases/DataQualityEnginePatcher.cs

@@ -23,7 +23,7 @@ public DataQualityEnginePatcher() : base(2, "Databases.DataQualityEngineDatabase
         }
         public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db)
         {
-            var header = GetHeader(InitialScriptName, new Version(1, 0, 0));
+            var header = GetHeader(db.Server.DatabaseType,InitialScriptName, new Version(1, 0, 0));
 
             var sql = new StringBuilder();
 
@@ -32,7 +32,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db)
                 new DatabaseColumnRequest("DateOfEvaluation",new DatabaseTypeRequest(typeof(DateTime))),
                 new DatabaseColumnRequest("CatalogueID",new DatabaseTypeRequest(typeof(int))){AllowNulls = false},
                 new DatabaseColumnRequest("ID",new DatabaseTypeRequest(typeof (int))){IsAutoIncrement = true, IsPrimaryKey = true}
-            },null,false,null));
+            },null,false,null).TrimEnd() + ";");
 
             // foreign keys
             var evaluationId = new DiscoveredColumn(db.ExpectTable("Evaluation"), "ID", false);
@@ -57,7 +57,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db)
             }, new Dictionary<DatabaseColumnRequest, DiscoveredColumn>
             {
                 {columnState_Evaluation_ID ,evaluationId }
-            },true , null));
+            },true , null).TrimEnd() + ";");
 
             sql.AppendLine(db.Helper.GetCreateTableSql(db, "RowState", new[]
             {
@@ -72,7 +72,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db)
             }, new Dictionary<DatabaseColumnRequest, DiscoveredColumn>
             {
                 {rowState_Evaluation_ID ,evaluationId }
-            }, true, null));
+            }, true, null).TrimEnd() + ";");
 
 
             sql.AppendLine(db.Helper.GetCreateTableSql(db, "PeriodicityState", new[]
@@ -86,7 +86,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db)
             }, new Dictionary<DatabaseColumnRequest, DiscoveredColumn>
             {
                 {periodicityState_Evaluation_ID ,evaluationId }
-            }, true, null));
+            }, true, null).TrimEnd() + ";");
 
             sql.AppendLine(db.Helper.GetCreateTableSql(db, "DQEGraphAnnotation", new[]
             {
@@ -104,7 +104,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db)
             }, new Dictionary<DatabaseColumnRequest, DiscoveredColumn>
             {
                 {annotation_Evaluation_ID ,evaluationId }
-            }, true, null));
+            }, true, null).TrimEnd() + ";");
 
             return new Patch(InitialScriptName, header + sql);
         }