Merge branch 'develop' into feature/database-selector-improv

CHANGELOG.md

@@ -8,6 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- More detailed logging of Type decisions when extracting to database
+- Added ability to cancel ongoing queries in CLI Sql Editor
+- Added 'Reset Sql' and 'Clear Sql' buttons to CLI Sql Editor
+- Added view data/aggregate etc on ColumnInfo objects to list of commands accessible from the CLI gui
 - Added 'Go To' commands to CLI gui
 - Exposed 'Add New Process Task...' to load stages in CLI menu
 

Rdmp.Core.Tests/DataLoad/Engine/Integration/DataTableUploadDestinationTests.cs

@@ -1263,6 +1263,48 @@ public void TwoBatch_MiscellaneousTest(DatabaseType dbType, string v1,string v2,
 
             Assert.AreEqual(expectedTypeForBatch2, tt.GetCSharpTypeForSQLDBType(colAfter.DataType.SQLType));
         }
+
+
+        [Test]
+        public void TwoBatch_ExplicitRealDataType()
+        {
+            var token = new GracefulCancellationToken();
+            DiscoveredDatabase db = GetCleanedServer(DatabaseType.MicrosoftSQLServer);
+            var toConsole = new ThrowImmediatelyDataLoadEventListener();
+            var toMemory = new ToMemoryDataLoadEventListener(true);
+
+            DataTableUploadDestination destination = new DataTableUploadDestination();
+            destination.PreInitialize(db, toConsole);
+            destination.AllowResizingColumnsAtUploadTime = true;
+            destination.AddExplicitWriteType("FloatCol","real");
+
+            DataTable dt1 = new DataTable();
+            dt1.Columns.Add("FloatCol", typeof(string));
+            dt1.Rows.Add(new[] { "1.51" });
+
+            dt1.TableName = "DataTableUploadDestinationTests";
+
+            DataTable dt2 = new DataTable();
+            dt2.Columns.Add("FloatCol", typeof(string));
+            dt2.Rows.Add(new[] { "99.9999" });
+
+            dt2.TableName = "DataTableUploadDestinationTests";
+
+            destination.ProcessPipelineData(dt1, toConsole, token);
+
+            Assert.AreEqual("real", db.ExpectTable("DataTableUploadDestinationTests").DiscoverColumn("FloatCol").DataType.SQLType);
+
+            destination.ProcessPipelineData(dt2, toMemory, token);
+
+            destination.Dispose(toConsole, null);
+
+            Assert.IsTrue(db.ExpectTable("DataTableUploadDestinationTests").Exists());
+            Assert.AreEqual(2, db.ExpectTable("DataTableUploadDestinationTests").GetRowCount());
+
+            // should still be real
+            Assert.AreEqual("real", db.ExpectTable("DataTableUploadDestinationTests").DiscoverColumn("FloatCol").DataType.SQLType);
+
+        }
         #endregion
     }
 }

Rdmp.Core/CommandExecution/AtomicCommandFactory.cs

@@ -283,6 +283,13 @@ public IEnumerable<CommandPresentation> GetCommandsWithPresentation(object o)
                 yield return new CommandPresentation(new ExecuteCommandNewObject(_activator,()=>new ColumnInfo(_activator.RepositoryLocator.CatalogueRepository, Guid.NewGuid().ToString(), "fish", ti)){OverrideCommandName = "Add New ColumnInfo" });
             }
 
+            if(Is(o,out ColumnInfo colInfo))
+            {
+                yield return new CommandPresentation(new ExecuteCommandViewData(_activator, ViewType.TOP_100, colInfo),"View Data");
+                yield return new CommandPresentation(new ExecuteCommandViewData(_activator, ViewType.Aggregate, colInfo),"View Data");
+                yield return new CommandPresentation(new ExecuteCommandViewData(_activator, ViewType.Distribution, colInfo),"View Data");
+            }
+
             if(Is(o, out AllStandardRegexesNode _))
                 yield return new CommandPresentation(new ExecuteCommandCreateNewStandardRegex(_activator));
 

Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs

@@ -141,7 +141,7 @@ private DataTableUploadDestination PrepareDestination(IDataLoadEventListener lis
 
             _destination = new DataTableUploadDestination();
 
-            PrimeDestinationTypesBasedOnCatalogueTypes(toProcess);
+            PrimeDestinationTypesBasedOnCatalogueTypes(listener,toProcess);
 
             _destination.AllowResizingColumnsAtUploadTime = true;
             _destination.AlterTimeout = AlterTimeout;
@@ -151,7 +151,7 @@ private DataTableUploadDestination PrepareDestination(IDataLoadEventListener lis
             return _destination;
         }
 
-        private void PrimeDestinationTypesBasedOnCatalogueTypes(DataTable toProcess)
+        private void PrimeDestinationTypesBasedOnCatalogueTypes(IDataLoadEventListener listener, DataTable toProcess)
         { 
             //if the extraction is of a Catalogue
             var datasetCommand = _request as IExtractDatasetCommand;
@@ -168,8 +168,22 @@ private void PrimeDestinationTypesBasedOnCatalogueTypes(DataTable toProcess)
                 var catItem = extractionInformation.CatalogueItem;
 
                 //if we do not know the data type or the ei is a transform
-                if (catItem == null || catItem.ColumnInfo == null || extractionInformation.IsProperTransform())
+                if (catItem == null)
+                {
+                    listener.OnNotify(this,new NotifyEventArgs(ProgressEventType.Warning,$"Did not copy Types for ExtractionInformation {extractionInformation} (ID={extractionInformation.ID}) because it had no associated CatalogueItem"));
+                        continue;
+                }
+                if(catItem.ColumnInfo == null)
+                {
+                    listener.OnNotify(this,new NotifyEventArgs(ProgressEventType.Warning,$"Did not copy Types for ExtractionInformation {extractionInformation} (ID={extractionInformation.ID}) because it had no associated ColumnInfo"));
+                        continue;
+                }
+
+                if(extractionInformation.IsProperTransform())
+                {
+                    listener.OnNotify(this,new NotifyEventArgs(ProgressEventType.Warning,$"Did not copy Types for ExtractionInformation {extractionInformation} (ID={extractionInformation.ID}) because it is a Transform"));
                     continue;
+                }
 
                 string destinationType = GetDestinationDatabaseType(extractionInformation);
 
@@ -183,6 +197,8 @@ private void PrimeDestinationTypesBasedOnCatalogueTypes(DataTable toProcess)
                 //if user wants to copy collation types and the destination server is the same type as the origin server
                 if (CopyCollations && _destinationDatabase.Server.DatabaseType == catItem.ColumnInfo.TableInfo.DatabaseType)
                     addedType.Collation = catItem.ColumnInfo.Collation;
+
+                listener.OnNotify(this,new NotifyEventArgs(ProgressEventType.Information,$"Set Type for {columnName} to {destinationType} (IsPrimaryKey={(addedType.IsPrimaryKey ? "true":"false")}) to match the source table"));
             }
 
             foreach (ReleaseIdentifierSubstitution sub in datasetCommand.QueryBuilder.SelectColumns.Where(sc => sc.IColumn is ReleaseIdentifierSubstitution).Select(sc => sc.IColumn))

Rdmp.Core/DataLoad/Engine/Pipeline/Destinations/DataTableUploadDestination.cs

@@ -14,6 +14,7 @@
 using FAnsi.Connections;
 using FAnsi.Discovery;
 using FAnsi.Discovery.TableCreation;
+using FAnsi.Extensions;
 using Rdmp.Core.Curation.Data;
 using Rdmp.Core.DataFlowPipeline;
 using Rdmp.Core.DataFlowPipeline.Requirements;
@@ -287,10 +288,17 @@ private void ResizeColumnsIfRequired(DataTable toProcess, IDataLoadEventListener
                 //if the SQL data type has degraded e.g. varchar(10) to varchar(50) or datetime to varchar(20)
                 if(oldSqlType != newSqlType)
                 {
-                    listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "Resizing column '" + column + "' from '" + oldSqlType + "' to '" + newSqlType + "'"));
-
                     var col = tbl.DiscoverColumn(column.ColumnName,_managedConnection.ManagedTransaction);
 
+
+                    if(AbandonAlter(col.DataType.SQLType,newSqlType, out string reason))
+                    {
+                        listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, $"Considered resizing column '{column}' from '{col.DataType.SQLType }' to '{ newSqlType }' but decided not to because:{reason}"));
+                        continue;
+                    }
+
+                    listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "Resizing column '" + column + "' from '" + col.DataType.SQLType + "' to '" + newSqlType + "'"));
+
                     //try changing the Type to the legit type
                     col.DataType.AlterTypeTo(newSqlType, _managedConnection.ManagedTransaction, AlterTimeout);
 
@@ -305,6 +313,29 @@ private void ResizeColumnsIfRequired(DataTable toProcess, IDataLoadEventListener
             listener.OnProgress(this,new ProgressEventArgs("Measuring DataType Sizes",new ProgressMeasurement(_affectedRows + toProcess.Rows.Count,ProgressType.Records),swMeasuringStrings.Elapsed));
         }
 
+        /// <summary>
+        /// Returns true if we should not be trying to do this alter after all
+        /// </summary>
+        /// <param name="oldSqlType">The database proprietary type you are considering altering from</param>
+        /// <param name="newSqlType">The ANSI SQL type you are considering altering to</param>
+        /// <param name="reason">Null or the reason we are returning true</param>
+        /// <returns>True if the proposed alter is a bad idea and shouldn't be attempted</returns>
+        protected virtual bool AbandonAlter(string oldSqlType, string newSqlType, out string reason)
+        {
+            var basicallyDecimalAlready = new List<string>(){ "real","double","float","single"};
+
+            var first = basicallyDecimalAlready.FirstOrDefault(c=>oldSqlType.Contains(c,CompareOptions.IgnoreCase));
+
+            if(first != null && newSqlType.Contains("decimal",CompareOptions.IgnoreCase))
+            {
+                reason = $"Resizing from {first} to decimal is a bad idea and likely to fail";
+                return true;
+            }
+
+            reason = null;
+            return false;
+        }
+
         public void Abort(IDataLoadEventListener listener)
         {
             _managedConnection.ManagedTransaction.AbandonAndCloseConnection();

Rdmp.UI/Menus/ColumnInfoMenu.cs

@@ -19,14 +19,7 @@ namespace Rdmp.UI.Menus
     class ColumnInfoMenu : RDMPContextMenuStrip
     {
         public ColumnInfoMenu(RDMPContextMenuStripArgs args, ColumnInfo columnInfo) : base(args, columnInfo)
-        {
-            var miViewData = new ToolStripMenuItem("View Data");
-            Items.Add(miViewData);
-
-            Add(new ExecuteCommandViewData(_activator, ViewType.TOP_100, columnInfo),Keys.None,miViewData);
-            Add(new ExecuteCommandViewData(_activator, ViewType.Aggregate, columnInfo), Keys.None, miViewData);
-            Add(new ExecuteCommandViewData(_activator, ViewType.Distribution, columnInfo), Keys.None, miViewData);
-
+        {            
             Add(new ExecuteCommandAddNewLookupTableRelationship(_activator, null,columnInfo.TableInfo));
 
             Items.Add(new ToolStripSeparator());

Tools/rdmp/CommandLine/Gui/ConsoleGuiSqlEditor.cs

@@ -10,7 +10,9 @@
 using System;
 using System.Collections.Generic;
 using System.Data;
+using System.Data.Common;
 using System.Text;
+using System.Threading.Tasks;
 using Terminal.Gui;
 
 namespace Rdmp.Core.CommandLine.Gui
@@ -21,6 +23,14 @@ class ConsoleGuiSqlEditor : Window
         private readonly IViewSQLAndResultsCollection _collection;
         private TableView tableView;
         private TextView textView;
+        private Button _btnRunOrCancel;
+        private Task _runSqlTask;
+        private DbCommand _runSqlCmd;
+
+        /// <summary>
+        /// The original SQL this control was launched with
+        /// </summary>
+        private string _orignalSql;
 
         public ConsoleGuiSqlEditor(IBasicActivateItems activator,IViewSQLAndResultsCollection collection)
         {
@@ -34,21 +44,38 @@ public ConsoleGuiSqlEditor(IBasicActivateItems activator,IViewSQLAndResultsColle
                 Y=0,
                 Width = Dim.Fill(),
                 Height = Dim.Percent(30),
-                Text = collection.GetSql().Replace("\r\n","\n")
+                Text = _orignalSql = collection.GetSql().Replace("\r\n","\n")
             };
 
             Add(textView);
 
-            var btnRun = new Button("Run"){
+            _btnRunOrCancel = new Button("Run"){
                 X= 0,
                 Y=Pos.Bottom(textView),
                 };
 
-            btnRun.Clicked += ()=>RunSql();
-            Add(btnRun);
+            _btnRunOrCancel.Clicked += ()=>RunOrCancel();
+            Add(_btnRunOrCancel);
+
+            var resetSql = new Button("Reset Sql"){
+                X= Pos.Right(_btnRunOrCancel),
+                Y= Pos.Bottom(textView),
+                };
+
+            resetSql.Clicked += ()=>ResetSql();
+            Add(resetSql);
+
+            var clearSql = new Button("Clear Sql"){
+                X= Pos.Right(resetSql),
+                Y= Pos.Bottom(textView),
+                };
+
+            clearSql.Clicked += ()=>ClearSql();
+            Add(clearSql);
+
 
             var btnClose = new Button("Close"){
-                X= Pos.Right(btnRun),
+                X= Pos.Right(clearSql),
                 Y= Pos.Bottom(textView),
                 };
             btnClose.Clicked += ()=>Application.RequestStop();
@@ -65,6 +92,39 @@ public ConsoleGuiSqlEditor(IBasicActivateItems activator,IViewSQLAndResultsColle
             Add(tableView);
         }
 
+        private void ClearSql()
+        {
+            textView.Text = "";
+            textView.SetNeedsDisplay();
+        }
+
+        private void ResetSql()
+        {
+            textView.Text = _orignalSql;
+            textView.SetNeedsDisplay();
+        }
+
+        private void RunOrCancel()
+        {
+            // if task is still running we should cancel
+            if(_runSqlTask  != null && !_runSqlTask.IsCompleted)
+            {
+                // Cancel the sql command and let that naturally end the task
+                _runSqlCmd?.Cancel();
+            }
+            else
+            {
+                _runSqlTask = Task.Run(()=>RunSql());
+                _btnRunOrCancel.Text = "Cancel";
+                _btnRunOrCancel.SetNeedsDisplay();
+            }
+        }
+
+        private void SetReadyToRun()
+        {
+            _btnRunOrCancel.Text = "Run";
+            _btnRunOrCancel.SetNeedsDisplay();
+        }
         private void RunSql()
         {
             try
@@ -82,7 +142,9 @@ private void RunSql()
                 using(var con = db.Server.GetConnection())
                 {
                     con.Open();
-                    using(var da = db.Server.GetDataAdapter(sql,con))
+                    _runSqlCmd = db.Server.GetCommand(sql,con);
+
+                    using(var da = db.Server.GetDataAdapter(_runSqlCmd))
                     {
                         var dt = new DataTable();
                         da.Fill(dt);
@@ -95,6 +157,10 @@ private void RunSql()
             {
                 _activator.ShowException("Failed to run query",ex);
             }
+            finally
+            {
+                SetReadyToRun();
+            }
         }
     }
 }

Tools/rdmp/CommandLine/Gui/Windows/TableView.cs

@@ -567,7 +567,7 @@ public override bool ProcessKey (KeyEvent keyEvent)
 		{
 			if(Table == null){
 				PositionCursor ();
-				return true;
+				return false;
 			}
 
 			if(keyEvent.Key == CellActivationKey && Table != null) {