Robustness improvements in prep for implementing Virtual Terminal Seq…

…uences (gui-cs#3094)

* Fixes gui-cs#2616. Support combining sequences that don't normalize

* Decouples Application from ConsoleDriver in TestHelpers

* Updates driver tests to match new arch

* Start on making all driver tests test all drivers

* Improves handling if combining marks.

* Fix unit tests fails.

* Fix unit tests fails.

* Handling combining mask.

* Tying to fix this unit test that sometimes fail.

* Add support for combining mask on NetDriver.

* Enable CombiningMarks as List<Rune>.

* Prevents combining marks on invalid runes default and space.

* Formatting for CI tests.

* Fix non-normalized combining mark to add 1 to Col.

* Reformatting for retest the CI.

* Forces non-normalized CMs to be ignored.

* Initial experiment

* Created ANSiDriver. Updated UI Catalog command line handling

* Fixed ForceDriver logic

* Fixed ForceDriver logic

* Updating P/Invoke

* Force16 colors WIP

* Fixed 16 colo mode

* Updated unit tests

* UI catalog tweak

* Added chinese scenario from bdisp

* Disabled AnsiDriver unit tests for now.

* Code cleanup

* Initial commit (fork from v2_fixes_2610_WT_VTS)

* Code cleanup

* Removed nativemethods.txt

* Removed not needed native stuff

* Code cleanup

* Ensures command line handler doesn't eat exceptions

---------

Co-authored-by: BDisp <bd.bdisp@gmail.com>

Terminal.Gui/Application.cs

@@ -33,15 +33,20 @@ namespace Terminal.Gui;
 /// </remarks>
 public static partial class Application {
 	/// <summary>
-	/// Gets the <see cref="ConsoleDriver"/> that has been selected. See also <see cref="UseSystemConsole"/>.
+	/// Gets the <see cref="ConsoleDriver"/> that has been selected. See also <see cref="ForceDriver"/>.
 	/// </summary>
 	public static ConsoleDriver Driver { get; internal set; }
 
 	/// <summary>
-	/// If <see langword="true"/>, forces the use of the System.Console-based (see <see cref="NetDriver"/>) driver. The default is <see langword="false"/>.
+	/// Forces the use of the specified driver (one of "fake", "ansi", "curses", "net", or "windows"). If
+	/// not specified, the driver is selected based on the platform.
 	/// </summary>
+	/// <remarks>
+	/// Note, <see cref="Application.Init(ConsoleDriver, string)"/> will override this configuration setting if
+	/// called with either `driver` or `driverName` specified.
+	/// </remarks>
 	[SerializableConfigurationProperty (Scope = typeof (SettingsScope))]
-	public static bool UseSystemConsole { get; set; } = false;
+	public static string ForceDriver { get; set; } = string.Empty;
 
 	/// <summary>
 	/// Gets or sets whether <see cref="Application.Driver"/> will be forced to output only the 16 colors defined in <see cref="ColorName"/>.
@@ -98,14 +103,13 @@ static List<CultureInfo> GetSupportedCultures ()
 	/// </para>
 	/// <para>
 	/// The <see cref="Run{T}(Func{Exception, bool}, ConsoleDriver)"/> function 
-	/// combines <see cref="Init(ConsoleDriver)"/> and <see cref="Run(Toplevel, Func{Exception, bool})"/>
+	/// combines <see cref="Init(ConsoleDriver, string)"/> and <see cref="Run(Toplevel, Func{Exception, bool})"/>
 	/// into a single call. An application cam use <see cref="Run{T}(Func{Exception, bool}, ConsoleDriver)"/> 
-	/// without explicitly calling <see cref="Init(ConsoleDriver)"/>.
+	/// without explicitly calling <see cref="Init(ConsoleDriver, string)"/>.
 	/// </para>
-	/// <param name="driver">
-	/// The <see cref="ConsoleDriver"/> to use. If not specified the default driver for the
-	/// platform will be used (see <see cref="WindowsDriver"/>, <see cref="CursesDriver"/>, and <see cref="NetDriver"/>).</param>
-	public static void Init (ConsoleDriver driver = null) => InternalInit (() => Toplevel.Create (), driver);
+	/// <param name="driver">The <see cref="ConsoleDriver"/> to use. If neither <paramref name="driver"/> or <paramref name="driverName"/> are specified the default driver for the platform will be used.</param>
+	/// <param name="driverName">The short name (e.g. "net", "windows", "ansi", "fake", or "curses") of the <see cref="ConsoleDriver"/> to use. If neither <paramref name="driver"/> or <paramref name="driverName"/> are specified the default driver for the platform will be used.</param>
+	public static void Init (ConsoleDriver driver = null, string driverName = null) => InternalInit (Toplevel.Create, driver, driverName);
 
 	internal static bool _initialized = false;
 	internal static int _mainThreadId = -1;
@@ -119,7 +123,7 @@ static List<CultureInfo> GetSupportedCultures ()
 	// Unit Tests - To initialize the app with a custom Toplevel, using the FakeDriver. calledViaRunT will be false, causing all state to be reset.
 	// 
 	// calledViaRunT: If false (default) all state will be reset. If true the state will not be reset.
-	internal static void InternalInit (Func<Toplevel> topLevelFactory, ConsoleDriver driver = null, bool calledViaRunT = false)
+	internal static void InternalInit (Func<Toplevel> topLevelFactory, ConsoleDriver driver = null, string driverName = null, bool calledViaRunT = false)
 	{
 		if (_initialized && driver == null) {
 			return;
@@ -147,15 +151,28 @@ internal static void InternalInit (Func<Toplevel> topLevelFactory, ConsoleDriver
 		Load (true);
 		Apply ();
 
-		Driver ??= Environment.OSVersion.Platform switch {
-			_ when _forceFakeConsole => new FakeDriver (), // for unit testing only
-			_ when UseSystemConsole => new NetDriver (),
-			PlatformID.Win32NT or PlatformID.Win32S or PlatformID.Win32Windows => new WindowsDriver (),
-			_ => new CursesDriver ()
-		};
+		// Ignore Configuration for ForceDriver if driverName is specified
+		if (!string.IsNullOrEmpty (driverName)) {
+			ForceDriver = driverName;
+		}
 
 		if (Driver == null) {
-			throw new InvalidOperationException ("Init could not determine the ConsoleDriver to use.");
+			var p = Environment.OSVersion.Platform;
+			if (string.IsNullOrEmpty (ForceDriver)) {
+				if (p == PlatformID.Win32NT || p == PlatformID.Win32S || p == PlatformID.Win32Windows) {
+					Driver = new WindowsDriver ();
+				} else {
+					Driver = new CursesDriver ();
+				}
+			} else {
+				var drivers = GetDriverTypes ();
+				var driverType = drivers.FirstOrDefault (t => t.Name.ToLower () == ForceDriver.ToLower ());
+				if (driverType != null) {
+					Driver = (ConsoleDriver)Activator.CreateInstance (driverType);
+				} else {
+					throw new ArgumentException ($"Invalid driver name: {ForceDriver}. Valid names are {string.Join (", ", drivers.Select (t => t.Name))}");
+				}
+			}
 		}
 
 		try {
@@ -168,10 +185,10 @@ internal static void InternalInit (Func<Toplevel> topLevelFactory, ConsoleDriver
 			throw new InvalidOperationException ("Unable to initialize the console. This can happen if the console is already in use by another process or in unit tests.", ex);
 		}
 
-		Driver.SizeChanged += Driver_SizeChanged;
-		Driver.KeyDown += Driver_KeyDown;
-		Driver.KeyUp += Driver_KeyUp;
-		Driver.MouseEvent += Driver_MouseEvent;
+		Driver.SizeChanged += (s, args) => OnSizeChanging (args);
+		Driver.KeyDown += (s, args) => OnKeyDown (args);
+		Driver.KeyUp += (s, args) => OnKeyUp (args);
+		Driver.MouseEvent += (s, args) => OnMouseEvent (args);
 
 		SynchronizationContext.SetSynchronizationContext (new MainLoopSyncContext ());
 
@@ -190,12 +207,29 @@ internal static void InternalInit (Func<Toplevel> topLevelFactory, ConsoleDriver
 
 	static void Driver_MouseEvent (object sender, MouseEventEventArgs e) => OnMouseEvent (e);
 
+	/// <summary>
+	/// Gets of list of <see cref="ConsoleDriver"/> types that are available.
+	/// </summary>
+	/// <returns></returns>
+	public static List<Type> GetDriverTypes ()
+	{
+		// use reflection to get the list of drivers
+		var driverTypes = new List<Type> ();
+		foreach (var asm in AppDomain.CurrentDomain.GetAssemblies ()) {
+			foreach (var type in asm.GetTypes ()) {
+				if (type.IsSubclassOf (typeof (ConsoleDriver)) && !type.IsAbstract) {
+					driverTypes.Add (type);
+				}
+			}
+		}
+		return driverTypes;
+	}
 
 	/// <summary>
-	/// Shutdown an application initialized with <see cref="Init(ConsoleDriver)"/>.
+	/// Shutdown an application initialized with <see cref="Init"/>.
 	/// </summary>
 	/// <remarks>
-	/// Shutdown must be called for every call to <see cref="Init(ConsoleDriver)"/> or <see cref="Application.Run(Toplevel, Func{Exception, bool})"/>
+	/// Shutdown must be called for every call to <see cref="Init"/> or <see cref="Application.Run(Toplevel, Func{Exception, bool})"/>
 	/// to ensure all resources are cleaned up (Disposed) and terminal settings are restored.
 	/// </remarks>
 	public static void Shutdown ()
@@ -394,7 +428,7 @@ public static RunState Begin (Toplevel Toplevel)
 	/// Runs the application by calling <see cref="Run(Toplevel, Func{Exception, bool})"/> 
 	/// with a new instance of the specified <see cref="Toplevel"/>-derived class.
 	/// <para>
-	/// Calling <see cref="Init(ConsoleDriver)"/> first is not needed as this function will initialize the application.
+	/// Calling <see cref="Init"/> first is not needed as this function will initialize the application.
 	/// </para>
 	/// <para>
 	/// <see cref="Shutdown"/> must be called when the application is closing (typically after Run> has 
@@ -407,7 +441,7 @@ public static RunState Begin (Toplevel Toplevel)
 	/// <param name="errorHandler"></param>
 	/// <param name="driver">The <see cref="ConsoleDriver"/> to use. If not specified the default driver for the
 	/// platform will be used (<see cref="WindowsDriver"/>, <see cref="CursesDriver"/>, or <see cref="NetDriver"/>).
-	/// Must be <see langword="null"/> if <see cref="Init(ConsoleDriver)"/> has already been called. 
+	/// Must be <see langword="null"/> if <see cref="Init"/> has already been called. 
 	/// </param>
 	public static void Run<T> (Func<Exception, bool> errorHandler = null, ConsoleDriver driver = null) where T : Toplevel, new ()
 	{
@@ -429,7 +463,7 @@ public static void Run<T> (Func<Exception, bool> errorHandler = null, ConsoleDri
 			}
 		} else {
 			// Init() has NOT been called.
-			InternalInit (() => new T (), driver, true);
+			InternalInit (() => new T (), driver, null, true);
 			Run (Top, errorHandler);
 		}
 	}
@@ -838,13 +872,12 @@ public static void End (RunState runState)
 	#endregion Run (Begin, Run, End)
 
 	#region Toplevel handling
-
 	/// <summary>
 	/// Holds the stack of TopLevel views.
 	/// </summary>
 	// BUGBUG: Techncally, this is not the full lst of TopLevels. THere be dragons hwre. E.g. see how Toplevel.Id is used. What
 	// about TopLevels that are just a SubView of another View?
-	static readonly Stack<Toplevel> _topLevels = new ();
+	static readonly Stack<Toplevel> _topLevels = new Stack<Toplevel> ();
 
 	/// <summary>
 	/// The <see cref="Toplevel"/> object used for the application on startup (<seealso cref="Application.Top"/>)
@@ -1296,7 +1329,7 @@ bool FrameHandledMouseEvent (Frame frame)
 	#endregion Mouse handling
 
 	#region Keyboard handling
-	static Key _alternateForwardKey = new (KeyCode.PageDown | KeyCode.CtrlMask);
+	static Key _alternateForwardKey = new Key (KeyCode.PageDown | KeyCode.CtrlMask);
 
 	/// <summary>
 	/// Alternative key to navigate forwards through views. Ctrl+Tab is the primary key.
@@ -1320,7 +1353,7 @@ static void OnAlternateForwardKeyChanged (KeyChangedEventArgs e)
 		}
 	}
 
-	static Key _alternateBackwardKey = new (KeyCode.PageUp | KeyCode.CtrlMask);
+	static Key _alternateBackwardKey = new Key (KeyCode.PageUp | KeyCode.CtrlMask);
 
 	/// <summary>
 	/// Alternative key to navigate backwards through views. Shift+Ctrl+Tab is the primary key.
@@ -1344,7 +1377,7 @@ static void OnAlternateBackwardKeyChanged (KeyChangedEventArgs oldKey)
 		}
 	}
 
-	static Key _quitKey = new (KeyCode.Q | KeyCode.CtrlMask);
+	static Key _quitKey = new Key (KeyCode.Q | KeyCode.CtrlMask);
 
 	/// <summary>
 	/// Gets or sets the key to quit the application.
@@ -1481,8 +1514,8 @@ public static bool OnKeyUp (Key a)
 	}
 	#endregion Keyboard handling
 }
-
 /// <summary>
 /// Event arguments for the <see cref="Application.Iteration"/> event.
 /// </summary>
-public class IterationEventArgs { }
+public class IterationEventArgs {
+}

Terminal.Gui/Configuration/ConfigurationManager.cs

@@ -257,7 +257,7 @@ public static void OnUpdated ()
 
 	/// <summary>
 	/// Resets the state of <see cref="ConfigurationManager"/>. Should be called whenever a new app session
-	/// (e.g. in <see cref="Application.Init(ConsoleDriver)"/> starts. Called by <see cref="Load"/>
+	/// (e.g. in <see cref="Application.Init"/> starts. Called by <see cref="Load"/>
 	/// if the <c>reset</c> parameter is <see langword="true"/>.
 	/// </summary>
 	/// <remarks>
@@ -412,7 +412,9 @@ public static void Load (bool reset = false)
 	{
 		Debug.WriteLine ($"ConfigurationManager.Load()");
 
-		if (reset) Reset ();
+		if (reset) {
+			Reset ();
+		}
 
 		// LibraryResources is always loaded by Reset
 		if (Locations == ConfigLocations.All) {

Terminal.Gui/ConsoleDrivers/ConsoleDriver.cs

@@ -58,12 +58,24 @@ public abstract class ConsoleDriver {
 	/// <summary>
 	/// The number of columns visible in the terminal.
 	/// </summary>
-	public virtual int Cols { get; internal set; }
+	public virtual int Cols {
+		get => _cols;
+		internal set {
+			_cols = value;
+			ClearContents();
+		}
+	}
 
 	/// <summary>
 	/// The number of rows visible in the terminal.
 	/// </summary>
-	public virtual int Rows { get; internal set; }
+	public virtual int Rows {
+		get => _rows;
+		internal set {
+			_rows = value;
+			ClearContents();
+		}
+	}
 
 	/// <summary>
 	/// The leftmost column in the terminal.
@@ -152,11 +164,19 @@ public void AddRune (Rune rune)
 			rune = rune.MakePrintable ();
 			runeWidth = rune.GetColumns ();
 			if (runeWidth == 0 && rune.IsCombiningMark ()) {
+				// AtlasEngine does not support NON-NORMALIZED combining marks in a way
+				// compatible with the driver architecture. Any CMs (except in the first col)
+				// are correctly combined with the base char, but are ALSO treated as 1 column
+				// width codepoints E.g. `echo "[e`u{0301}`u{0301}]"` will output `[é  ]`.
+				// 
+				// Until this is addressed (see Issue #), we do our best by 
+				// a) Attempting to normalize any CM with the base char to it's left
+				// b) Ignoring any CMs that don't normalize
 				if (Col > 0) {
 					if (Contents [Row, Col - 1].CombiningMarks.Count > 0) {
 						// Just add this mark to the list
 						Contents [Row, Col - 1].CombiningMarks.Add (rune);
-						// Don't move to next column (let the driver figure out what to do).
+						// Ignore. Don't move to next column (let the driver figure out what to do).
 					} else {
 						// Attempt to normalize the cell to our left combined with this mark
 						string combined = Contents [Row, Col - 1].Rune + rune.ToString ();
@@ -167,11 +187,11 @@ public void AddRune (Rune rune)
 							// It normalized! We can just set the Cell to the left with the
 							// normalized codepoint 
 							Contents [Row, Col - 1].Rune = (Rune)normalized [0];
-							// Don't move to next column because we're already there
+							// Ignore. Don't move to next column because we're already there
 						} else {
 							// It didn't normalize. Add it to the Cell to left's CM list
 							Contents [Row, Col - 1].CombiningMarks.Add (rune);
-							// Don't move to next column (let the driver figure out what to do).
+							// Ignore. Don't move to next column (let the driver figure out what to do).
 						}
 					}
 					Contents [Row, Col - 1].Attribute = CurrentAttribute;
@@ -398,6 +418,8 @@ public void ClearContents ()
 	}
 
 	Attribute _currentAttribute;
+	int _cols;
+	int _rows;
 
 	/// <summary>
 	/// The <see cref="Attribute"/> that will be used for the next <see cref="AddRune(Rune)"/> or <see cref="AddStr"/> call.

Terminal.Gui/ConsoleDrivers/ConsoleKeyMapping.cs

@@ -90,6 +90,10 @@ public static uint MapVKtoChar (VK vk)
 	[DllImport ("user32.dll")]
 	extern static bool GetKeyboardLayoutName ([Out] StringBuilder pwszKLID);
 
+	/// <summary>
+	/// Retrieves the name of the active input locale identifier (formerly called the keyboard layout) for the calling thread.
+	/// </summary>
+	/// <returns></returns>
 	public static string GetKeyboardLayoutName ()
 	{
 		if (Environment.OSVersion.Platform != PlatformID.Win32NT) {

Terminal.Gui/ConsoleDrivers/CursesDriver/CursesDriver.cs

@@ -17,12 +17,18 @@ namespace Terminal.Gui;
 class CursesDriver : ConsoleDriver {
 	public override int Cols {
 		get => Curses.Cols;
-		internal set => Curses.Cols = value;
+		internal set {
+			Curses.Cols = value;
+			ClearContents();
+		}
 	}
 
 	public override int Rows {
 		get => Curses.Lines;
-		internal set => Curses.Lines = value;
+		internal set {
+			Curses.Lines = value;
+			ClearContents();
+		}
 	}
 
 	CursorVisibility? _initialCursorVisibility = null;

Terminal.Gui/ConsoleDrivers/EscSeqUtils/EscSeqUtils.cs

@@ -170,10 +170,10 @@ public enum ClearScreenOptions {
 	/// <summary>
 	/// ESC [ y ; x H - CUP Cursor Position - Cursor moves to x ; y coordinate within the viewport, where x is the column of the y line
 	/// </summary>
-	/// <param name="x"></param>
-	/// <param name="y"></param>
+	/// <param name="row">Origin is (1,1).</param>
+	/// <param name="col">Origin is (1,1).</param>
 	/// <returns></returns>
-	public static string CSI_SetCursorPosition (int y, int x) => $"{CSI}{y};{x}H";
+	public static string CSI_SetCursorPosition (int row, int col) => $"{CSI}{row};{col}H";
 
 
 	//ESC [ <y> ; <x> f - HVP     Horizontal Vertical Position* Cursor moves to<x>; <y> coordinate within the viewport, where <x> is the column of the<y> line
@@ -248,15 +248,29 @@ public enum DECSCUSR_Style {
 	/// </summary>
 	public static string CSI_SetGraphicsRendition (params int [] parameters) => $"{CSI}{string.Join (";", parameters)}m";
 
+	/// <summary>
+	/// ESC [ (n) m - Uses <see cref="CSI_SetGraphicsRendition(int[])" /> to set the foreground color.
+	/// </summary>
+	/// <param name="code">One of the 16 color codes.</param>
+	/// <returns></returns>
+	public static string CSI_SetForegroundColor (AnsiColorCode code) => CSI_SetGraphicsRendition ((int)code);
+
+	/// <summary>
+	/// ESC [ (n) m - Uses <see cref="CSI_SetGraphicsRendition(int[])" /> to set the background color.
+	/// </summary>
+	/// <param name="code">One of the 16 color codes.</param>
+	/// <returns></returns>
+	public static string CSI_SetBackgroundColor (AnsiColorCode code) => CSI_SetGraphicsRendition ((int)code+10);
+
 	/// <summary>
 	/// ESC[38;5;{id}m - Set foreground color (256 colors)
 	/// </summary>
-	public static string CSI_SetForegroundColor (int id) => $"{CSI}38;5;{id}m";
+	public static string CSI_SetForegroundColor256 (int color) => $"{CSI}38;5;{color}m";
 
 	/// <summary>
 	/// ESC[48;5;{id}m - Set background color (256 colors)
 	/// </summary>
-	public static string CSI_SetBackgroundColor (int id) => $"{CSI}48;5;{id}m";
+	public static string CSI_SetBackgroundColor256 (int color) => $"{CSI}48;5;{color}m";
 
 	/// <summary>
 	/// ESC[38;2;{r};{g};{b}m	Set foreground color as RGB.