Save device logs when UI Tests fail

src/Controls/tests/UITests/Tests/Concepts/InputTransparencyGalleryTests.cs

@@ -20,17 +20,17 @@ protected override void NavigateToGallery()
 		}
 
 		[Test]
-		public void Simple([Values] Test.InputTransparency test) => RunTest(test.ToString());
+		public void InputTransparencySimple([Values] Test.InputTransparency test) => RunTest(test.ToString());
 
-		// [Test]
-		// [Combinatorial]
-		// public void Matrix([Values] bool rootTrans, [Values] bool rootCascade, [Values] bool nestedTrans, [Values] bool nestedCascade, [Values] bool trans)
-		// {
-		// 	var (clickable, passthru) = Test.InputTransparencyMatrix.States[(rootTrans, rootCascade, nestedTrans, nestedCascade, trans)];
-		// 	var key = Test.InputTransparencyMatrix.GetKey(rootTrans, rootCascade, nestedTrans, nestedCascade, trans, clickable, passthru);
+		[Test]
+		[Combinatorial]
+		public void InputTransparencyMatrix([Values] bool rootTrans, [Values] bool rootCascade, [Values] bool nestedTrans, [Values] bool nestedCascade, [Values] bool trans)
+		{
+			var (clickable, passthru) = Test.InputTransparencyMatrix.States[(rootTrans, rootCascade, nestedTrans, nestedCascade, trans)];
+			var key = Test.InputTransparencyMatrix.GetKey(rootTrans, rootCascade, nestedTrans, nestedCascade, trans, clickable, passthru);
 
-		// 	RunTest(key, clickable, passthru);
-		// }
+			RunTest(key, clickable, passthru);
+		}
 
 		void RunTest(string test, bool? clickable = null, bool? passthru = null)
 		{

src/TestUtils/src/UITest.Appium/AppiumAndroidApp.cs

@@ -58,8 +58,8 @@ public override ApplicationState AppState
 
 				return Convert.ToInt32(state) switch
 				{
-					0 => ApplicationState.Not_Installed,
-					1 => ApplicationState.Not_Running,
+					0 => ApplicationState.NotInstalled,
+					1 => ApplicationState.NotRunning,
 					3 or
 					4 => ApplicationState.Running,
 					_ => ApplicationState.Unknown,

src/TestUtils/src/UITest.Appium/AppiumApp.cs

@@ -5,16 +5,16 @@
 
 namespace UITest.Appium
 {
-	public abstract class AppiumApp : IApp
+	public abstract class AppiumApp : IApp, IScreenshotSupportedApp, ILogsSupportedApp
 	{
 		protected readonly AppiumDriver _driver;
 		protected readonly IConfig _config;
 		protected readonly AppiumCommandExecutor _commandExecutor;
 
 		public AppiumApp(AppiumDriver driver, IConfig config)
 		{
-			_driver = driver;
-			_config = config;
+			_driver = driver ?? throw new ArgumentNullException(nameof(driver));
+			_config = config ?? throw new ArgumentNullException(nameof(config));
 
 			_commandExecutor = new AppiumCommandExecutor();
 			_commandExecutor.AddCommandGroup(new AppiumPointerActions(this));
@@ -34,40 +34,34 @@ public AppiumApp(AppiumDriver driver, IConfig config)
 		public ICommandExecution CommandExecutor => _commandExecutor;
 		public string ElementTree => _driver.PageSource;
 
-		public void Click(float x, float y)
-		{
-			CommandExecutor.Execute("click", new Dictionary<string, object>()
-			{
-				{ "x", x },
-				{ "y", y }
-			});
-		}
-
 		public FileInfo Screenshot(string fileName)
 		{
-			if (_driver == null)
-			{
-				throw new NullReferenceException("Screenshot: _driver is null");
-			}
-
-			string filename = $"{fileName}.png";
 			Screenshot screenshot = _driver.GetScreenshot();
-			screenshot.SaveAsFile(filename, ScreenshotImageFormat.Png);
-			var file = new FileInfo(filename);
+			screenshot.SaveAsFile(fileName, ScreenshotImageFormat.Png);
+			var file = new FileInfo(fileName);
 			return file;
 		}
 
 		public byte[] Screenshot()
 		{
-			if (_driver == null)
-			{
-				throw new NullReferenceException("Screenshot: _driver is null");
-			}
-
 			Screenshot screenshot = _driver.GetScreenshot();
 			return screenshot.AsByteArray;
 		}
 
+		public IEnumerable<string> GetLogTypes()
+		{
+			return _driver.Manage().Logs.AvailableLogTypes;
+		}
+
+		public IEnumerable<string> GetLogEntries(string logType)
+		{
+			var entries = _driver.Manage().Logs.GetLog(logType);
+			foreach (var entry in entries)
+			{
+				yield return entry.Message;
+			}
+		}
+
 #nullable disable
 		public virtual IUIElement FindElement(string id)
 		{

src/TestUtils/src/UITest.Appium/AppiumCatalystApp.cs

@@ -26,7 +26,7 @@ public override ApplicationState AppState
 				// https://developer.apple.com/documentation/xctest/xcuiapplicationstate?language=objc
 				return Convert.ToInt32(state) switch
 				{
-					1 => ApplicationState.Not_Running,
+					1 => ApplicationState.NotRunning,
 					2 or
 					3 or
 					4 => ApplicationState.Running,

src/TestUtils/src/UITest.Appium/AppiumIOSApp.cs

@@ -28,7 +28,7 @@ public override ApplicationState AppState
 				// https://developer.apple.com/documentation/xctest/xcuiapplicationstate?language=objc
 				return Convert.ToInt32(state) switch
 				{
-					1 => ApplicationState.Not_Running,
+					1 => ApplicationState.NotRunning,
 					2 or
 					3 or
 					4 => ApplicationState.Running,

src/TestUtils/src/UITest.Appium/AppiumWindowsApp.cs

@@ -23,7 +23,7 @@ public override ApplicationState AppState
 				}
 				catch (NoSuchWindowException)
 				{
-					return ApplicationState.Not_Running;
+					return ApplicationState.NotRunning;
 				}
 			}
 		}

src/TestUtils/src/UITest.Core/ApplicationState.cs

@@ -2,9 +2,9 @@
 {
 	public enum ApplicationState
 	{
-		Not_Installed,
+		NotInstalled,
 		Installed,
-		Not_Running,
+		NotRunning,
 		Running,
 		Unknown
 	}

src/TestUtils/src/UITest.Core/IApp.cs

@@ -5,14 +5,65 @@ public interface IApp : IDisposable
 		IConfig Config { get; }
 		IUIElementQueryable Query { get; }
 		ApplicationState AppState { get; }
+
 		IUIElement FindElement(string id);
 		IUIElement FindElement(IQuery query);
 		IReadOnlyCollection<IUIElement> FindElements(string id);
 		IReadOnlyCollection<IUIElement> FindElements(IQuery query);
+		string ElementTree { get; }
+
 		ICommandExecution CommandExecutor { get; }
-		void Click(float x, float y);
+	}
+
+	public interface IScreenshotSupportedApp : IApp
+	{
 		FileInfo Screenshot(string fileName);
 		byte[] Screenshot();
-		string ElementTree { get; }
+	}
+
+	public interface ILogsSupportedApp : IApp
+	{
+		IEnumerable<string> GetLogTypes();
+		IEnumerable<string> GetLogEntries(string logType);
+	}
+
+	public static class AppExtensions
+	{
+		public static void Click(this IApp app, float x, float y)
+		{
+			app.CommandExecutor.Execute("click", new Dictionary<string, object>()
+			{
+				{ "x", x },
+				{ "y", y }
+			});
+		}
+
+
+		internal static T As<T>(this IApp app)
+			where T : IApp
+		{
+			if (app is not T derivedApp)
+				throw new NotImplementedException($"The app '{app}' does not implement '{typeof(T).FullName}'.");
+
+			return derivedApp;
+		}
+	}
+
+	public static class ScreenshotSupportedAppExtensions
+	{
+		public static FileInfo Screenshot(this IApp app, string fileName) =>
+			app.As<IScreenshotSupportedApp>().Screenshot(fileName);
+
+		public static byte[] Screenshot(this IApp app) =>
+			app.As<IScreenshotSupportedApp>().Screenshot();
+	}
+
+	public static class LogsSupportedAppExtensions
+	{
+		public static IEnumerable<string> GetLogTypes(this IApp app) =>
+			app.As<ILogsSupportedApp>().GetLogTypes();
+
+		public static IEnumerable<string> GetLogEntries(this IApp app, string logType) =>
+			app.As<ILogsSupportedApp>().GetLogEntries(logType);
 	}
 }

src/TestUtils/src/UITest.NUnit/UITestBase.cs

@@ -1,4 +1,5 @@
 using System.Reflection;
+using System.Runtime.CompilerServices;
 using NUnit.Framework;
 using NUnit.Framework.Interfaces;
 using UITest.Core;
@@ -55,21 +56,24 @@ protected virtual void FixtureTeardown()
 		[TearDown]
 		public void UITestBaseTearDown()
 		{
-
-			if (App.AppState == ApplicationState.Not_Running)
+			if (App.AppState == ApplicationState.NotRunning)
 			{
-				// Assert.Fail will immediately exit the test which is desirable as the app is not
-				// running anymore so we don't want to log diagnostic data as there is nothing to collect from
+				SaveAppLogs();
+
 				Reset();
 				FixtureSetup();
+
+				// Assert.Fail will immediately exit the test which is desirable as the app is not
+				// running anymore so we can't capture any UI structures or any screenshots
 				Assert.Fail("The app was expected to be running still, investigate as possible crash");
 			}
 
 			var testOutcome = TestContext.CurrentContext.Result.Outcome;
 			if (testOutcome == ResultState.Error ||
 				testOutcome == ResultState.Failure)
 			{
-				SaveDiagnosticLogs("UITestBaseTearDown");
+				SaveAppLogs();
+				SaveAppSnapshots();
 			}
 		}
 
@@ -84,7 +88,8 @@ public void OneTimeSetup()
 			}
 			catch
 			{
-				SaveDiagnosticLogs("FixtureSetup");
+				SaveAppLogs();
+				SaveAppSnapshots();
 				throw;
 			}
 		}
@@ -98,55 +103,87 @@ public void OneTimeTearDown()
 			if (outcome.Status == ResultState.SetUpFailure.Status &&
 				outcome.Site == ResultState.SetUpFailure.Site)
 			{
-				SaveDiagnosticLogs("OneTimeTearDown");
+				SaveAppLogs();
+				SaveAppSnapshots();
 			}
 
 			FixtureTeardown();
 		}
 
-		void SaveDiagnosticLogs(string? note = null)
+		void SaveAppLogs([CallerMemberName] string? note = null)
 		{
-			if (string.IsNullOrEmpty(note))
-				note = "-";
-			else
-				note = $"-{note}-";
-
-			var logDir = (Path.GetDirectoryName(Environment.GetEnvironmentVariable("APPIUM_LOG_FILE")) ?? Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location))!;
+			var types = App.GetLogTypes().ToArray();
+			TestContext.Progress.WriteLine($">>>>> {DateTime.Now} Log types: {string.Join(", ", types)}");
 
-			// App could be null if UITestContext was not able to connect to the test process (e.g. port already in use etc...)
-			if (UITestContext is not null)
+			foreach (var logType in new[] { "logcat" })
 			{
-				string name = TestContext.CurrentContext.Test.MethodName ?? TestContext.CurrentContext.Test.Name;
+				if (!types.Contains(logType, StringComparer.InvariantCultureIgnoreCase))
+					continue;
+
+				var logsPath = GetGeneratedFilePath($"AppLogs-{logType}.log", note);
+				if (logsPath is not null)
+				{
+					var entries = App.GetLogEntries(logType);
+					File.WriteAllLines(logsPath, entries);
+
+					AddTestAttachment(logsPath, Path.GetFileName(logsPath));
+				}
+			}
+		}
 
-				var screenshotPath = Path.Combine(logDir, $"{name}-{_testDevice}{note}ScreenShot");
+		void SaveAppSnapshots([CallerMemberName] string? note = null)
+		{
+			var screenshotPath = GetGeneratedFilePath("ScreenShot.png", note);
+			if (screenshotPath is not null)
+			{
 				_ = App.Screenshot(screenshotPath);
-				// App.Screenshot appends a ".png" extension always, so include that here
-				var screenshotPathWithExtension = screenshotPath + ".png";
-				AddTestAttachment(screenshotPathWithExtension, Path.GetFileName(screenshotPathWithExtension));
 
-				var pageSourcePath = Path.Combine(logDir, $"{name}-{_testDevice}{note}PageSource.txt");
+				AddTestAttachment(screenshotPath, Path.GetFileName(screenshotPath));
+			}
+
+			var pageSourcePath = GetGeneratedFilePath("PageSource.txt", note);
+			if (pageSourcePath is not null)
+			{
 				File.WriteAllText(pageSourcePath, App.ElementTree);
+
 				AddTestAttachment(pageSourcePath, Path.GetFileName(pageSourcePath));
 			}
 		}
 
+		string? GetGeneratedFilePath(string filename, string? note = null)
+		{
+			// App could be null if UITestContext was not able to connect to the test process (e.g. port already in use etc...)
+			if (UITestContext is null)
+				return null;
+
+			if (string.IsNullOrEmpty(note))
+				note = "-";
+			else
+				note = $"-{note}-";
+
+			filename = $"{Path.GetFileNameWithoutExtension(filename)}-{Guid.NewGuid().ToString("N")}{Path.GetExtension(filename)}";
+
+			var logDir =
+				Path.GetDirectoryName(Environment.GetEnvironmentVariable("APPIUM_LOG_FILE") ??
+				Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location))!;
+
+			var name =
+				TestContext.CurrentContext.Test.MethodName ??
+				TestContext.CurrentContext.Test.Name;
+
+			return Path.Combine(logDir, $"{name}-{_testDevice}{note}{filename}");
+		}
+
 		void AddTestAttachment(string filePath, string? description = null)
 		{
 			try
 			{
 				TestContext.AddTestAttachment(filePath, description);
 			}
-			catch (FileNotFoundException e)
+			catch (FileNotFoundException e) when (e.Message == "Test attachment file path could not be found.")
 			{
 				// Add the file path to better troubleshoot when these errors occur
-				if (e.Message == "Test attachment file path could not be found.")
-				{
-					throw new FileNotFoundException($"{e.Message}: {filePath}");
-				}
-				else
-				{
-					throw;
-				}
+				throw new FileNotFoundException($"Test attachment file path could not be found: '{filePath}' {description}", e);
 			}
 		}
 	}