[tests] Improve NUnit runner reporting and dry-run auditing

build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/RunInstrumentationTests.cs

@@ -94,7 +94,7 @@ public override bool Execute ()
 				new CommandInfo {
 					ArgumentsString = $"{AdbTarget} {AdbOptions} logcat -v threadtime -d",
 					StdoutFilePath = LogcatFilename,
-					StdoutAppend = true,
+					StdoutAppend = false,
 				},
 
 				new CommandInfo {

build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/RunUITests.cs

@@ -36,7 +36,7 @@ protected override void AfterCommand (int commandIndex, CommandInfo info)
 					ArgumentsString = $"{AdbTarget} {AdbOptions} logcat -v threadtime -d",
 					MergeStdoutAndStderr = false,
 					StdoutFilePath = LogcatFilename,
-					StdoutAppend = true,
+					StdoutAppend = false,
 				},
 
 				new CommandInfo {

build-tools/scripts/TestApks.targets

@@ -274,6 +274,8 @@
     <PropertyGroup>
       <_IncludeCategories Condition=" '$(IncludeCategories)' != '' ">include=$(IncludeCategories)</_IncludeCategories>
       <_ExcludeCategories Condition=" '$(ExcludeCategories)' != '' ">exclude=$(ExcludeCategories)</_ExcludeCategories>
+      <_DryRunTests Condition=" '$(DryRunTests)' == 'true' ">dryrun=true</_DryRunTests>
+      <_NoTestExclusions Condition=" '$(NoTestExclusions)' == 'true' ">noexclusions=true</_NoTestExclusions>
       <PlotDataLabelSuffix Condition=" '$(PlotDataLabelSuffix)' == '' ">$(TestsFlavor)</PlotDataLabelSuffix>
     </PropertyGroup>
     <RunInstrumentationTests
@@ -287,7 +289,7 @@
         Component="%(TestApkInstrumentation.Package)/%(TestApkInstrumentation.Identity)"
         NUnit2TestResultsFile="%(TestApkInstrumentation.ResultsPath)"
         LogcatFilename="$(_LogcatFilenameBase)-%(TestApkInstrumentation.Package)%(TestApkInstrumentation.LogcatFilenameDistincion).txt"
-        InstrumentationArguments="$(_IncludeCategories);$(_ExcludeCategories)"
+        InstrumentationArguments="$(_IncludeCategories);$(_ExcludeCategories);$(_DryRunTests);$(_NoTestExclusions)"
         TestFixture="$(TestFixture)"
         ToolExe="$(AdbToolExe)"
         ToolPath="$(AdbToolPath)"

tests/TestRunner.Core/TestInstrumentation.cs

@@ -19,6 +19,8 @@ protected sealed class KnownArguments
 			public const string Suite = "suite";
 			public const string Include = "include";
 			public const string Exclude = "exclude";
+			public const string DryRun = "dryrun";
+			public const string NoExclusions = "noexclusions";
 		}
 
 		const string ResultExecutedTests = "run";
@@ -40,6 +42,8 @@ protected sealed class KnownArguments
 		protected LogWriter Logger { get; } = new LogWriter ();
 		protected Dictionary<string, string> StringExtrasInBundle { get; set; } = new Dictionary<string, string> ();
 		protected string TestSuiteToRun { get; set; }
+		protected bool DryRunRequested { get; set; }
+		protected bool NoExclusionsRequested { get; set; }
 
 		protected TestInstrumentation ()
 		{}
@@ -104,6 +108,22 @@ protected virtual void ProcessArguments ()
 			if (StringExtrasInBundle.ContainsKey (KnownArguments.Suite)) {
 				TestSuiteToRun = StringExtrasInBundle [KnownArguments.Suite]?.Trim ();
 			}
+
+			if (StringExtrasInBundle.TryGetValue (KnownArguments.DryRun, out string dryRunValue)) {
+				DryRunRequested = ParseBool (dryRunValue);
+			}
+
+			if (StringExtrasInBundle.TryGetValue (KnownArguments.NoExclusions, out string noExclusionsValue)) {
+				NoExclusionsRequested = ParseBool (noExclusionsValue);
+			}
+		}
+
+		static bool ParseBool (string value)
+		{
+			string trimmed = value?.Trim ();
+			return String.Equals (trimmed, "true", StringComparison.OrdinalIgnoreCase) ||
+				String.Equals (trimmed, "1", StringComparison.OrdinalIgnoreCase) ||
+				String.Equals (trimmed, "yes", StringComparison.OrdinalIgnoreCase);
 		}
 
 		public override void OnStart ()
@@ -259,8 +279,13 @@ bool RunTests (ref Bundle results)
 
 			TRunner runner = CreateRunner (Logger, arguments);
 			runner.LogTag = LogTag;
+			runner.DryRun = DryRunRequested;
 			ConfigureFilters (runner);
 
+			if (runner.DryRun) {
+				Log.Info (LogTag, "Dry-run discovery mode enabled; tests will be enumerated but not executed.");
+			}
+
 			Log.Info (LogTag, "Starting unit tests");
 			runner.Run (assemblies);
 			Log.Info (LogTag, "Unit tests completed");

tests/TestRunner.Core/TestRunner.cs

@@ -19,6 +19,7 @@ public abstract class TestRunner
 		public long ExecutedTests { get; protected set; } = 0;
 		public long TotalTests { get; protected set; } = 0;
 		public long FilteredTests { get; protected set; } = 0;
+		public bool DryRun { get; set; } = false;
 		public bool RunInParallel { get; set; } = false;
 		public string TestsRootDirectory { get; set; }
 		public Context Context { get; }

tests/TestRunner.NUnit/NUnitTestInstrumentation.cs

@@ -17,6 +17,8 @@ public abstract class NUnitTestInstrumentation : TestInstrumentation <NUnitTestR
 		protected IEnumerable<string> IncludedCategories { get; set; }
 		protected IEnumerable<string> ExcludedCategories { get; set; }
 		protected IEnumerable<string> ExcludedTestNames { get; set; }
+		protected IDictionary<string, string> ExcludedCategoryReasons { get; set; }
+		protected IDictionary<string, string> ExcludedTestReasons { get; set; }
 		protected string TestsDirectory { get; set; }
 
 		protected NUnitTestInstrumentation ()
@@ -67,14 +69,22 @@ protected override void ConfigureFilters (NUnitTestRunner runner)
 			Log.Info (LogTag, "Configuring test categories to include from extras:");
 			ChainCategoryFilter (GetFilterValuesFromExtras (KnownArguments.Include), false, ref filter);
 
-			Log.Info (LogTag, "Configuring test categories to exclude:");
-			ChainCategoryFilter (ExcludedCategories, true, ref filter);
+			if (NoExclusionsRequested) {
+				Log.Info (LogTag, "Skipping built-in test exclusions due to noexclusions=true.");
+			} else {
+				Log.Info (LogTag, "Configuring test categories to exclude:");
+				RegisterExcludedCategories (runner, ExcludedCategories);
+			}
 
 			Log.Info(LogTag, "Configuring test categories to exclude from extras:");
-			ChainCategoryFilter (GetFilterValuesFromExtras (KnownArguments.Exclude), true, ref filter);
+			RegisterExcludedCategories (runner, GetFilterValuesFromExtras (KnownArguments.Exclude));
 
-			Log.Info (LogTag, "Configuring tests to exclude (by name):");
-			ChainTestNameFilter (ExcludedTestNames?.ToArray (), ref filter);
+			if (NoExclusionsRequested) {
+				Log.Info (LogTag, "Skipping built-in test-name exclusions due to noexclusions=true.");
+			} else {
+				Log.Info (LogTag, "Configuring tests to exclude (by name):");
+				RegisterExcludedTestNames (runner, ExcludedTestNames?.ToArray ());
+			}
 
 			if (filter.IsEmpty)
 				return;
@@ -104,7 +114,31 @@ void ChainCategoryFilter (IEnumerable <string> categories, bool negate, ref ITes
 				Log.Info (LogTag, "  none");
 		}
 
-		void ChainTestNameFilter (string[] testNames, ref ITestFilter filter)
+		void RegisterExcludedCategories (NUnitTestRunner runner, IEnumerable<string> categories)
+		{
+			bool gotCategories = false;
+			if (categories != null) {
+				foreach (string c in categories) {
+					Log.Info (LogTag, $"  {c}");
+					runner.AddExcludedCategory (c, GetExcludedCategoryReason (c));
+					gotCategories = true;
+				}
+			}
+
+			if (!gotCategories)
+				Log.Info (LogTag, "  none");
+		}
+
+		string GetExcludedCategoryReason (string category)
+		{
+			if (ExcludedCategoryReasons != null && !String.IsNullOrEmpty (category) && ExcludedCategoryReasons.TryGetValue (category, out string reason)) {
+				return reason;
+			}
+
+			return $"Excluded category '{category}'.";
+		}
+
+		void RegisterExcludedTestNames (NUnitTestRunner runner, string[] testNames)
 		{
 			if (testNames == null || testNames.Length == 0) {
 				Log.Info (LogTag, "  none");
@@ -115,10 +149,17 @@ void ChainTestNameFilter (string[] testNames, ref ITestFilter filter)
 				if (String.IsNullOrEmpty (name))
 					continue;
 				Log.Info (LogTag, $"  {name}");
+				runner.AddExcludedTestName (name, GetExcludedTestReason (name));
+			}
+		}
+
+		string GetExcludedTestReason (string testName)
+		{
+			if (ExcludedTestReasons != null && !String.IsNullOrEmpty (testName) && ExcludedTestReasons.TryGetValue (testName, out string reason)) {
+				return reason;
 			}
 
-			var excludeTestNamesFilter  = new SimpleNameFilter (testNames);
-			filter = new AndFilter (filter, new NotFilter (excludeTestNamesFilter));
+			return $"Excluded test '{testName}'.";
 		}
 	}
 }