diff --git a/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/RunInstrumentationTests.cs b/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/RunInstrumentationTests.cs index eb03232b392..8e63422fbe7 100644 --- a/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/RunInstrumentationTests.cs +++ b/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/RunInstrumentationTests.cs @@ -94,7 +94,7 @@ public override bool Execute () new CommandInfo { ArgumentsString = $"{AdbTarget} {AdbOptions} logcat -v threadtime -d", StdoutFilePath = LogcatFilename, - StdoutAppend = true, + StdoutAppend = false, }, new CommandInfo { diff --git a/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/RunUITests.cs b/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/RunUITests.cs index a1aa2c9b27c..0b4c4cbc54b 100644 --- a/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/RunUITests.cs +++ b/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/RunUITests.cs @@ -36,7 +36,7 @@ protected override void AfterCommand (int commandIndex, CommandInfo info) ArgumentsString = $"{AdbTarget} {AdbOptions} logcat -v threadtime -d", MergeStdoutAndStderr = false, StdoutFilePath = LogcatFilename, - StdoutAppend = true, + StdoutAppend = false, }, new CommandInfo { diff --git a/build-tools/scripts/TestApks.targets b/build-tools/scripts/TestApks.targets index 2a98197cabf..00182feb939 100644 --- a/build-tools/scripts/TestApks.targets +++ b/build-tools/scripts/TestApks.targets @@ -274,6 +274,8 @@ <_IncludeCategories Condition=" '$(IncludeCategories)' != '' ">include=$(IncludeCategories) <_ExcludeCategories Condition=" '$(ExcludeCategories)' != '' ">exclude=$(ExcludeCategories) + <_DryRunTests Condition=" '$(DryRunTests)' == 'true' ">dryrun=true + <_NoTestExclusions Condition=" '$(NoTestExclusions)' == 'true' ">noexclusions=true $(TestsFlavor) StringExtrasInBundle { get; set; } = new Dictionary (); protected string TestSuiteToRun { get; set; } + protected bool DryRunRequested { get; set; } + protected bool NoExclusionsRequested { get; set; } protected TestInstrumentation () {} @@ -104,6 +108,22 @@ protected virtual void ProcessArguments () if (StringExtrasInBundle.ContainsKey (KnownArguments.Suite)) { TestSuiteToRun = StringExtrasInBundle [KnownArguments.Suite]?.Trim (); } + + if (StringExtrasInBundle.TryGetValue (KnownArguments.DryRun, out string dryRunValue)) { + DryRunRequested = ParseBool (dryRunValue); + } + + if (StringExtrasInBundle.TryGetValue (KnownArguments.NoExclusions, out string noExclusionsValue)) { + NoExclusionsRequested = ParseBool (noExclusionsValue); + } + } + + static bool ParseBool (string value) + { + string trimmed = value?.Trim (); + return String.Equals (trimmed, "true", StringComparison.OrdinalIgnoreCase) || + String.Equals (trimmed, "1", StringComparison.OrdinalIgnoreCase) || + String.Equals (trimmed, "yes", StringComparison.OrdinalIgnoreCase); } public override void OnStart () @@ -259,8 +279,13 @@ bool RunTests (ref Bundle results) TRunner runner = CreateRunner (Logger, arguments); runner.LogTag = LogTag; + runner.DryRun = DryRunRequested; ConfigureFilters (runner); + if (runner.DryRun) { + Log.Info (LogTag, "Dry-run discovery mode enabled; tests will be enumerated but not executed."); + } + Log.Info (LogTag, "Starting unit tests"); runner.Run (assemblies); Log.Info (LogTag, "Unit tests completed"); diff --git a/tests/TestRunner.Core/TestRunner.cs b/tests/TestRunner.Core/TestRunner.cs index 886063a53bf..62d3c6904a4 100644 --- a/tests/TestRunner.Core/TestRunner.cs +++ b/tests/TestRunner.Core/TestRunner.cs @@ -19,6 +19,7 @@ public abstract class TestRunner public long ExecutedTests { get; protected set; } = 0; public long TotalTests { get; protected set; } = 0; public long FilteredTests { get; protected set; } = 0; + public bool DryRun { get; set; } = false; public bool RunInParallel { get; set; } = false; public string TestsRootDirectory { get; set; } public Context Context { get; } diff --git a/tests/TestRunner.NUnit/NUnitTestInstrumentation.cs b/tests/TestRunner.NUnit/NUnitTestInstrumentation.cs index 6ce32f987ca..15674ad1b01 100644 --- a/tests/TestRunner.NUnit/NUnitTestInstrumentation.cs +++ b/tests/TestRunner.NUnit/NUnitTestInstrumentation.cs @@ -17,6 +17,8 @@ public abstract class NUnitTestInstrumentation : TestInstrumentation IncludedCategories { get; set; } protected IEnumerable ExcludedCategories { get; set; } protected IEnumerable ExcludedTestNames { get; set; } + protected IDictionary ExcludedCategoryReasons { get; set; } + protected IDictionary ExcludedTestReasons { get; set; } protected string TestsDirectory { get; set; } protected NUnitTestInstrumentation () @@ -67,14 +69,22 @@ protected override void ConfigureFilters (NUnitTestRunner runner) Log.Info (LogTag, "Configuring test categories to include from extras:"); ChainCategoryFilter (GetFilterValuesFromExtras (KnownArguments.Include), false, ref filter); - Log.Info (LogTag, "Configuring test categories to exclude:"); - ChainCategoryFilter (ExcludedCategories, true, ref filter); + if (NoExclusionsRequested) { + Log.Info (LogTag, "Skipping built-in test exclusions due to noexclusions=true."); + } else { + Log.Info (LogTag, "Configuring test categories to exclude:"); + RegisterExcludedCategories (runner, ExcludedCategories); + } Log.Info(LogTag, "Configuring test categories to exclude from extras:"); - ChainCategoryFilter (GetFilterValuesFromExtras (KnownArguments.Exclude), true, ref filter); + RegisterExcludedCategories (runner, GetFilterValuesFromExtras (KnownArguments.Exclude)); - Log.Info (LogTag, "Configuring tests to exclude (by name):"); - ChainTestNameFilter (ExcludedTestNames?.ToArray (), ref filter); + if (NoExclusionsRequested) { + Log.Info (LogTag, "Skipping built-in test-name exclusions due to noexclusions=true."); + } else { + Log.Info (LogTag, "Configuring tests to exclude (by name):"); + RegisterExcludedTestNames (runner, ExcludedTestNames?.ToArray ()); + } if (filter.IsEmpty) return; @@ -104,7 +114,31 @@ void ChainCategoryFilter (IEnumerable categories, bool negate, ref ITes Log.Info (LogTag, " none"); } - void ChainTestNameFilter (string[] testNames, ref ITestFilter filter) + void RegisterExcludedCategories (NUnitTestRunner runner, IEnumerable categories) + { + bool gotCategories = false; + if (categories != null) { + foreach (string c in categories) { + Log.Info (LogTag, $" {c}"); + runner.AddExcludedCategory (c, GetExcludedCategoryReason (c)); + gotCategories = true; + } + } + + if (!gotCategories) + Log.Info (LogTag, " none"); + } + + string GetExcludedCategoryReason (string category) + { + if (ExcludedCategoryReasons != null && !String.IsNullOrEmpty (category) && ExcludedCategoryReasons.TryGetValue (category, out string reason)) { + return reason; + } + + return $"Excluded category '{category}'."; + } + + void RegisterExcludedTestNames (NUnitTestRunner runner, string[] testNames) { if (testNames == null || testNames.Length == 0) { Log.Info (LogTag, " none"); @@ -115,10 +149,17 @@ void ChainTestNameFilter (string[] testNames, ref ITestFilter filter) if (String.IsNullOrEmpty (name)) continue; Log.Info (LogTag, $" {name}"); + runner.AddExcludedTestName (name, GetExcludedTestReason (name)); + } + } + + string GetExcludedTestReason (string testName) + { + if (ExcludedTestReasons != null && !String.IsNullOrEmpty (testName) && ExcludedTestReasons.TryGetValue (testName, out string reason)) { + return reason; } - var excludeTestNamesFilter = new SimpleNameFilter (testNames); - filter = new AndFilter (filter, new NotFilter (excludeTestNamesFilter)); + return $"Excluded test '{testName}'."; } } } diff --git a/tests/TestRunner.NUnit/NUnitTestRunner.cs b/tests/TestRunner.NUnit/NUnitTestRunner.cs index 3da3ea4e07e..de8f835a73b 100644 --- a/tests/TestRunner.NUnit/NUnitTestRunner.cs +++ b/tests/TestRunner.NUnit/NUnitTestRunner.cs @@ -20,8 +20,12 @@ namespace Xamarin.Android.UnitTests.NUnit { public class NUnitTestRunner : TestRunner, ITestListener { + const string DryRunSkipReason = "Dry run: discovery only."; + Dictionary builderSettings; TestSuiteResult results; + readonly Dictionary excludedCategories = new Dictionary (StringComparer.OrdinalIgnoreCase); + readonly Dictionary excludedTestNames = new Dictionary (StringComparer.Ordinal); public ITestFilter Filter { get; set; } = TestFilter.Empty; public bool GCAfterEachFixture { get; set; } @@ -33,6 +37,24 @@ public NUnitTestRunner (Context context, LogWriter logger, Bundle bundle) : base builderSettings = new Dictionary (StringComparer.OrdinalIgnoreCase); } + public void AddExcludedCategory (string category, string reason) + { + if (String.IsNullOrEmpty (category)) { + return; + } + + excludedCategories [category] = reason; + } + + public void AddExcludedTestName (string testName, string reason) + { + if (String.IsNullOrEmpty (testName)) { + return; + } + + excludedTestNames [testName] = reason; + } + public override void Run (IList testAssemblies) { if (testAssemblies == null) @@ -51,8 +73,14 @@ public override void Run (IList testAssemblies) OnWarning ($"Failed to load tests from assembly '{assemblyInfo.Assembly}"); continue; } - if (runner.LoadedTest is NUnitTest tests) + if (runner.LoadedTest is NUnitTest tests) { testSuite.Add (tests); + ApplyIgnoredExclusions (tests); + UpdateDiscoveredTestCounts (tests); + if (DryRun) { + ApplyDryRunToMatchingTests (tests); + } + } // Messy API. .Run returns ITestResult which is, in reality, an instance of TestResult since that's // what WorkItem returns and we need an instance of TestResult to add it to TestSuiteResult. So, cast @@ -72,11 +100,139 @@ public override void Run (IList testAssemblies) if (testResult == null) throw new InvalidOperationException ($"Unexpected test result type '{result.GetType ()}'"); results.AddResult (testResult); + UpdateSummaryCounts (); } LogFailureSummary (); } + void UpdateSummaryCounts () + { + if (results == null) { + return; + } + + PassedTests = results.PassCount; + FailedTests = results.FailCount; + SkippedTests = results.SkipCount; + InconclusiveTests = results.InconclusiveCount; + ExecutedTests = PassedTests + FailedTests + SkippedTests + InconclusiveTests; + } + + void ApplyIgnoredExclusions (NUnitTest test) + { + if (test.RunState != RunState.Runnable && test.RunState != RunState.Explicit) { + return; + } + + if (!TryGetSkipReason (test, out string reason)) { + if (test is TestSuite suite) { + foreach (NUnitTest child in suite.Tests) { + ApplyIgnoredExclusions (child); + } + } + return; + } + + test.RunState = RunState.Ignored; + test.Properties.Set (PropertyNames.SkipReason, reason); + } + + void UpdateDiscoveredTestCounts (NUnitTest test) + { + if (test is TestSuite suite) { + foreach (NUnitTest child in suite.Tests) { + UpdateDiscoveredTestCounts (child); + } + return; + } + + TotalTests++; + if (Filter == null || Filter.IsEmpty || Filter.Pass (test)) { + FilteredTests++; + } + } + + void ApplyDryRunToMatchingTests (NUnitTest test) + { + if (test is TestSuite suite) { + foreach (NUnitTest child in suite.Tests) { + ApplyDryRunToMatchingTests (child); + } + return; + } + + if (Filter != null && !Filter.IsEmpty && !Filter.Pass (test)) { + return; + } + + if (test.RunState == RunState.Runnable || test.RunState == RunState.Explicit) { + test.RunState = RunState.Ignored; + test.Properties.Set (PropertyNames.SkipReason, DryRunSkipReason); + } + + string reason = test.Properties.Get (PropertyNames.SkipReason) as string; + if (String.IsNullOrEmpty (reason)) { + Logger.OnInfo (LogTag, $"[DRY-RUN] {test.FullName}"); + } else { + Logger.OnInfo (LogTag, $"[DRY-RUN] {test.FullName} [{reason}]"); + } + } + + bool TryGetSkipReason (NUnitTest test, out string reason) + { + if (TryGetNamedSkipReason (test, out reason)) { + return true; + } + + return TryGetCategorySkipReason (test, out reason); + } + + bool TryGetNamedSkipReason (NUnitTest test, out string reason) + { + foreach (var kvp in excludedTestNames) { + if (TestNameMatches (test.FullName, kvp.Key)) { + reason = kvp.Value; + return true; + } + } + + reason = String.Empty; + return false; + } + + static bool TestNameMatches (string fullName, string excludedName) + { + if (String.IsNullOrEmpty (fullName) || String.IsNullOrEmpty (excludedName)) { + return false; + } + + if (fullName == excludedName || + fullName.StartsWith (excludedName + ".", StringComparison.Ordinal) || + fullName.StartsWith (excludedName + "+", StringComparison.Ordinal) || + fullName.Contains ("." + excludedName + ".", StringComparison.Ordinal) || + fullName.Contains ("." + excludedName + "+", StringComparison.Ordinal) || + fullName.Contains (", " + excludedName, StringComparison.Ordinal)) { + return true; + } + + return false; + } + + bool TryGetCategorySkipReason (NUnitTest test, out string reason) + { + if (test.Properties [PropertyNames.Category] is IList categories) { + foreach (object value in categories) { + if (value is string category && excludedCategories.TryGetValue (category, out reason)) { + return true; + } + } + } + + reason = String.Empty; + return false; + } + public bool Pass (ITest test) { return true; @@ -101,10 +257,8 @@ public void TestFinished (ITestResult result) Action log = Logger.OnInfo; StringBuilder failedMessage = null; - ExecutedTests++; if (result.ResultState.Status == TestStatus.Passed) { Logger.OnInfo (LogTag, $"\t{result.ResultState.ToString ()}"); - PassedTests++; } else if (result.ResultState.Status == TestStatus.Failed) { Logger.OnError (LogTag, "\t[FAIL]"); log = Logger.OnError; @@ -113,24 +267,12 @@ public void TestFinished (ITestResult result) if (result.Test.FixtureType != null) failedMessage.Append ($" ({result.Test.FixtureType.Assembly.GetName ().Name})"); failedMessage.AppendLine (); - FailedTests++; } else { - string status; - switch (result.ResultState.Status) { - case TestStatus.Skipped: - SkippedTests++; - status = "SKIPPED"; - break; - - case TestStatus.Inconclusive: - InconclusiveTests++; - status = "INCONCLUSIVE"; - break; - - default: - status = "UNKNOWN"; - break; - } + string status = result.ResultState.Status switch { + TestStatus.Skipped => "SKIPPED", + TestStatus.Inconclusive => "INCONCLUSIVE", + _ => "UNKNOWN", + }; Logger.OnInfo (LogTag, $"\t[{status}]"); }