Test for SocketException caused by --explore

src/NUnitConsole/nunit3-console.tests/ResultReporterTests.cs

@@ -95,7 +95,7 @@ public void SummaryReportTest()
             var expected = new [] {
                 "Test Run Summary",
                 "  Overall result: Failed",
-                "  Test Count: 29, Passed: 15, Failed: 5, Warnings: 1, Inconclusive: 1, Skipped: 7",
+               $"  Test Count: {MockAssembly.Tests}, Passed: {MockAssembly.Passed}, Failed: 5, Warnings: 1, Inconclusive: 1, Skipped: 7",
                 "    Failed Tests - Failures: 1, Errors: 1, Invalid: 3",
                 "    Skipped Tests - Ignored: 4, Explicit: 3, Other: 0",
                 "  Start time: 2015-10-19 02:12:28Z",

src/NUnitEngine/mock-assembly.netstandard/mock-assembly.netstandard.csproj

@@ -17,6 +17,7 @@
     <Compile Include="..\EngineVersion.cs" Link="Properties\EngineVersion.cs" />
     <Compile Include="..\mock-assembly\Properties\AssemblyInfo.cs" Link="Properties\AssemblyInfo.cs" />
     <Compile Include="..\mock-assembly\MockAssembly.cs" Link="MockAssembly.cs" />
+    <Compile Include="..\mock-assembly\AccessesCurrentTestContextDuringDiscovery.cs" Link="AccessesCurrentTestContextDuringDiscovery.cs" />
   </ItemGroup>
 
   <ItemGroup>

src/NUnitEngine/mock-assembly/AccessesCurrentTestContextDuringDiscovery.cs

@@ -0,0 +1,22 @@
+using NUnit.Framework;
+
+namespace NUnit.Tests
+{
+    public class AccessesCurrentTestContextDuringDiscovery
+    {
+        public const int Tests = 2;
+        public const int Suites = 1;
+
+        public static int[] TestCases()
+        {
+            var _ = TestContext.CurrentContext;
+            return new[] { 0 };
+        }
+
+        [TestCaseSource(nameof(TestCases))]
+        public void Access_by_TestCaseSource(int arg) { }
+
+        [Test]
+        public void Access_by_ValueSource([ValueSource(nameof(TestCases))] int arg) { }
+    }
+}

src/NUnitEngine/mock-assembly/MockAssembly.cs

@@ -56,7 +56,8 @@ public class MockAssembly
                         + BadFixture.Tests
                         + FixtureWithTestCases.Tests
                         + ParameterizedFixture.Tests
-                        + GenericFixtureConstants.Tests;
+                        + GenericFixtureConstants.Tests
+                        + AccessesCurrentTestContextDuringDiscovery.Tests;
 
             public const int Suites = MockTestFixture.Suites
                         + Singletons.OneTestCase.Suites
@@ -67,6 +68,7 @@ public class MockAssembly
                         + FixtureWithTestCases.Suites
                         + ParameterizedFixture.Suites
                         + GenericFixtureConstants.Suites
+                        + AccessesCurrentTestContextDuringDiscovery.Suites
                         + NamespaceSuites;
 
             public const int TestStartedEvents = Tests - IgnoredFixture.Tests - BadFixture.Tests - ExplicitFixture.Tests;
@@ -84,7 +86,8 @@ public class MockAssembly
                         + TestAssembly.MockTestFixture.Tests
                         + FixtureWithTestCases.Tests
                         + ParameterizedFixture.Tests
-                        + GenericFixtureConstants.Tests;
+                        + GenericFixtureConstants.Tests
+                        + AccessesCurrentTestContextDuringDiscovery.Tests;
 
             public const int Skipped_Ignored = MockTestFixture.Skipped_Ignored + IgnoredFixture.Tests;
             public const int Skipped_Explicit = MockTestFixture.Skipped_Explicit + ExplicitFixture.Tests;

src/NUnitEngine/mock-assembly/mock-assembly.csproj

@@ -95,6 +95,7 @@
       <SubType>Code</SubType>
     </Compile>
     <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="AccessesCurrentTestContextDuringDiscovery.cs" />
   </ItemGroup>
   <ItemGroup>
     <Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />

src/NUnitEngine/nunit.engine.tests/Helpers/On.cs

@@ -0,0 +1,21 @@
+using System;
+using System.Threading;
+
+namespace NUnit.Engine.Tests.Helpers
+{
+    public static class On
+    {
+        public static IDisposable Dispose(Action action) => new OnDisposeAction(action);
+        private sealed class OnDisposeAction : IDisposable
+        {
+            private Action action;
+
+            public OnDisposeAction(Action action)
+            {
+                this.action = action;
+            }
+
+            public void Dispose() => Interlocked.Exchange(ref action, null)?.Invoke();
+        }
+    }
+}

src/NUnitEngine/nunit.engine.tests/Helpers/ProcessUtils.cs

@@ -0,0 +1,116 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Text;
+
+namespace NUnit.Engine.Tests.Helpers
+{
+    public static class ProcessUtils
+    {
+        public static ProcessResult Run(ProcessStartInfo startInfo)
+        {
+            startInfo.UseShellExecute = false;
+            startInfo.RedirectStandardOutput = true;
+            startInfo.RedirectStandardError = true;
+            startInfo.CreateNoWindow = true;
+
+            using (var process = new Process { StartInfo = startInfo })
+            {
+                var standardStreamData = new List<StandardStreamData>();
+                var currentData = new StringBuilder();
+                var currentDataIsError = false;
+
+                process.OutputDataReceived += (sender, e) =>
+                {
+                    if (e.Data == null) return;
+                    if (currentDataIsError)
+                    {
+                        if (currentData.Length != 0)
+                            standardStreamData.Add(new StandardStreamData(currentDataIsError, currentData.ToString()));
+                        currentData = new StringBuilder();
+                        currentDataIsError = false;
+                    }
+                    currentData.AppendLine(e.Data);
+                };
+                process.ErrorDataReceived += (sender, e) =>
+                {
+                    if (e.Data == null) return;
+                    if (!currentDataIsError)
+                    {
+                        if (currentData.Length != 0)
+                            standardStreamData.Add(new StandardStreamData(currentDataIsError, currentData.ToString()));
+                        currentData = new StringBuilder();
+                        currentDataIsError = true;
+                    }
+                    currentData.AppendLine(e.Data);
+                };
+
+                process.Start();
+                process.BeginOutputReadLine();
+                process.BeginErrorReadLine();
+                process.WaitForExit();
+
+                if (currentData.Length != 0)
+                    standardStreamData.Add(new StandardStreamData(currentDataIsError, currentData.ToString()));
+
+                return new ProcessResult(process.ExitCode, standardStreamData.ToArray());
+            }
+        }
+
+        [DebuggerDisplay("{ToString(),nq}")]
+        public struct ProcessResult
+        {
+            public ProcessResult(int exitCode, StandardStreamData[] standardStreamData)
+            {
+                ExitCode = exitCode;
+                StandardStreamData = standardStreamData;
+            }
+
+            public int ExitCode { get; }
+            public StandardStreamData[] StandardStreamData { get; }
+
+            public override string ToString() => ToString(true);
+
+            /// <param name="showStreamSource">If true, appends "[stdout] " or "[stderr] " to the beginning of each line.</param>
+            public string ToString(bool showStreamSource)
+            {
+                var r = new StringBuilder("Exit code ").Append(ExitCode);
+
+                if (StandardStreamData.Length != 0) r.AppendLine();
+
+                foreach (var data in StandardStreamData)
+                {
+                    if (showStreamSource)
+                    {
+                        var lines = data.Data.Split(new[] { Environment.NewLine }, StringSplitOptions.None);
+
+                        // StandardStreamData.Data always ends with a blank line, so skip that
+                        for (var i = 0; i < lines.Length - 1; i++)
+                            r.Append(data.IsError ? "[stderr] " : "[stdout] ").AppendLine(lines[i]);
+                    }
+                    else
+                    {
+                        r.Append(data.Data);
+                    }
+                }
+
+                return r.ToString();
+            }
+        }
+
+        [DebuggerDisplay("{ToString(),nq}")]
+        public struct StandardStreamData
+        {
+            public StandardStreamData(bool isError, string data)
+            {
+                IsError = isError;
+                Data = data;
+            }
+
+            public bool IsError { get; }
+            public string Data { get; }
+
+            public override string ToString() => (IsError ? "[stderr] " : "[stdout] ") + Data;
+        }
+    }
+}

src/NUnitEngine/nunit.engine.tests/Helpers/ShadowCopyUtils.cs

@@ -0,0 +1,36 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+
+namespace NUnit.Engine.Tests.Helpers
+{
+    public static class ShadowCopyUtils
+    {
+        /// <summary>
+        /// Returns the transitive closure of assemblies needed to copy.
+        /// Deals with assembly names rather than paths to work with runners that shadow copy.
+        /// </summary>
+        public static ICollection<string> GetAllNeededAssemblyPaths(params string[] assemblyNames)
+        {
+            var r = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+
+            var dependencies = StackEnumerator.Create(
+                from assemblyName in assemblyNames
+                select new AssemblyName(assemblyName));
+
+            foreach (var dependencyName in dependencies)
+            {
+                var dependency = Assembly.ReflectionOnlyLoad(dependencyName.FullName);
+
+                if (!dependency.GlobalAssemblyCache && r.Add(Path.GetFullPath(dependency.Location)))
+                {
+                    dependencies.Recurse(dependency.GetReferencedAssemblies());
+                }
+            }
+
+            return r;
+        }
+    }
+}

src/NUnitEngine/nunit.engine.tests/Helpers/StackEnumerator.cs

@@ -0,0 +1,76 @@
+using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Linq;
+
+namespace NUnit.Engine.Tests.Helpers
+{
+    public static class StackEnumerator
+    {
+        public static StackEnumerator<T> Create<T>(params T[] initial) => new StackEnumerator<T>(initial);
+        public static StackEnumerator<T> Create<T>(IEnumerable<T> initial) => new StackEnumerator<T>(initial);
+        public static StackEnumerator<T> Create<T>(IEnumerator<T> initial) => new StackEnumerator<T>(initial);
+    }
+
+    public sealed class StackEnumerator<T> : IDisposable
+    {
+        private readonly Stack<IEnumerator<T>> stack = new Stack<IEnumerator<T>>();
+        private IEnumerator<T> current;
+
+        public bool MoveNext()
+        {
+            while (!current.MoveNext())
+            {
+                current.Dispose();
+                if (stack.Count == 0) return false;
+                current = stack.Pop();
+            }
+
+            return true;
+        }
+
+        public T Current => current.Current;
+
+        public void Recurse(IEnumerator<T> newCurrent)
+        {
+            if (newCurrent == null) return;
+            stack.Push(current);
+            current = newCurrent;
+        }
+        public void Recurse(IEnumerable<T> newCurrent)
+        {
+            if (newCurrent == null) return;
+            Recurse(newCurrent.GetEnumerator());
+        }
+        public void Recurse(params T[] newCurrent)
+        {
+            Recurse((IEnumerable<T>)newCurrent);
+        }
+
+        public StackEnumerator(IEnumerator<T> initial)
+        {
+            current = initial ?? Enumerable.Empty<T>().GetEnumerator();
+        }
+        public StackEnumerator(IEnumerable<T> initial) : this(initial?.GetEnumerator())
+        {
+        }
+        public StackEnumerator(params T[] initial) : this((IEnumerable<T>)initial)
+        {
+        }
+
+        // Foreach support
+        [EditorBrowsable(EditorBrowsableState.Never)]
+        public StackEnumerator<T> GetEnumerator()
+        {
+            return this;
+        }
+
+        public void Dispose()
+        {
+            current.Dispose();
+            foreach (var item in stack)
+                item.Dispose();
+            stack.Clear();
+        }
+    }
+}

src/NUnitEngine/nunit.engine.tests/Integration/DirectoryWithNeededAssemblies.cs

@@ -0,0 +1,31 @@
+using System;
+using System.IO;
+using NUnit.Engine.Tests.Helpers;
+
+namespace NUnit.Engine.Tests.Integration
+{
+    internal sealed class DirectoryWithNeededAssemblies : IDisposable
+    {
+        public string Directory { get; }
+
+        /// <summary>
+        /// Returns the transitive closure of assemblies needed to copy.
+        /// Deals with assembly names rather than paths to work with runners that shadow copy.
+        /// </summary>
+        public DirectoryWithNeededAssemblies(params string[] assemblyNames)
+        {
+            Directory = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
+            System.IO.Directory.CreateDirectory(Directory);
+
+            foreach (var neededAssembly in ShadowCopyUtils.GetAllNeededAssemblyPaths(assemblyNames))
+            {
+                File.Copy(neededAssembly, Path.Combine(Directory, Path.GetFileName(neededAssembly)));
+            }
+        }
+
+        public void Dispose()
+        {
+            System.IO.Directory.Delete(Directory, true);
+        }
+    }
+}

src/NUnitEngine/nunit.engine.tests/Integration/IntegrationTests.cs

@@ -0,0 +1,9 @@
+using NUnit.Framework;
+
+namespace NUnit.Engine.Tests.Integration
+{
+    [TestFixture, Category("Integration")]
+    public abstract class IntegrationTests
+    {
+    }
+}

src/NUnitEngine/nunit.engine.tests/Integration/MockAssemblyInDirectoryWithFramework.cs

@@ -0,0 +1,25 @@
+using System;
+using System.IO;
+using NUnit.Framework;
+
+namespace NUnit.Engine.Tests.Integration
+{
+    internal sealed class MockAssemblyInDirectoryWithFramework : IDisposable
+    {
+        private readonly DirectoryWithNeededAssemblies directory;
+
+        public string MockAssemblyDll => Path.Combine(directory.Directory, "mock-assembly.dll");
+
+        public MockAssemblyInDirectoryWithFramework()
+        {
+            directory = new DirectoryWithNeededAssemblies("mock-assembly");
+
+            Assert.That(Path.Combine(directory.Directory, "nunit.framework.dll"), Does.Exist, "This test must be run with nunit.framework.dll in the same directory as the mock assembly.");
+        }
+
+        public void Dispose()
+        {
+            directory.Dispose();
+        }
+    }
+}