Added ability to sort test collections.

Author: pawelpabich

src/xunit.core/Sdk/ITestCollectionOrderer.cs

@@ -0,0 +1,22 @@
+using System.Collections.Generic;
+using System.Linq;
+using Xunit.Abstractions;
+
+namespace Xunit.Sdk
+{
+    /// <summary>
+    /// A class implements this interface to participate in ordering test collections
+    /// for the test runner. Test collection orderers are applied using the
+    /// <see cref="TestCollectionOrdererAttribute"/>, which can be applied at
+    /// the assembly level.
+    /// </summary>
+    public interface ITestCollectionOrderer
+    {
+        /// <summary>
+        /// Orders test collections for execution.
+        /// </summary>
+        /// <param name="testCollections">The test collections to be ordered.</param>
+        /// <returns>The test collections in the order to be run.</returns>
+        IEnumerable<IGrouping<ITestCollection, TTestCase>> OrderTestCollections<TTestCase>(IEnumerable<IGrouping<ITestCollection, TTestCase>> testCollections) where TTestCase : ITestCase;
+    }
+}

src/xunit.core/TestCollectionOrdererAttribute.cs

@@ -0,0 +1,19 @@
+using System;
+
+namespace Xunit
+{
+    /// <summary>
+    /// Used to decorate an assembly to allow
+    /// the use a custom <see cref="T:Xunit.Sdk.ICollectionOrderer"/>.
+    /// </summary>
+    [AttributeUsage(AttributeTargets.Assembly, Inherited = true, AllowMultiple = false)]
+    public sealed class TestCollectionOrdererAttribute : Attribute
+    {
+        /// <summary>
+        /// Initializes a new instance of the <see cref="TestCollectionOrdererAttribute"/> class.
+        /// </summary>
+        /// <param name="ordererTypeName">The type name of the orderer class (that implements <see cref="T:Xunit.Sdk.ICollectionOrderer"/>).</param>
+        /// <param name="ordererAssemblyName">The assembly that <paramref name="ordererTypeName"/> exists in.</param>
+        public TestCollectionOrdererAttribute(string ordererTypeName, string ordererAssemblyName) { }
+    }
+}

src/xunit.core/xunit.core.csproj

@@ -56,6 +56,8 @@
     <Compile Include="CollectionBehavior.cs" />
     <Compile Include="CollectionBehaviorAttribute.cs" />
     <Compile Include="CollectionDefinitionAttribute.cs" />
+    <Compile Include="Sdk\ITestCollectionOrderer.cs" />
+    <Compile Include="TestCollectionOrdererAttribute.cs" />
     <Compile Include="FactAttribute.cs" />
     <Compile Include="IClassFixture.cs" />
     <Compile Include="ICollectionFixture.cs" />

src/xunit.execution/Sdk/ExtensibilityPointFactory.cs

@@ -78,6 +78,14 @@ public static ITestCaseOrderer GetTestCaseOrderer(Type ordererType)
             return Get<ITestCaseOrderer>(ordererType);
         }
 
+        /// <summary>
+        /// Gets a test collection orderer.
+        /// </summary>
+        public static ITestCollectionOrderer GetTestCollectionOrderer(Type ordererType)
+        {
+            return Get<ITestCollectionOrderer>(ordererType);
+        }
+
         /// <summary>
         /// Gets a test case orderer, as specified in a reflected <see cref="TestCaseOrdererAttribute"/>.
         /// </summary>
@@ -93,6 +101,17 @@ public static ITestCaseOrderer GetTestCaseOrderer(IAttributeInfo testCaseOrderer
             return GetTestCaseOrderer(ordererType);
         }
 
+
+        public static ITestCollectionOrderer GetTestCollectionOrderer(IAttributeInfo testCollectionOrdererAttribute)
+        {
+            var args = testCollectionOrdererAttribute.GetConstructorArguments().Cast<string>().ToList();
+            var ordererType = Reflector.GetType(args[1], args[0]);
+            if (ordererType == null)
+                return null;
+
+            return GetTestCollectionOrderer(ordererType);
+        }
+
         /// <summary>
         /// Gets a trait discoverer.
         /// </summary>

src/xunit.execution/Sdk/Frameworks/Runners/TestAssemblyRunner.cs

@@ -63,6 +63,11 @@ public TestAssemblyRunner(IAssemblyInfo assemblyInfo,
         /// </summary>
         protected ITestCaseOrderer TestCaseOrderer { get; set; }
 
+        /// <summary>
+        /// Gets or sets the test collection orderer that will be used to decide how to order test collections.
+        /// </summary>
+        protected ITestCollectionOrderer TestCollectionOrderer { get; set; }
+
         /// <summary>
         /// Gets or sets the test cases to be run.
         /// </summary>

src/xunit.execution/Sdk/Frameworks/Runners/XunitTestAssemblyRunner.cs

@@ -92,20 +92,25 @@ protected override void OnAssemblyStarting()
             scheduler = GetTaskScheduler(maxParallelThreads);
 
             var ordererAttribute = AssemblyInfo.GetCustomAttributes(typeof(TestCaseOrdererAttribute)).SingleOrDefault();
-            if (ordererAttribute != null)
-                TestCaseOrderer = ExtensibilityPointFactory.GetTestCaseOrderer(ordererAttribute);
+            if (ordererAttribute != null) TestCaseOrderer = ExtensibilityPointFactory.GetTestCaseOrderer(ordererAttribute);
+
+            var collectionOrdererAttribute = AssemblyInfo.GetCustomAttributes(typeof(TestCollectionOrdererAttribute)).SingleOrDefault();
+            if (collectionOrdererAttribute != null) TestCollectionOrderer = ExtensibilityPointFactory.GetTestCollectionOrderer(collectionOrdererAttribute);
         }
 
         /// <inheritdoc/>
         protected override async Task<RunSummary> RunTestCollectionsAsync(IMessageBus messageBus, CancellationTokenSource cancellationTokenSource)
         {
-            if (disableParallelization)
-                return await base.RunTestCollectionsAsync(messageBus, cancellationTokenSource);
-            var testCollections = TestCases.Cast<IXunitTestCase>()
-                                           .GroupBy(tc => tc.TestCollection, TestCollectionComparer.Instance)
-                                           .ToArray();
+            if (disableParallelization) return await base.RunTestCollectionsAsync(messageBus, cancellationTokenSource);
+            var testCollections = TestCases.GroupBy(tc => tc.TestCollection, TestCollectionComparer.Instance).ToArray();
+            if (TestCollectionOrderer != null)
+            {
+                testCollections = TestCollectionOrderer.OrderTestCollections(testCollections).ToArray();
+            }
 
+            var order = testCollections.Aggregate(String.Empty, (value, group) => value + Environment.NewLine + group.Key.DisplayName);                
             Console.WriteLine("Queueing {0} test collections.", testCollections.Length);
+            Console.WriteLine("Test collection will be queued in the following order: {0}", order);
 
             var tasks = testCollections.Select(collectionGroup => Task.Factory.StartNew(() => RunTestCollectionAsync(messageBus, collectionGroup.Key, collectionGroup, cancellationTokenSource),
                                                                                               cancellationTokenSource.Token,