Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Enabling Surefire to run test classes and test methods in any order specified by a new runOrder #348

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Enabling Surefire to run test classes and test methods in any order specified by a new runOrder #348

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
35 changes: 35 additions & 0 deletions surefire-api/src/main/java/org/apache/maven/surefire/api/testset/TestListResolver.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,9 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;

import static java.util.Collections.unmodifiableSet;
Expand Down Expand Up @@ -61,6 +63,8 @@ public class TestListResolver

private final boolean hasExcludedMethodPatterns;

private final Map<String, Integer> patternMapper = new HashMap<String, Integer>();

public TestListResolver( Collection<String> tests )
{
final IncludedExcludedPatterns patterns = new IncludedExcludedPatterns();
Expand Down Expand Up @@ -208,6 +212,7 @@ public boolean shouldRun( String testClassFile, String methodName )
else
{
boolean shouldRun = false;
ResolvedTest matchedFilter = null;

if ( getIncludedPatterns().isEmpty() )
{
Expand All @@ -220,6 +225,7 @@ public boolean shouldRun( String testClassFile, String methodName )
if ( filter.matchAsInclusive( testClassFile, methodName ) )
{
shouldRun = true;
matchedFilter = filter;
break;
}
}
Expand All @@ -236,6 +242,15 @@ public boolean shouldRun( String testClassFile, String methodName )
}
}
}

if ( shouldRun )
{
String test = testClassFile + "#" + methodName;
if ( ! this.patternMapper.containsKey( test ) )
{
this.patternMapper.put( test, new ArrayList<>( this.includedPatterns ).indexOf( matchedFilter ) );
}
}
return shouldRun;
}
}
Expand Down Expand Up @@ -514,4 +529,24 @@ private static boolean haveMethodPatterns( Set<ResolvedTest> patterns )
}
return false;
}

public Integer testOrderComparator( String className1, String className2, String methodName1, String methodName2 )
{
String classFileName1 = toClassFileName( className1 );
String classFileName2 = toClassFileName( className2 );
boolean shouldRunMethodName1 = shouldRun( classFileName1 , methodName1 );
boolean shouldRunMethodName2 = shouldRun( classFileName2 , methodName2 );
if ( ! shouldRunMethodName1 )
{
return -1;
}
if ( ! shouldRunMethodName2 )
{
return 1;
}

String test1 = classFileName1 + "#" + methodName1;
String test2 = classFileName2 + "#" + methodName2;
return patternMapper.get( test1 ) - patternMapper.get( test2 );
}
}
85 changes: 84 additions & 1 deletion surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,10 @@

import org.apache.maven.surefire.api.runorder.RunEntryStatisticsMap;
import org.apache.maven.surefire.api.testset.RunOrderParameters;
import org.apache.maven.surefire.api.testset.TestListResolver;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collections;
import java.util.Comparator;
Expand All @@ -48,6 +50,8 @@ public class DefaultRunOrderCalculator

private final Random random;

private final TestListResolver testListResolver;

public DefaultRunOrderCalculator( RunOrderParameters runOrderParameters, int threadCount )
{
this.runOrderParameters = runOrderParameters;
Expand All @@ -61,6 +65,14 @@ public DefaultRunOrderCalculator( RunOrderParameters runOrderParameters, int thr
runOrderParameters.setRunOrderRandomSeed( runOrderRandomSeed );
}
this.random = new Random( runOrderRandomSeed );
if ( RunOrder.TESTORDER.equals( getRunOrderMethod() ) )
{
this.testListResolver = getTestListResolver();
}
else
{
this.testListResolver = null;
}
}

@Override
Expand All @@ -78,9 +90,80 @@ public TestsToRun orderTestClasses( TestsToRun scannedClasses )
return new TestsToRun( new LinkedHashSet<>( result ) );
}

@Override
public Comparator<String> comparatorForTestMethods()
{
RunOrder methodRunOrder = getRunOrderMethod();
if ( RunOrder.TESTORDER.equals( methodRunOrder ) )
{
return new Comparator<String>()
{
@Override
public int compare( String o1, String o2 )
{
String[] classAndMethod1 = getClassAndMethod( o1 );
String className1 = classAndMethod1[0];
String methodName1 = classAndMethod1[1];
String[] classAndMethod2 = getClassAndMethod( o2 );
String className2 = classAndMethod2[0];
String methodName2 = classAndMethod2[1];
return testListResolver.testOrderComparator( className1, className2, methodName1, methodName2 );
}
};
}
else
{
return null;
}
}

public TestListResolver getTestListResolver()
{
String orderParam = System.getProperty( "test" );
if ( orderParam == null )
{
throw new IllegalStateException( "TestListResolver in RunOrderCalculator should be used only when "
+ "system property -Dtest is set and runOrder is testorder" );
}
return new TestListResolver( Arrays.asList( orderParam.split( "," ) ) );
}

public String[] getClassAndMethod( String request )
{
String[] classAndMethod = { request, request };
if ( request.contains( "(" ) )
{
String[] nameSplit1 = request.split( "\\(" );
classAndMethod[0] = nameSplit1[1].substring( 0, nameSplit1[1].length() - 1 );
classAndMethod[1] = nameSplit1[0];
}
return classAndMethod;
}

private RunOrder getRunOrderMethod()
{
if ( runOrder.length > 1 && Arrays.asList( runOrder ).contains( RunOrder.TESTORDER ) )
{
// Use of testorder and other runOrders are currently not supported
throw new IllegalStateException( "Expected only testorder. Got: " + runOrder.length );
}
return runOrder[0];
}

private void orderTestClasses( List<Class<?>> testClasses, RunOrder runOrder )
{
if ( RunOrder.RANDOM.equals( runOrder ) )
if ( RunOrder.TESTORDER.equals( runOrder ) )
{
Collections.sort( testClasses, new Comparator<Class<?>>()
{
@Override
public int compare( Class<?> o1, Class<?> o2 )
{
return testListResolver.testOrderComparator( o1.getName(), o2.getName(), null, null );
}
} );
}
else if ( RunOrder.RANDOM.equals( runOrder ) )
{
Collections.shuffle( testClasses, random );
}
Expand Down
5 changes: 4 additions & 1 deletion surefire-api/src/main/java/org/apache/maven/surefire/api/util/RunOrder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ public class RunOrder

public static final RunOrder FAILEDFIRST = new RunOrder( "failedfirst" );

public static final RunOrder TESTORDER = new RunOrder( "testorder" );

public static final RunOrder[] DEFAULT = new RunOrder[]{ FILESYSTEM };

/**
Expand Down Expand Up @@ -108,7 +110,8 @@ private static String createMessageForMissingRunOrder( String name )

private static RunOrder[] values()
{
return new RunOrder[]{ ALPHABETICAL, FILESYSTEM, HOURLY, RANDOM, REVERSE_ALPHABETICAL, BALANCED, FAILEDFIRST };
return new RunOrder[]{ ALPHABETICAL, FILESYSTEM, HOURLY, RANDOM, REVERSE_ALPHABETICAL, BALANCED, FAILEDFIRST,
TESTORDER };
}

public static String asString( RunOrder[] runOrder )
Expand Down
4 changes: 4 additions & 0 deletions surefire-api/src/main/java/org/apache/maven/surefire/api/util/RunOrderCalculator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,14 @@
* under the License.
*/

import java.util.Comparator;

/**
* @author Kristian Rosenvold
*/
public interface RunOrderCalculator
{
TestsToRun orderTestClasses( TestsToRun scannedClasses );

Comparator<String> comparatorForTestMethods();
}
65 changes: 65 additions & 0 deletions surefire-api/src/test/java/org/apache/maven/surefire/api/testset/TestListResolverTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;

import static java.util.Collections.addAll;
Expand Down Expand Up @@ -502,4 +503,68 @@ private static Set<ResolvedTest> resolveClass( String patterns )
}
return resolved;
}

public void testOrderComparatorTest()
{
List<String> orderParamList = new ArrayList<String>();
orderParamList.add( "TestClass1#testa2d" );
orderParamList.add( "TestClass1#testabc" );
orderParamList.add( "TestClass1#testa1b" );
orderParamList.add( "TestClass2#testa1b" );
orderParamList.add( "TestClass2#testaBc" );
TestListResolver tlr = new TestListResolver( orderParamList );
String className = "TestClass1";
String className2 = "TestClass2";
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testa2d", "testa1b" ) < 0 );
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testa2d", "testabc" ) < 0 );
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testa1b", "testabc" ) > 0 );
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testa2d", "testaBc" ) > 0 );
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testa3d", "testa1b" ) < 0 );
assertTrue( ( int ) tlr.testOrderComparator( className, className2, "testa2d", "testa1b" ) < 0 );
assertTrue( ( int ) tlr.testOrderComparator( className2, className, "testaBc", "testa1b" ) > 0 );
assertTrue( ( int ) tlr.testOrderComparator( className, className2, "testa3d", "testa1b" ) < 0 );
assertTrue( ( int ) tlr.testOrderComparator( className, className2, "testa2d", "testabc" ) > 0 );
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testa2d", "testa2d" ) == 0 );
}

public void testRegexMethodOrderComparator()
{
List<String> orderParamList = new ArrayList<String>();
orderParamList.add( "TestClass1#testa?c" );
orderParamList.add( "TestClass1#testa?b" );
orderParamList.add( "TestClass2#test?1*" );
orderParamList.add( "!TestClass1#testa4b" );
orderParamList.add( "!TestClass2#test11MyTest" );
TestListResolver tlr = new TestListResolver( orderParamList );
String className = "TestClass1";
String className2 = "TestClass2";
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testabc", "testa1b" ) < 0 );
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testaBc", "testa2b" ) < 0 );
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testa1b", "testa3c" ) > 0 );
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testa1b", "testa4b" ) > 0 );
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testa4b", "testabc" ) < 0 );
assertTrue( ( int ) tlr.testOrderComparator( className, className2, "testa1b", "test1123" ) < 0 );
assertTrue( ( int ) tlr.testOrderComparator( className2, className, "testa1b", "testa1b" ) > 0 );
assertTrue( ( int ) tlr.testOrderComparator( className2, className2, "testa1b", "test1123" ) == 0 );
assertTrue( ( int ) tlr.testOrderComparator( className2, className2, "test1123", "test11MyTest" ) > 0 );
assertTrue( ( int ) tlr.testOrderComparator( className2, className2, "test11MyTest", "test456" ) < 0 );
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testa1c", "testa1c" ) == 0 );
}

public void testRegexClassOrderComparator()
{
List<String> orderParamList = new ArrayList<String>();
orderParamList.add( "My2*Test.java" );
orderParamList.add( "???My1*Test" );
orderParamList.add( "!abcMy1PeaceTest" );
TestListResolver tlr = new TestListResolver( orderParamList );
String className = "My2ConnectTest";
String className2 = "456My1ConnectTest";
String className3 = "abcMy1PeaceTest";
assertTrue( ( int ) tlr.testOrderComparator( className, className2, null, null ) < 0 );
assertTrue( ( int ) tlr.testOrderComparator( className2, className, null, null ) > 0 );
assertTrue( ( int ) tlr.testOrderComparator( className3, className2, null, null ) < 0 );
assertTrue( ( int ) tlr.testOrderComparator( className, className3, null, null ) > 0 );
assertTrue( ( int ) tlr.testOrderComparator( className, className, null, null ) == 0 );
}
}
51 changes: 51 additions & 0 deletions surefire-api/src/test/java/org/apache/maven/surefire/api/util/RunOrderCalculatorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,10 @@
* under the License.
*/

import java.util.Arrays;
import java.util.Comparator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;

import org.apache.maven.surefire.api.testset.RunOrderParameters;
Expand Down Expand Up @@ -59,4 +62,52 @@ static class B
{

}

public void testOrderTestMethods()
{
RunOrderParameters runOrderParameters = new RunOrderParameters( "testorder" , null );
System.setProperty( "test", "TestClass#a2d,TestClass#aBc,TestClass#abc,TestClass#a1b" );
DefaultRunOrderCalculator runOrderCalculator = new DefaultRunOrderCalculator( runOrderParameters, 1 );
Comparator<String> testOrderRunOrderComparator = runOrderCalculator.comparatorForTestMethods();
String[] strArray = { "abc(TestClass)", "a1b(TestClass)", "a2d(TestClass)", "aBc(TestClass)" };
List<String> actual = Arrays.asList( strArray );
actual.sort( testOrderRunOrderComparator );
String[] strArray2 = { "a2d(TestClass)", "aBc(TestClass)", "abc(TestClass)", "a1b(TestClass)" };
List<String> expected = Arrays.asList( strArray2 );
assertEquals( actual, expected );
}

public void testOrderTestClassesAndMethods()
{
RunOrderParameters runOrderParameters = new RunOrderParameters( "testorder" , null );
System.setProperty( "test", "TestClass1#a2d,TestClass2#aBc,TestClass2#abc,TestClass2#a1b" );
DefaultRunOrderCalculator runOrderCalculator = new DefaultRunOrderCalculator( runOrderParameters, 1 );
Comparator<String> testOrderRunOrderComparator = runOrderCalculator.comparatorForTestMethods();
String[] strArray = { "abc(TestClass2)", "a1b(TestClass2)", "a2d(TestClass1)", "aBc(TestClass2)" };
List<String> actual = Arrays.asList( strArray );
actual.sort( testOrderRunOrderComparator );
String[] strArray2 = { "a2d(TestClass1)", "aBc(TestClass2)", "abc(TestClass2)", "a1b(TestClass2)" };
List<String> expected = Arrays.asList( strArray2 );
assertEquals( actual, expected );
}

public void testOrderTestRegexClassesAndMethods()
{
RunOrderParameters runOrderParameters = new RunOrderParameters( "testorder" , null );
System.setProperty( "test", "Amber*Test#a?c,My???Test#test*" );
DefaultRunOrderCalculator runOrderCalculator = new DefaultRunOrderCalculator( runOrderParameters, 1 );
Comparator<String> testOrderRunOrderComparator = runOrderCalculator.comparatorForTestMethods();
String[] strArray = { "abc(AmberGoodTest)",
"testabc(MyabcTest)",
"a2c(AmberBadTest)",
"testefg(MyefgTest)",
"aBc(AmberGoodTest)" };
List<String> actual = Arrays.asList( strArray );
actual.sort( testOrderRunOrderComparator );
assertEquals( runOrderCalculator.getClassAndMethod( actual.get( 0 ) )[0].substring( 0, 5 ), "Amber" );
assertEquals( runOrderCalculator.getClassAndMethod( actual.get( 1 ) )[0].substring( 0, 5 ), "Amber" );
assertEquals( runOrderCalculator.getClassAndMethod( actual.get( 2 ) )[0].substring( 0, 5 ), "Amber" );
assertEquals( runOrderCalculator.getClassAndMethod( actual.get( 3 ) )[0].substring( 0, 2 ), "My" );
assertEquals( runOrderCalculator.getClassAndMethod( actual.get( 4 ) )[0].substring( 0, 2 ), "My" );
}
}