Skip to content

Commit

Permalink
[SUREFIRE-1037] Elapsed time is reported incorrectly for tests run in…
Browse files Browse the repository at this point in the history 
… parallel
  • Loading branch information
Tibor17 committed May 10, 2015
1 parent 84fd723 commit c14b25f
Show file tree
Hide file tree
Showing 6 changed files with 106 additions and 74 deletions.
26 changes: 22 additions & 4 deletions ...e-integration-tests/src/test/java/org/apache/maven/surefire/its/Junit47concurrencyIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,14 @@
* under the License.
*/

import org.apache.maven.surefire.its.fixture.OutputValidator;
import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
import org.junit.Test;

import static org.hamcrest.CoreMatchers.containsString;
import static org.junit.Assert.*;
import static org.hamcrest.CoreMatchers.*;

/**
* Basic suite test using all known versions of JUnit 4.x
*
Expand All @@ -34,9 +39,22 @@ public class Junit47concurrencyIT
public void test47()
throws Exception
{
// todo: Align with others
unpack( "concurrentjunit47" ).sysProp( "junitVersion", "4.7" ).setJUnitVersion(
"4.7" ).executeTest().verifyErrorFree( 1 );

OutputValidator validator = unpack( "concurrentjunit47" )
.sysProp( "junitVersion", "4.7" )
.setJUnitVersion( "4.7" )
.executeTest()
.verifyErrorFree( 4 );
String result = null;
for ( String line : validator.loadLogLines() )
{
if ( line.startsWith( "Tests run: 4, Failures: 0, Errors: 0, Skipped: 0, Time elapsed:" ) )
{
result = line;
break;
}
}
assertNotNull( result);
assertThat( result, anyOf( containsString( "Time elapsed: 1." ), containsString( "Time elapsed: 0.9" ) ) );
assertThat( result, endsWith( " sec - in concurrentjunit47.src.test.java.junit47.BasicTest" ) );
}
}
19 changes: 19 additions & 0 deletions ...t/java/org/apache/maven/surefire/its/jiras/Surefire747MethodParallelWithSuiteCountIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,9 @@
import java.util.Set;
import java.util.TreeSet;

import static org.hamcrest.CoreMatchers.anyOf;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.endsWith;
import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
Expand Down Expand Up @@ -80,6 +83,22 @@ public void testMethodsParallelWithSuite()
long min = 750, max = 1250;
assertTrue( String.format( "duration %d should be between %d and %d millis", duration, min, max ),
duration > min && duration < max );

for ( String line : validator.loadLogLines() )
{
if ( line.startsWith( "Tests run: 3, Failures: 0, Errors: 0, Skipped: 0, Time elapsed:" ) )
{
assertThat( line, anyOf( // 0.5 sec, the delta -1.0/+0.3 can be varying depending on CI jobs
containsString( "Time elapsed: 0.4" ),
containsString( "Time elapsed: 0.5" ),
containsString( "Time elapsed: 0.6" ),
containsString( "Time elapsed: 0.7" ),
containsString( "Time elapsed: 0.8" ) ) );
assertThat( line, anyOf(
endsWith(" sec - in surefire747.SuiteTest1" ),
endsWith(" sec - in surefire747.SuiteTest2" ) ) );
}
}
}

@Test
Expand Down
3 changes: 2 additions & 1 deletion surefire-integration-tests/src/test/resources/concurrentjunit47/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,8 @@
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<parallel>methods</parallel>
<threadCount>2</threadCount>
<perCoreThreadCount>false</perCoreThreadCount>
<threadCount>3</threadCount>
</configuration>
</plugin>
</plugins>
Expand Down
21 changes: 16 additions & 5 deletions ...tegration-tests/src/test/resources/concurrentjunit47/src/test/java/junit47/BasicTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,27 +21,23 @@

import org.junit.*;

import java.util.concurrent.TimeUnit;

public class BasicTest
{

private boolean setUpCalled = false;

private static boolean tearDownCalled = false;

@Before
public void setUp()
{
setUpCalled = true;
tearDownCalled = false;
System.out.println( "Called setUp" );
}

@After
public void tearDown()
{
setUpCalled = false;
tearDownCalled = true;
System.out.println( "Called tearDown" );
}

Expand All @@ -51,6 +47,21 @@ public void testSetUp()
Assert.assertTrue( "setUp was not called", setUpCalled );
}

@Test
public void a() throws Exception {
TimeUnit.SECONDS.sleep( 1 );
}

@Test
public void b() throws Exception {
TimeUnit.SECONDS.sleep( 1 );
}

@Test
public void c() throws Exception {
TimeUnit.SECONDS.sleep( 1 );
}

@AfterClass
public static void oneTimeTearDown()
{
Expand Down
33 changes: 21 additions & 12 deletions ...viders/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/TestMethod.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ class TestMethod

private final long startTime;

private long endTime;
private volatile long endTime;

private volatile ReportEntry testFailure;

Expand All @@ -52,31 +52,31 @@ class TestMethod

private volatile LogicalStream output;

public TestMethod( ReportEntry description, TestSet testSet )
TestMethod( ReportEntry description, TestSet testSet )
{
this.description = description;
this.testSet = testSet;
startTime = System.currentTimeMillis();
}

public void testFinished()
void testFinished()
{
setEndTime();
}

public void testIgnored( ReportEntry description )
void testIgnored( ReportEntry description )
{
ignored = description;
setEndTime();
}

public void testFailure( ReportEntry failure )
void testFailure( ReportEntry failure )
{
this.testFailure = failure;
setEndTime();
}

public void testError( ReportEntry failure )
void testError( ReportEntry failure )
{
this.testError = failure;
setEndTime();
Expand All @@ -87,13 +87,22 @@ private void setEndTime()
this.endTime = System.currentTimeMillis();
}

public int getElapsed()
int getElapsed()
{
return endTime > 0 ? (int) ( endTime - startTime ) : 0;
}

long getStartTime()
{
return startTime;
}

long getEndTime()
{
return endTime;
}

public void replay( RunListener reporter )
void replay( RunListener reporter )
{

if ( ignored != null )
Expand Down Expand Up @@ -129,25 +138,25 @@ private ReportEntry createReportEntry( ReportEntry reportEntry )
reportEntry.getStackTraceWriter(), getElapsed(), reportEntry.getMessage() );
}

public void attachToThread()
void attachToThread()
{
TEST_METHOD.set( this );
ConsoleOutputReceiverForCurrentThread.set( this );

}

public void detachFromCurrentThread()
void detachFromCurrentThread()
{
TEST_METHOD.remove();
ConsoleOutputReceiverForCurrentThread.remove();
}

public static TestMethod getThreadTestMethod()
static TestMethod getThreadTestMethod()
{
return TEST_METHOD.get();
}

public LogicalStream getLogicalStream()
LogicalStream getLogicalStream()
{
if ( output == null )
{
Expand Down
78 changes: 26 additions & 52 deletions ...providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/TestSet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.maven.surefire.report.ConsoleOutputReceiver;
import org.apache.maven.surefire.report.ReportEntry;
import org.apache.maven.surefire.report.RunListener;
import org.apache.maven.surefire.report.SimpleReportEntry;
Expand Down Expand Up @@ -53,51 +52,48 @@ public class TestSet

private final AtomicBoolean played = new AtomicBoolean();

private volatile LogicalStream beforeClass;

private volatile LogicalStream afterClass;

public TestSet( Description testSetDescription )
{
this.testSetDescription = testSetDescription;
}

public void replay( RunListener target )
{
if ( !played.compareAndSet( false, true ) )
if ( played.compareAndSet( false, true ) )
{
return;
}
try
{
ReportEntry report = createReportEntry( null );

try
{
ReportEntry report = createReportEntry( null );
target.testSetStarting( report );

target.testSetStarting( report );
long startTile = 0;
long endTime = 0;
for ( TestMethod testMethod : testMethods )
{
if ( startTile == 0 || testMethod.getStartTime() < startTile )
{
startTile = testMethod.getStartTime();
}

if ( beforeClass != null )
{
beforeClass.writeDetails( ( (ConsoleOutputReceiver) target ) );
}
if ( endTime == 0 || testMethod.getEndTime() > endTime )
{
endTime = testMethod.getEndTime();
}

int elapsed = 0;
for ( TestMethod testMethod : testMethods )
{
elapsed += testMethod.getElapsed();
testMethod.replay( target );
}
testMethod.replay( target );
}

int elapsed = (int) ( endTime - startTile );

report = createReportEntry( elapsed );
report = createReportEntry( elapsed );

if ( afterClass != null )
target.testSetCompleted( report );
}
catch ( Exception e )
{
afterClass.writeDetails( ( (ConsoleOutputReceiver) target ) );
throw new RuntimeException( e );
}
target.testSetCompleted( report );
}
catch ( Exception e )
{
throw new RuntimeException( e );
}
}

Expand Down Expand Up @@ -159,26 +155,4 @@ public static TestSet getThreadTestSet()
{
return TEST_SET.get();
}

public LogicalStream getClassLevelLogicalStream()
{
if ( numberOfCompletedChildren.get() > 0 )
{
if ( afterClass == null )
{
afterClass = new LogicalStream();
}
return afterClass;
}
else
{
if ( beforeClass == null )
{
beforeClass = new LogicalStream();
}
return beforeClass;
}
}


}

0 comments on commit c14b25f

Please sign in to comment.