[SUREFIRE-1264] Some tests can be lost when running in parallel with …

…parameterized tests

surefire-api/src/main/java/org/apache/maven/surefire/util/internal/TestClassMethodNameUtils.java

@@ -1 +1,52 @@
-package org.apache.maven.surefire.util.internal;/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements.  See the NOTICE file * distributed with this work for additional information * regarding copyright ownership.  The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License.  You may obtain a copy of the License at * *     http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied.  See the License for the * specific language governing permissions and limitations * under the License. */import java.util.regex.Matcher;import java.util.regex.Pattern;/** * JUnit Description parser. * Used by JUnit Version lower than 4.7. * * @author <a href="mailto:tibordigana@apache.org">Tibor Digana (tibor17)</a> * @since 2.20 */public final class TestClassMethodNameUtils{    private static final Pattern METHOD_CLASS_PATTERN = Pattern.compile( "([\\s\\S]*)\\((.*)\\)" );    private TestClassMethodNameUtils()    {        throw new IllegalStateException( "no instantiable constructor" );    }    public static String extractClassName( String displayName )    {        Matcher m = METHOD_CLASS_PATTERN.matcher( displayName );        return m.matches() ? m.group( 2 ) : displayName;    }    public static String extractMethodName( String displayName )    {        int i = displayName.indexOf( "(" );        return i >= 0 ? displayName.substring( 0, i ) : displayName;    }}
+package org.apache.maven.surefire.util.internal;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * JUnit Description parser.
+ * Used by JUnit Version lower than 4.7.
+ *
+ * @author <a href="mailto:tibordigana@apache.org">Tibor Digana (tibor17)</a>
+ * @since 2.20
+ */
+public final class TestClassMethodNameUtils
+{
+    private static final Pattern METHOD_CLASS_PATTERN = Pattern.compile( "([\\s\\S]*)\\((.*)\\)" );
+
+    private TestClassMethodNameUtils()
+    {
+        throw new IllegalStateException( "no instantiable constructor" );
+    }
+
+    public static String extractClassName( String displayName )
+    {
+        Matcher m = METHOD_CLASS_PATTERN.matcher( displayName );
+        return m.matches() ? m.group( 2 ) : displayName;
+    }
+
+    public static String extractMethodName( String displayName )
+    {
+        Matcher m = METHOD_CLASS_PATTERN.matcher( displayName );
+        return m.matches() ? m.group( 1 ) : null;
+    }
+}

surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/TestFile.java

@@ -39,7 +39,6 @@
  */
 public class TestFile
 {
-
     private final File file;
 
     private final Charset encoding;
@@ -148,4 +147,9 @@ public URI toURI()
     {
         return file.toURI();
     }
+
+    public File getFile()
+    {
+        return file;
+    }
 }

surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1082ParallelJUnitParameterizedIT.java

@@ -23,17 +23,22 @@
 import org.apache.maven.surefire.its.fixture.OutputValidator;
 import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
 import org.apache.maven.surefire.its.fixture.SurefireLauncher;
+import org.apache.maven.surefire.its.fixture.TestFile;
 import org.hamcrest.BaseMatcher;
 import org.hamcrest.Description;
 import org.hamcrest.Matcher;
 import org.junit.Test;
 
+import java.nio.charset.Charset;
+import java.util.Collection;
 import java.util.Iterator;
 import java.util.Set;
+import java.util.SortedSet;
 import java.util.TreeSet;
 
 import static org.hamcrest.core.AnyOf.anyOf;
 import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.StringContains.containsString;
 import static org.junit.Assert.assertThat;
 
 /**
@@ -44,17 +49,29 @@
 public class Surefire1082ParallelJUnitParameterizedIT
     extends SurefireJUnit4IntegrationTestCase
 {
+    private static Set<String> printOnlyTestLinesFromConsole( OutputValidator validator )
+            throws VerificationException
+    {
+        return printOnlyTestLines( validator.loadLogLines() );
+    }
+
+    private static Set<String> printOnlyTestLinesFromOutFile( OutputValidator validator )
+            throws VerificationException
+    {
+        TestFile report = validator.getSurefireReportsFile( "jiras.surefire1082.Jira1082Test-output.txt" );
+        report.assertFileExists();
+        return printOnlyTestLines( validator.loadFile( report.getFile(), Charset.forName( "UTF-8" ) ) );
+    }
 
-    private static Set<String> printOnlyTestLines( OutputValidator validator )
+    private static Set<String> printOnlyTestLines( Collection<String> logs )
         throws VerificationException
     {
-        Set<String> log = new TreeSet<String>( validator.loadLogLines() );
-        for ( Iterator<String> it = log.iterator(); it.hasNext(); )
+        SortedSet<String> log = new TreeSet<String>();
+        for ( String line : logs )
         {
-            String line = it.next();
-            if ( !line.startsWith( "class jiras.surefire1082." ) )
+            if ( line.startsWith( "class jiras.surefire1082." ) )
             {
-                it.remove();
+                log.add( line );
             }
         }
         return log;
@@ -65,14 +82,8 @@ private static Matcher<Set<String>> regex( Set<String> r )
         return new IsRegex( r );
     }
 
-    @Test
-    public void test()
-        throws VerificationException
+    private static void assertParallelRun( Set<String> log )
     {
-        OutputValidator validator = unpack().setTestToRun(
-            "Jira1082Test" ).parallelClasses().useUnlimitedThreads().executeTest().verifyErrorFree( 4 );
-
-        Set<String> log = printOnlyTestLines( validator );
         assertThat( log.size(), is( 4 ) );
 
         Set<String> expectedLogs1 = new TreeSet<String>();
@@ -90,6 +101,71 @@ public void test()
         assertThat( log, anyOf( regex( expectedLogs1 ), regex( expectedLogs2 ) ) );
     }
 
+    @Test
+    public void checkClassesRunParallel()
+        throws VerificationException
+    {
+        OutputValidator validator = unpack().setTestToRun( "Jira1082Test" )
+                                            .parallelClasses()
+                                            .useUnlimitedThreads()
+                                            .executeTest()
+                                            .verifyErrorFree( 4 );
+
+        validator.getSurefireReportsXmlFile( "TEST-jiras.surefire1082.Jira1082Test.xml" )
+                .assertFileExists();
+
+        validator.assertThatLogLine( containsString( "Running jiras.surefire1082.Jira1082Test" ), is( 1 ) );
+
+        Set<String> log = printOnlyTestLinesFromConsole( validator );
+        assertParallelRun( log );
+    }
+
+    @Test
+    public void checkOutFileClassesRunParallel()
+            throws VerificationException
+    {
+        OutputValidator validator = unpack().redirectToFile( true )
+                                            .setTestToRun( "Jira1082Test" )
+                                            .parallelClasses()
+                                            .useUnlimitedThreads()
+                                            .executeTest()
+                                            .verifyErrorFree( 4 );
+
+        validator.getSurefireReportsXmlFile( "TEST-jiras.surefire1082.Jira1082Test.xml" )
+                .assertFileExists();
+
+        validator.assertThatLogLine( containsString( "Running jiras.surefire1082.Jira1082Test" ), is( 1 ) );
+
+        Set<String> log = printOnlyTestLinesFromOutFile( validator );
+        assertParallelRun( log );
+    }
+
+    @Test
+    public void shouldRunTwo() throws VerificationException
+    {
+        OutputValidator validator = unpack().redirectToFile( true )
+                                            .parallelClasses()
+                                            .useUnlimitedThreads()
+                                            .executeTest()
+                                            .verifyErrorFree( 8 );
+
+        validator.getSurefireReportsXmlFile( "TEST-jiras.surefire1082.Jira1082Test.xml" )
+                .assertFileExists();
+
+        validator.getSurefireReportsXmlFile( "TEST-jiras.surefire1082.Jira1264Test.xml" )
+                .assertFileExists();
+
+        validator.getSurefireReportsFile( "jiras.surefire1082.Jira1082Test-output.txt" )
+                .assertFileExists();
+
+        validator.getSurefireReportsFile( "jiras.surefire1082.Jira1264Test-output.txt" )
+                .assertFileExists();
+
+        validator.assertThatLogLine( containsString( "Running jiras.surefire1082.Jira1082Test" ), is( 1 ) );
+
+        validator.assertThatLogLine( containsString( "Running jiras.surefire1082.Jira1264Test" ), is( 1 ) );
+    }
+
     private SurefireLauncher unpack()
     {
         return unpack( "surefire-1082-parallel-junit-parameterized" );

surefire-integration-tests/src/test/resources/surefire-1082-parallel-junit-parameterized/src/test/java/jiras/surefire1082/Jira1082Test.java

@@ -27,9 +27,9 @@
 import java.util.concurrent.TimeUnit;
 
 @RunWith( Parameterized.class )
-public final class Jira1082Test
+public class Jira1082Test
 {
-    private int x;
+    private final int x;
 
     public Jira1082Test( int x )
     {

surefire-integration-tests/src/test/resources/surefire-1082-parallel-junit-parameterized/src/test/java/jiras/surefire1082/Jira1264Test.java

@@ -0,0 +1,36 @@
+package jiras.surefire1082;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.Arrays;
+import java.util.concurrent.TimeUnit;
+
+@RunWith( Parameterized.class )
+public final class Jira1264Test extends Jira1082Test
+{
+    public Jira1264Test( int x )
+    {
+        super( x );
+    }
+}

surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreRunListener.java

@@ -19,16 +19,18 @@
  * under the License.
  */
 
-import java.util.ArrayList;
-import java.util.Map;
 import org.apache.maven.surefire.common.junit4.JUnit4RunListener;
 import org.apache.maven.surefire.common.junit48.JUnit46StackTraceWriter;
 import org.apache.maven.surefire.report.RunListener;
 import org.apache.maven.surefire.report.StackTraceWriter;
-
 import org.junit.runner.Description;
 import org.junit.runner.Result;
+import org.junit.runner.RunWith;
 import org.junit.runner.notification.Failure;
+import org.junit.runners.Parameterized;
+
+import java.lang.annotation.Annotation;
+import java.util.Map;
 
 /**
  * Noteworthy things about JUnit4 listening:
@@ -74,38 +76,57 @@ public void testRunFinished( Result result )
         reporter.testSetCompleted( null );
     }
 
-    private void fillTestCountMap( Description description )
+    private void fillTestCountMap( Description testDesc )
     {
-        final ArrayList<Description> children = description.getChildren();
+        TestSet testSet = new TestSet( testDesc );
+
+        String itemTestClassName =
+                isParameterizedRunner( testDesc ) ? testDesc.getClassName() : asSuiteRunner( testDesc, testSet );
 
-        TestSet testSet = new TestSet( description );
+        if ( itemTestClassName != null )
+        {
+            classMethodCounts.put( itemTestClassName, testSet );
+        }
+    }
+
+    private String asSuiteRunner( Description description, TestSet testSet )
+    {
         String itemTestClassName = null;
-        for ( Description item : children )
+        for ( Description child : description.getChildren() )
         {
-            if ( !item.isTest() )
+            if ( !child.isTest() )
             {
-                fillTestCountMap( item );
+                fillTestCountMap( child );
             }
             else
             {
-                if ( extractDescriptionMethodName( item ) != null )
+                if ( extractDescriptionMethodName( child ) != null )
                 {
                     testSet.incrementTestMethodCount();
                     if ( itemTestClassName == null )
                     {
-                        itemTestClassName = extractDescriptionClassName( item );
+                        itemTestClassName = extractDescriptionClassName( child );
                     }
                 }
                 else
                 {
-                    classMethodCounts.put( extractDescriptionClassName( item ), new TestSet( item ) );
+                    classMethodCounts.put( extractDescriptionClassName( child ), new TestSet( child ) );
                 }
             }
         }
-        if ( itemTestClassName != null )
+        return itemTestClassName;
+    }
+
+    private static boolean isParameterizedRunner( Description description )
+    {
+        for ( Annotation ann : description.getAnnotations() )
         {
-            classMethodCounts.put( itemTestClassName, testSet );
+            if ( ann.annotationType() == RunWith.class )
+            {
+                return Parameterized.class.isAssignableFrom( ( (RunWith) ann ).value() );
+            }
         }
+        return false;
     }
 
     @Override