From cb977633f0f1d2be4b2556a4743a5f52d8a7008a Mon Sep 17 00:00:00 2001 From: Sean Flanigan Date: Sat, 11 Apr 2015 11:32:37 +1000 Subject: [PATCH 1/6] [SUREFIRE-1152] Support rerunFailingTestsCount for test suites Submitted by: Sean Flanigan --- ...urefire1152RerunFailingTestsInSuiteIT.java | 42 +++++++++ .../pom.xml | 91 +++++++++++++++++++ .../test/java/jiras/surefire1152/FlakyIT.java | 35 +++++++ .../java/jiras/surefire1152/FlakyITSuite.java | 28 ++++++ .../java/jiras/surefire1152/FlakyTest.java | 35 +++++++ .../jiras/surefire1152/FlakyTestSuite.java | 28 ++++++ .../common/junit4/JUnit4ProviderUtil.java | 10 +- .../maven/surefire/junit4/JUnit4Provider.java | 20 +++- 8 files changed, 279 insertions(+), 10 deletions(-) create mode 100644 surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1152RerunFailingTestsInSuiteIT.java create mode 100644 surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/pom.xml create mode 100644 surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyIT.java create mode 100644 surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyITSuite.java create mode 100644 surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyTest.java create mode 100644 surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyTestSuite.java diff --git a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1152RerunFailingTestsInSuiteIT.java b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1152RerunFailingTestsInSuiteIT.java new file mode 100644 index 0000000000..7385b22ba8 --- /dev/null +++ b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1152RerunFailingTestsInSuiteIT.java @@ -0,0 +1,42 @@ +package org.apache.maven.surefire.its.jiras; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.surefire.its.fixture.OutputValidator; +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.junit.Test; + +/** + * SUREFIRE-1152 Assert rerunFailingTestsCount works with test suites + * + * @author Sean Flanigan + */ +public class Surefire1152RerunFailingTestsInSuiteIT + extends SurefireJUnit4IntegrationTestCase +{ + + @Test + public void rerunsFailingTestInSuite() + { + OutputValidator outputValidator = unpack( "surefire-1152-rerunFailingTestsCount-suite" ).executeVerify(); + outputValidator.assertTestSuiteResults( 1, 0, 0, 0 ); + outputValidator.assertIntegrationTestSuiteResults( 1, 0, 0, 0 ); + } +} diff --git a/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/pom.xml b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/pom.xml new file mode 100644 index 0000000000..b0206b7aa5 --- /dev/null +++ b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/pom.xml @@ -0,0 +1,91 @@ + + + + 4.0.0 + + + org.apache.maven.surefire + it-parent + 1.0 + ../pom.xml + + org.apache.maven.plugins.surefire + jiras-surefire-1152 + + 1.0 + + + true + 2 + 2 + + + + + + maven-compiler-plugin + + 1.5 + 1.5 + + + + org.apache.maven.plugins + maven-surefire-plugin + + + **/FlakyTestSuite.java + + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + + + + integration-test + verify + + + + + + **/FlakyITSuite.java + + + + + + + + + junit + junit + 4.11 + test + + + + diff --git a/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyIT.java b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyIT.java new file mode 100644 index 0000000000..54888db4ac --- /dev/null +++ b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyIT.java @@ -0,0 +1,35 @@ +package jiras.surefire1152; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.junit.Test; + +public class FlakyIT { + + static int n = 0; + + @Test + public void testFlaky() { + if (n++ == 0) { + throw new AssertionError("deliberately flaky test (should pass the next time)"); + } + } + +} diff --git a/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyITSuite.java b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyITSuite.java new file mode 100644 index 0000000000..77d1c9ff92 --- /dev/null +++ b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyITSuite.java @@ -0,0 +1,28 @@ +package jiras.surefire1152; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +@RunWith(Suite.class) +@Suite.SuiteClasses({FlakyIT.class}) +public class FlakyITSuite { +} diff --git a/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyTest.java b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyTest.java new file mode 100644 index 0000000000..613b7df0b4 --- /dev/null +++ b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyTest.java @@ -0,0 +1,35 @@ +package jiras.surefire1152; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.junit.Test; + +public class FlakyTest { + + static int n = 0; + + @Test + public void testFlaky() { + if (n++ == 0) { + throw new AssertionError("deliberately flaky test (should pass the next time)"); + } + } + +} diff --git a/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyTestSuite.java b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyTestSuite.java new file mode 100644 index 0000000000..09e100b25b --- /dev/null +++ b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyTestSuite.java @@ -0,0 +1,28 @@ +package jiras.surefire1152; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +@RunWith(Suite.class) +@Suite.SuiteClasses({FlakyTest.class}) +public class FlakyTestSuite { +} diff --git a/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java index 610ac2c162..1e35552397 100644 --- a/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java +++ b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java @@ -90,14 +90,14 @@ public static Map, Set> generateFailingTests( List all } /** - * Get the name of all test methods from a list of Failures + * Get all test methods from a list of Failures * * @param allFailures the list of failures for a given test class - * @return the list of test method names + * @return the list of test methods */ - public static Set generateFailingTests( List allFailures ) + public static Set generateFailingTests( List allFailures ) { - Set failingMethods = new HashSet(); + Set failingMethods = new HashSet(); for ( Failure failure : allFailures ) { @@ -107,7 +107,7 @@ public static Set generateFailingTests( List allFailures ) ClassMethod classMethod = cutTestClassAndMethod( description ); if ( classMethod.isValid() ) { - failingMethods.add( classMethod.getMethod() ); + failingMethods.add( classMethod ); } } } diff --git a/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java b/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java index 12c3983098..b622aea910 100644 --- a/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java +++ b/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java @@ -171,11 +171,12 @@ private void executeWithRerun( Class clazz, RunNotifier listeners ) { for ( int i = 0; i < rerunFailingTestsCount && !failureListener.getAllFailures().isEmpty(); i++ ) { - Set failedTests = JUnit4ProviderUtil.generateFailingTests( failureListener.getAllFailures() ); + Set failedTests = JUnit4ProviderUtil.generateFailingTests( + failureListener.getAllFailures( ) ); failureListener.reset(); if ( !failedTests.isEmpty() ) { - executeFailedMethod( clazz, listeners, failedTests ); + executeFailedMethod( listeners, failedTests ); } } } @@ -273,11 +274,20 @@ private static void execute( Class testClass, RunNotifier notifier, Filter fi } } - private static void executeFailedMethod( Class testClass, RunNotifier notifier, Set failedMethods ) + private void executeFailedMethod( RunNotifier notifier, Set failedMethods ) { - for ( String failedMethod : failedMethods ) + for ( ClassMethod failedMethod : failedMethods ) { - Request.method( testClass, failedMethod ).getRunner().run( notifier ); + try + { + Class methodClass = Class.forName( failedMethod.getClazz(), true, testClassLoader ); + String methodName = failedMethod.getMethod(); + Request.method( methodClass, methodName ).getRunner().run( notifier ); + } + catch ( ClassNotFoundException e ) + { + throw new RuntimeException( "Unable to create test class '" + failedMethod.getClazz() + "'", e ); + } } } From a561181a0414f62244b63ab78ea7488bb1e9b2e7 Mon Sep 17 00:00:00 2001 From: Sean Flanigan Date: Mon, 13 Apr 2015 11:03:36 +1000 Subject: [PATCH 2/6] Add review changes requested by Tibor Digana --- .../pom.xml | 5 ++--- .../src/test/java/jiras/surefire1152/FlakyIT.java | 14 +++++++++----- .../test/java/jiras/surefire1152/FlakyITSuite.java | 7 ++++--- .../test/java/jiras/surefire1152/FlakyTest.java | 14 +++++++++----- .../java/jiras/surefire1152/FlakyTestSuite.java | 7 ++++--- .../maven/surefire/junit4/JUnit4Provider.java | 5 +++-- 6 files changed, 31 insertions(+), 21 deletions(-) diff --git a/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/pom.xml b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/pom.xml index b0206b7aa5..5719efd7e7 100644 --- a/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/pom.xml +++ b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/pom.xml @@ -26,7 +26,6 @@ org.apache.maven.surefire it-parent 1.0 - ../pom.xml org.apache.maven.plugins.surefire jiras-surefire-1152 @@ -53,7 +52,7 @@ maven-surefire-plugin - **/FlakyTestSuite.java + FlakyTestSuite @@ -72,7 +71,7 @@ - **/FlakyITSuite.java + FlakyITSuite diff --git a/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyIT.java b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyIT.java index 54888db4ac..27d5b5f500 100644 --- a/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyIT.java +++ b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyIT.java @@ -21,14 +21,18 @@ import org.junit.Test; -public class FlakyIT { +import static org.junit.Assert.fail; - static int n = 0; +public class FlakyIT +{ + private static int n; @Test - public void testFlaky() { - if (n++ == 0) { - throw new AssertionError("deliberately flaky test (should pass the next time)"); + public void testFlaky() + { + if ( n++ == 0 ) + { + fail( "deliberately flaky test (should pass the next time)" ); } } diff --git a/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyITSuite.java b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyITSuite.java index 77d1c9ff92..0ebd03cb05 100644 --- a/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyITSuite.java +++ b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyITSuite.java @@ -22,7 +22,8 @@ import org.junit.runner.RunWith; import org.junit.runners.Suite; -@RunWith(Suite.class) -@Suite.SuiteClasses({FlakyIT.class}) -public class FlakyITSuite { +@RunWith( Suite.class ) +@Suite.SuiteClasses( { FlakyIT.class } ) +public class FlakyITSuite +{ } diff --git a/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyTest.java b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyTest.java index 613b7df0b4..cab834c7ce 100644 --- a/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyTest.java +++ b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyTest.java @@ -21,14 +21,18 @@ import org.junit.Test; -public class FlakyTest { +import static org.junit.Assert.fail; - static int n = 0; +public class FlakyTest +{ + private static int n; @Test - public void testFlaky() { - if (n++ == 0) { - throw new AssertionError("deliberately flaky test (should pass the next time)"); + public void testFlaky() + { + if ( n++ == 0 ) + { + fail( "deliberately flaky test (should pass the next time)" ); } } diff --git a/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyTestSuite.java b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyTestSuite.java index 09e100b25b..85ec1b33b1 100644 --- a/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyTestSuite.java +++ b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyTestSuite.java @@ -22,7 +22,8 @@ import org.junit.runner.RunWith; import org.junit.runners.Suite; -@RunWith(Suite.class) -@Suite.SuiteClasses({FlakyTest.class}) -public class FlakyTestSuite { +@RunWith( Suite.class ) +@Suite.SuiteClasses( { FlakyTest.class } ) +public class FlakyTestSuite +{ } diff --git a/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java b/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java index b622aea910..536da715c6 100644 --- a/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java +++ b/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java @@ -159,7 +159,7 @@ private void executeTestSet( Class clazz, RunListener reporter, RunNotifier l } } - private void executeWithRerun( Class clazz, RunNotifier listeners ) + private void executeWithRerun( Class clazz, RunNotifier listeners ) throws TestSetFailedException { JUnitTestFailureListener failureListener = new JUnitTestFailureListener(); listeners.addListener( failureListener ); @@ -275,6 +275,7 @@ private static void execute( Class testClass, RunNotifier notifier, Filter fi } private void executeFailedMethod( RunNotifier notifier, Set failedMethods ) + throws TestSetFailedException { for ( ClassMethod failedMethod : failedMethods ) { @@ -286,7 +287,7 @@ private void executeFailedMethod( RunNotifier notifier, Set failedM } catch ( ClassNotFoundException e ) { - throw new RuntimeException( "Unable to create test class '" + failedMethod.getClazz() + "'", e ); + throw new TestSetFailedException( "Unable to create test class '" + failedMethod.getClazz() + "'", e ); } } } From 62aa79835aaf2552035ee7de91c5e1dfc2e1db68 Mon Sep 17 00:00:00 2001 From: Sean Flanigan Date: Thu, 16 Apr 2015 16:55:44 +1000 Subject: [PATCH 3/6] Check that flaky tests are counted as such --- .../its/jiras/Surefire1152RerunFailingTestsInSuiteIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1152RerunFailingTestsInSuiteIT.java b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1152RerunFailingTestsInSuiteIT.java index 7385b22ba8..c17c6617a5 100644 --- a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1152RerunFailingTestsInSuiteIT.java +++ b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1152RerunFailingTestsInSuiteIT.java @@ -36,7 +36,7 @@ public class Surefire1152RerunFailingTestsInSuiteIT public void rerunsFailingTestInSuite() { OutputValidator outputValidator = unpack( "surefire-1152-rerunFailingTestsCount-suite" ).executeVerify(); - outputValidator.assertTestSuiteResults( 1, 0, 0, 0 ); + outputValidator.assertTestSuiteResults( 3, 0, 0, 0, 3 ); outputValidator.assertIntegrationTestSuiteResults( 1, 0, 0, 0 ); } } From 8cf6e57536eea65240d13114a8bcdd400f55770b Mon Sep 17 00:00:00 2001 From: Sean Flanigan Date: Thu, 16 Apr 2015 16:57:06 +1000 Subject: [PATCH 4/6] Check that inherited tests in suites are rerun correctly --- .../java/jiras/surefire1152/FlakyParent.java | 45 +++++++++++++++++++ .../java/jiras/surefire1152/FlakyTest.java | 2 +- .../jiras/surefire1152/FlakyTestSuite.java | 2 +- 3 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyParent.java diff --git a/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyParent.java b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyParent.java new file mode 100644 index 0000000000..86a08db845 --- /dev/null +++ b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyParent.java @@ -0,0 +1,45 @@ +package jiras.surefire1152; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.junit.Test; + +import java.util.HashSet; +import java.util.Set; + +import static org.junit.Assert.fail; + +public class FlakyParent +{ + // set of test classes which have previously invoked testFlakyParent + private static final Set> previouslyRun = new HashSet>(); + + @Test + public void testFlakyParent() + { + Class clazz = getClass(); + if ( !previouslyRun.contains( clazz ) ) + { + previouslyRun.add( clazz ); + fail( "deliberately flaky test (should pass the next time)" ); + } + } + +} diff --git a/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyTest.java b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyTest.java index cab834c7ce..ee20f9eb43 100644 --- a/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyTest.java +++ b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyTest.java @@ -23,7 +23,7 @@ import static org.junit.Assert.fail; -public class FlakyTest +public class FlakyTest extends FlakyParent { private static int n; diff --git a/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyTestSuite.java b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyTestSuite.java index 85ec1b33b1..f4a7d6773a 100644 --- a/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyTestSuite.java +++ b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/src/test/java/jiras/surefire1152/FlakyTestSuite.java @@ -23,7 +23,7 @@ import org.junit.runners.Suite; @RunWith( Suite.class ) -@Suite.SuiteClasses( { FlakyTest.class } ) +@Suite.SuiteClasses( { FlakyTest.class, FlakyParent.class } ) public class FlakyTestSuite { } From 7552ea926a2df2869aead38a52f4bad49acaea8c Mon Sep 17 00:00:00 2001 From: Sean Flanigan Date: Tue, 28 Apr 2015 14:29:09 +1000 Subject: [PATCH 5/6] Support rerunFailingTestsCount for JUnit 4.7 test suites --- ...urefire1152RerunFailingTestsInSuiteIT.java | 27 +++++++- .../pom.xml | 61 +++++++++++++++++++ .../common/junit4/JUnit4ProviderUtil.java | 44 ++++++------- .../common/junit4/JUnit4ProviderUtilTest.java | 8 +-- .../surefire/junitcore/JUnitCoreProvider.java | 2 +- 5 files changed, 108 insertions(+), 34 deletions(-) diff --git a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1152RerunFailingTestsInSuiteIT.java b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1152RerunFailingTestsInSuiteIT.java index c17c6617a5..e0bc7de0ba 100644 --- a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1152RerunFailingTestsInSuiteIT.java +++ b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1152RerunFailingTestsInSuiteIT.java @@ -21,6 +21,7 @@ import org.apache.maven.surefire.its.fixture.OutputValidator; import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.apache.maven.surefire.its.fixture.SurefireLauncher; import org.junit.Test; /** @@ -31,12 +32,32 @@ public class Surefire1152RerunFailingTestsInSuiteIT extends SurefireJUnit4IntegrationTestCase { + private static final String RUNNING_WITH_JUNIT48 = + "Using configured provider org.apache.maven.surefire.junitcore.JUnitCoreProvider"; - @Test - public void rerunsFailingTestInSuite() + public OutputValidator runMethodPattern( String projectName, String... goals ) { - OutputValidator outputValidator = unpack( "surefire-1152-rerunFailingTestsCount-suite" ).executeVerify(); + SurefireLauncher launcher = unpack( projectName ); + for ( String goal : goals ) + { + launcher.addGoal( goal ); + } + OutputValidator outputValidator = launcher.executeTest(); outputValidator.assertTestSuiteResults( 3, 0, 0, 0, 3 ); outputValidator.assertIntegrationTestSuiteResults( 1, 0, 0, 0 ); + return outputValidator; + } + + @Test + public void testJUnit48Provider4() + { + runMethodPattern( "surefire-1152-rerunFailingTestsCount-suite", "-P surefire-junit4" ); } + + @Test + public void testJUnit48Provider47() + { + runMethodPattern( "surefire-1152-rerunFailingTestsCount-suite", "-P surefire-junit47" ).verifyTextInLog( RUNNING_WITH_JUNIT48 ); + } + } diff --git a/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/pom.xml b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/pom.xml index 5719efd7e7..fd11c141c8 100644 --- a/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/pom.xml +++ b/surefire-integration-tests/src/test/resources/surefire-1152-rerunFailingTestsCount-suite/pom.xml @@ -78,6 +78,67 @@ + + + surefire-junit47 + + + + org.apache.maven.plugins + maven-surefire-plugin + + + org.apache.maven.surefire + surefire-junit47 + ${surefire.version} + + + + + org.apache.maven.plugins + maven-failsafe-plugin + + + org.apache.maven.surefire + surefire-junit47 + ${surefire.version} + + + + + + + + surefire-junit4 + + + + org.apache.maven.plugins + maven-surefire-plugin + + + org.apache.maven.surefire + surefire-junit4 + ${surefire.version} + + + + + org.apache.maven.plugins + maven-failsafe-plugin + + + org.apache.maven.surefire + surefire-junit4 + ${surefire.version} + + + + + + + + junit diff --git a/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java index 1e35552397..6861a2138f 100644 --- a/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java +++ b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java @@ -19,8 +19,6 @@ * under the License. */ -import org.apache.maven.surefire.util.TestsToRun; - import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -28,6 +26,7 @@ import java.util.Map; import java.util.Set; +import org.apache.maven.surefire.testset.TestSetFailedException; import org.apache.maven.surefire.util.internal.StringUtils; import org.junit.runner.Description; @@ -52,38 +51,31 @@ private JUnit4ProviderUtil() * Organize all the failures in previous run into a map between test classes and corresponding failing test methods * * @param allFailures all the failures in previous run - * @param testsToRun all the test classes + * @param testClassLoader ClassLoader used for test classes * @return a map between failing test classes and their corresponding failing test methods */ - public static Map, Set> generateFailingTests( List allFailures, TestsToRun testsToRun ) + public static Map, Set> generateFailingTests( List allFailures, + ClassLoader testClassLoader ) + throws TestSetFailedException { Map, Set> testClassMethods = new HashMap, Set>(); - - for ( Failure failure : allFailures ) + Set failingTests = generateFailingTests( allFailures ); + for ( ClassMethod classMethod: failingTests ) { - Description description = failure.getDescription(); - if ( description.isTest() && !isFailureInsideJUnitItself( description ) ) + try { - ClassMethod classMethod = cutTestClassAndMethod( description ); - if ( classMethod.isValid() ) + Class testClassObj = Class.forName( classMethod.getClazz(), true, testClassLoader ); + Set failingMethods = testClassMethods.get( testClassObj ); + if ( failingMethods == null ) { - Class testClassObj = testsToRun.getClassByName( classMethod.getClazz() ); - - if ( testClassObj != null ) - { - Set failingMethods = testClassMethods.get( testClassObj ); - if ( failingMethods == null ) - { - failingMethods = new HashSet(); - failingMethods.add( classMethod.getMethod() ); - testClassMethods.put( testClassObj, failingMethods ); - } - else - { - failingMethods.add( classMethod.getMethod() ); - } - } + failingMethods = new HashSet(); + testClassMethods.put( testClassObj, failingMethods ); } + failingMethods.add( classMethod.getMethod() ); + } + catch ( ClassNotFoundException e ) + { + throw new TestSetFailedException( "Unable to create test class '" + classMethod.getClazz() + "'", e ); } } return testClassMethods; diff --git a/surefire-providers/common-junit4/src/test/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtilTest.java b/surefire-providers/common-junit4/src/test/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtilTest.java index f4fdc95258..6a6e2e7db1 100644 --- a/surefire-providers/common-junit4/src/test/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtilTest.java +++ b/surefire-providers/common-junit4/src/test/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtilTest.java @@ -20,6 +20,7 @@ */ import junit.framework.TestCase; +import org.apache.maven.surefire.testset.TestSetFailedException; import org.apache.maven.surefire.util.TestsToRun; import org.junit.runner.Description; import org.junit.runner.notification.Failure; @@ -39,9 +40,8 @@ public class JUnit4ProviderUtilTest extends TestCase { - public void testGenerateFailingTests() + public void testGenerateFailingTests() throws TestSetFailedException { - TestsToRun testsToRun = new TestsToRun( Arrays.asList( new Class[]{ T1.class, T2.class } ) ); List failures = new ArrayList( ); Description test1Description = Description.createTestDescription( T1.class, "testOne" ); @@ -56,7 +56,7 @@ public void testGenerateFailingTests() failures.add( new Failure( test4Description, new AssertionError() ) ); failures.add( new Failure( test5Description, new RuntimeException() ) ); - Map, Set> result = generateFailingTests( failures, testsToRun ); + Map, Set> result = generateFailingTests( failures, getClass().getClassLoader() ); assertEquals( 2, result.size() ); Set resultForT1 = result.get( T1.class ); @@ -108,4 +108,4 @@ class T2 { } -} \ No newline at end of file +} diff --git a/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java b/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java index 3276b4ea19..bc2dd3b01c 100644 --- a/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java +++ b/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java @@ -147,7 +147,7 @@ else if ( forkTestSet instanceof Class ) for ( int i = 0; i < rerunFailingTestsCount && !testFailureListener.getAllFailures().isEmpty(); i++ ) { Map, Set> failingTests = - JUnit4ProviderUtil.generateFailingTests( testFailureListener.getAllFailures(), testsToRun ); + JUnit4ProviderUtil.generateFailingTests( testFailureListener.getAllFailures(), testClassLoader ); testFailureListener.reset(); final FilterFactory filterFactory = new FilterFactory( testClassLoader ); Filter failingMethodsFilter = filterFactory.createFailingMethodFilter( failingTests ); From 78fb9b054a12cfeb5a4e4306e12fe894e50a5dff Mon Sep 17 00:00:00 2001 From: Sean Flanigan Date: Thu, 30 Apr 2015 17:51:36 +1000 Subject: [PATCH 6/6] Add review changes requested by Tibor Digana --- .../its/jiras/Surefire1152RerunFailingTestsInSuiteIT.java | 2 +- .../apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java | 2 +- .../org/apache/maven/surefire/junitcore/JUnitCoreProvider.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1152RerunFailingTestsInSuiteIT.java b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1152RerunFailingTestsInSuiteIT.java index e0bc7de0ba..fbf87d5d40 100644 --- a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1152RerunFailingTestsInSuiteIT.java +++ b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1152RerunFailingTestsInSuiteIT.java @@ -42,7 +42,7 @@ public OutputValidator runMethodPattern( String projectName, String... goals ) { launcher.addGoal( goal ); } - OutputValidator outputValidator = launcher.executeTest(); + OutputValidator outputValidator = launcher.executeVerify(); outputValidator.assertTestSuiteResults( 3, 0, 0, 0, 3 ); outputValidator.assertIntegrationTestSuiteResults( 1, 0, 0, 0 ); return outputValidator; diff --git a/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java index 6861a2138f..05988b7fa5 100644 --- a/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java +++ b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java @@ -64,7 +64,7 @@ public static Map, Set> generateFailingTests( List all { try { - Class testClassObj = Class.forName( classMethod.getClazz(), true, testClassLoader ); + Class testClassObj = Class.forName( classMethod.getClazz(), false, testClassLoader ); Set failingMethods = testClassMethods.get( testClassObj ); if ( failingMethods == null ) { diff --git a/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java b/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java index bc2dd3b01c..66147adccf 100644 --- a/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java +++ b/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java @@ -152,7 +152,7 @@ else if ( forkTestSet instanceof Class ) final FilterFactory filterFactory = new FilterFactory( testClassLoader ); Filter failingMethodsFilter = filterFactory.createFailingMethodFilter( failingTests ); JUnitCoreWrapper.execute( consoleLogger, testsToRun, jUnitCoreParameters, customRunListeners, - filterFactory.and( filter, failingMethodsFilter ) ); + failingMethodsFilter ); } } return reporterFactory.close();