other ITs should not be so sensitive to TestNG and Mockito Reflection…

… [ERROR]s
apache · May 12, 2020 · d715f82 · d715f82
1 parent c60df38
commit d715f82
Show file tree

Hide file tree

Showing 7 changed files with 38 additions and 20 deletions.
diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/CheckTestNgListenerReporterIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/CheckTestNgListenerReporterIT.java
@@ -103,7 +103,7 @@ public void testNgListenerReporter()
         }
 
         launcher.executeTest()
-                .verifyErrorFree( 1 )
+                .assertTestSuiteResults( 1, 0, 0, 0 )
                 .getTargetFile( "resultlistener-output.txt" ).assertFileExists()
                 .getTargetFile( "suitelistener-output.txt" ).assertFileExists()
                 .getTargetFile( "reporter-output.txt" ).assertFileExists();

diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/CheckTestNgSuiteXmlIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/CheckTestNgSuiteXmlIT.java
@@ -34,8 +34,9 @@ public class CheckTestNgSuiteXmlIT
     @Test
     public void suiteXml()
     {
-        unpack().executeTest()
-                .verifyErrorFree( 2 );
+        unpack()
+            .executeTest()
+            .assertTestSuiteResults( 2, 0, 0, 0 );
     }
 
     @Test
@@ -52,7 +53,7 @@ public void suiteXmlForkCountTwoReuse()
         unpack().forkCount( 2 )
                 .reuseForks( true )
                 .executeTest()
-                .verifyErrorFree( 2 );
+                .assertTestSuiteResults( 2, 0, 0, 0 );
     }
 
     private SurefireLauncher unpack()

diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformIT.java
@@ -78,7 +78,7 @@ public void testVintageEngine()
                 .sysProp( "junit5.version", junit5Version )
                 .sysProp( "jqwik.version", jqwikVersion )
                 .executeTest()
-                .verifyErrorFree( 1 );
+                .assertTestSuiteResults( 1, 0, 0, 0 );
     }
 
     @Test
@@ -88,7 +88,7 @@ public void testJQwikEngine()
                 .sysProp( "junit5.version", junit5Version )
                 .sysProp( "jqwik.version", jqwikVersion )
                 .executeTest()
-                .verifyErrorFree( 1 );
+                .assertTestSuiteResults( 1, 0, 0, 0 );
     }
 
     @Test
@@ -99,7 +99,7 @@ public void testMultipleEngines()
                 .sysProp( "junit5.version", junit5Version )
                 .sysProp( "jqwik.version", jqwikVersion )
                 .executeTest()
-                .verifyErrorFree( 7 );
+                .assertTestSuiteResults( 7, 0, 0, 0 );
 
 
         validator.getSurefireReportsFile( "TEST-junitplatformenginejupiter.BasicJupiterTest.xml", UTF_8 )

diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/TwoTestCasesIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/TwoTestCasesIT.java
@@ -120,7 +120,7 @@ public void testTestNGSuite()
                                                         .sysProp( "testNgVersion", "5.7" )
                                                         .sysProp( "testNgClassifier", "jdk15" )
                                                         .executeTest();
-        outputValidator.verifyErrorFreeLog().assertTestSuiteResults( 2, 0, 0, 0 );
+        outputValidator.assertTestSuiteResults( 2, 0, 0, 0 );
         List<ReportTestSuite> reports = HelperAssertions.extractReports( outputValidator.getBaseDir() );
         Set<String> classNames = extractClassNames( reports );
         assertContains( classNames, "testng.two.TestNGTestTwo" );

diff --git a/...java/org/apache/maven/surefire/its/jiras/Surefire1712ExtractedModulenameWithoutASMIT.java b/...java/org/apache/maven/surefire/its/jiras/Surefire1712ExtractedModulenameWithoutASMIT.java
@@ -22,7 +22,8 @@
 import org.apache.maven.surefire.its.AbstractJigsawIT;
 import org.junit.Test;
 
-import java.io.IOException;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.is;
 
 /**
  * See the JIRA https://issues.apache.org/jira/browse/SUREFIRE-1712
@@ -35,14 +36,15 @@ public class Surefire1712ExtractedModulenameWithoutASMIT
 {
     @Test
     public void test()
-            throws IOException
+            throws Exception
     {
         assumeJava9()
-                .debugLogging()
-                .executeTest()
-                .assertTestSuiteResults( 1, 0, 0, 0 )
-                .verifyErrorFreeLog()
-                .verifyTextInLog( "main module descriptor name: wtf.g4s8.oot" );
+            .debugLogging()
+            .executeTest()
+            .assertTestSuiteResults( 1, 0, 0, 0 )
+            .assertThatLogLine( containsString( "Unsupported class file major version" ), is( 0 ) )
+            .assertThatLogLine( containsString( "at org.objectweb.asm.ClassReader.<init>" ), is( 0 ) )
+            .verifyTextInLog( "main module descriptor name: wtf.g4s8.oot" );
     }
 
     @Override

diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire673MockitoIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire673MockitoIT.java
@@ -33,6 +33,8 @@ public class Surefire673MockitoIT
     @Test
     public void testBuildFailingWhenErrors()
     {
-        unpack( "/surefire-673-mockito" ).executeTest().verifyErrorFreeLog();
+        unpack( "/surefire-673-mockito" )
+            .executeTest()
+            .assertTestSuiteResults( 1, 0, 0, 0 );
     }
-}
+}
diff --git a/...a/org/apache/maven/surefire/its/jiras/Surefire735ForkFailWithRedirectConsoleOutputIT.java b/...a/org/apache/maven/surefire/its/jiras/Surefire735ForkFailWithRedirectConsoleOutputIT.java
@@ -19,15 +19,18 @@
  * under the License.
  */
 
+import org.apache.maven.it.VerificationException;
 import org.apache.maven.surefire.its.fixture.OutputValidator;
 import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
 import org.apache.maven.surefire.its.fixture.SurefireLauncher;
 import org.junit.Test;
 
 import java.io.File;
 import java.io.FilenameFilter;
+import java.util.Collection;
 
 import static org.fest.assertions.Assertions.assertThat;
+import static org.hamcrest.CoreMatchers.containsString;
 
 /**
  * @author Kristian Rosenvold
@@ -37,14 +40,14 @@ public class Surefire735ForkFailWithRedirectConsoleOutputIT
 {
 
     @Test
-    public void vmStartFail()
+    public void vmStartFail() throws VerificationException
     {
         OutputValidator outputValidator = unpack().failNever().executeTest();
         assertJvmCrashed( outputValidator );
     }
 
     @Test
-    public void vmStartFailShouldFailBuildk()
+    public void vmStartFailShouldFailBuildk() throws VerificationException
     {
         OutputValidator outputValidator = unpack().maven().withFailure().executeTest();
         assertJvmCrashed( outputValidator );
@@ -55,8 +58,18 @@ private SurefireLauncher unpack()
         return unpack( "fork-fail" );
     }
 
-    private static void assertJvmCrashed( OutputValidator outputValidator )
+    private static void assertJvmCrashed( OutputValidator outputValidator ) throws VerificationException
     {
+        Collection<String> matchedLines =
+            outputValidator.loadLogLines( containsString( "Invalid maximum heap size: -Xmxxxx712743m" ) );
+        if ( !matchedLines.isEmpty() )
+        {
+            // the error line was printed in std/err by the JVM
+            return;
+        }
+
+        // the error line should be printed in std/out by the JVM if we use the process pipes
+        // then the ForkClient caught it and printed in the dump stream
         File reportDir = outputValidator.getSurefireReportsDirectory();
         String[] dumpFiles = reportDir.list( new FilenameFilter()
                                              {