Merge branch '10.1'

sebastianbergmann · Apr 22, 2023 · 5c70b1b · 5c70b1b
2 parents d975b55 + 5fe2a35
commit 5c70b1b
Show file tree

Hide file tree

Showing 19 changed files with 168 additions and 23 deletions.
diff --git a/src/Framework/TestRunner.php b/src/Framework/TestRunner.php
@@ -194,18 +194,9 @@ public function run(TestCase $test): void
 
         ErrorHandler::instance()->disable();
 
-        if (isset($e)) {
-            if ($test->wasPrepared()) {
-                Event\Facade::emitter()->testFinished(
-                    $test->valueObjectForEvents(),
-                    $test->numberOfAssertionsPerformed()
-                );
-            }
-
-            return;
-        }
-
-        if ($this->configuration->reportUselessTests() &&
+        if (!$incomplete &&
+            !$skipped &&
+            $this->configuration->reportUselessTests() &&
             !$test->doesNotPerformAssertions() &&
             $test->numberOfAssertionsPerformed() === 0) {
             Event\Facade::emitter()->testConsideredRisky(

diff --git a/tests/end-to-end/cli/stop-on/stop-on-defect-for-error.phpt b/tests/end-to-end/cli/stop-on/stop-on-defect-for-error.phpt
@@ -37,6 +37,8 @@ Test Preparation Started (PHPUnit\TestFixture\TestRunnerStopping\ErrorTest::test
 Test Prepared (PHPUnit\TestFixture\TestRunnerStopping\ErrorTest::testOne)
 Test Errored (PHPUnit\TestFixture\TestRunnerStopping\ErrorTest::testOne)
 message
+Test Considered Risky (PHPUnit\TestFixture\TestRunnerStopping\ErrorTest::testOne)
+This test did not perform any assertions
 Test Finished (PHPUnit\TestFixture\TestRunnerStopping\ErrorTest::testOne)
 Test Runner Execution Aborted
 Test Suite Finished (PHPUnit\TestFixture\TestRunnerStopping\ErrorTest, 2 tests)

diff --git a/tests/end-to-end/cli/stop-on/stop-on-error.phpt b/tests/end-to-end/cli/stop-on/stop-on-error.phpt
@@ -37,6 +37,8 @@ Test Preparation Started (PHPUnit\TestFixture\TestRunnerStopping\ErrorTest::test
 Test Prepared (PHPUnit\TestFixture\TestRunnerStopping\ErrorTest::testOne)
 Test Errored (PHPUnit\TestFixture\TestRunnerStopping\ErrorTest::testOne)
 message
+Test Considered Risky (PHPUnit\TestFixture\TestRunnerStopping\ErrorTest::testOne)
+This test did not perform any assertions
 Test Finished (PHPUnit\TestFixture\TestRunnerStopping\ErrorTest::testOne)
 Test Runner Execution Aborted
 Test Suite Finished (PHPUnit\TestFixture\TestRunnerStopping\ErrorTest, 2 tests)

diff --git a/tests/end-to-end/event/exception-in-setup.phpt b/tests/end-to-end/event/exception-in-setup.phpt
@@ -37,6 +37,8 @@ Before Test Method Called (PHPUnit\TestFixture\Event\ExceptionInSetUpTest::setUp
 Before Test Method Finished:
 - PHPUnit\TestFixture\Event\ExceptionInSetUpTest::setUp
 Test Errored (PHPUnit\TestFixture\Event\ExceptionInSetUpTest::testOne)
+Test Considered Risky (PHPUnit\TestFixture\Event\ExceptionInSetUpTest::testOne)
+This test did not perform any assertions
 Test Suite Finished (PHPUnit\TestFixture\Event\ExceptionInSetUpTest, 1 test)
 Test Runner Execution Finished
 Test Runner Finished

diff --git a/tests/end-to-end/event/process-isolation-fatal.phpt b/tests/end-to-end/event/process-isolation-fatal.phpt
@@ -37,6 +37,8 @@ Test Preparation Started (PHPUnit\TestFixture\Event\FatalTest::testOne)
 Test Prepared (PHPUnit\TestFixture\Event\FatalTest::testOne)
 Test Errored (PHPUnit\TestFixture\Event\FatalTest::testOne)
 Call to undefined function PHPUnit\TestFixture\Event\doesNotExist()
+Test Considered Risky (PHPUnit\TestFixture\Event\FatalTest::testOne)
+This test did not perform any assertions
 Test Finished (PHPUnit\TestFixture\Event\FatalTest::testOne)
 Test Suite Finished (PHPUnit\TestFixture\Event\FatalTest, 1 test)
 Test Runner Execution Finished

diff --git a/tests/end-to-end/event/test-error.phpt b/tests/end-to-end/event/test-error.phpt
@@ -36,6 +36,8 @@ Test Preparation Started (PHPUnit\TestFixture\Event\ErrorTest::testError)
 Test Prepared (PHPUnit\TestFixture\Event\ErrorTest::testError)
 Test Errored (PHPUnit\TestFixture\Event\ErrorTest::testError)
 message
+Test Considered Risky (PHPUnit\TestFixture\Event\ErrorTest::testError)
+This test did not perform any assertions
 Test Finished (PHPUnit\TestFixture\Event\ErrorTest::testError)
 Test Suite Finished (PHPUnit\TestFixture\Event\ErrorTest, 1 test)
 Test Runner Execution Finished

diff --git a/tests/end-to-end/event/test-registered-failure-interface.phpt b/tests/end-to-end/event/test-registered-failure-interface.phpt
@@ -42,6 +42,8 @@ Test Preparation Started (PHPUnit\TestFixture\Event\CustomFailureInterfaceTest::
 Test Prepared (PHPUnit\TestFixture\Event\CustomFailureInterfaceTest::testTwo)
 Test Errored (PHPUnit\TestFixture\Event\CustomFailureInterfaceTest::testTwo)
 this should be treated as an error
+Test Considered Risky (PHPUnit\TestFixture\Event\CustomFailureInterfaceTest::testTwo)
+This test did not perform any assertions
 Test Finished (PHPUnit\TestFixture\Event\CustomFailureInterfaceTest::testTwo)
 Test Suite Finished (PHPUnit\TestFixture\Event\CustomFailureInterfaceTest, 2 tests)
 Test Runner Execution Finished

diff --git a/tests/end-to-end/generic/exception-stack.phpt b/tests/end-to-end/generic/exception-stack.phpt
@@ -4,6 +4,7 @@ phpunit ../../_files/ExceptionStackTest.php
 <?php declare(strict_types=1);
 $_SERVER['argv'][] = '--do-not-cache-result';
 $_SERVER['argv'][] = '--no-configuration';
+$_SERVER['argv'][] = '--dont-report-useless-tests';
 $_SERVER['argv'][] = __DIR__ . '/../../_files/ExceptionStackTest.php';
 
 require_once __DIR__ . '/../../bootstrap.php';

diff --git a/tests/end-to-end/generic/outcome-and-issues.phpt b/tests/end-to-end/generic/outcome-and-issues.phpt
@@ -61,13 +61,28 @@ Failed asserting that false is true.
 
 --
 
-There was 1 risky test:
+There were 4 risky tests:
 
 1) PHPUnit\TestFixture\OutcomesAndIssuesTest::testSuccessWithRisky
 This test did not perform any assertions
 
 %sOutcomesAndIssuesTest.php:%d
 
+2) PHPUnit\TestFixture\OutcomesAndIssuesTest::testErrorWithDeprecation
+This test did not perform any assertions
+
+%sOutcomesAndIssuesTest.php:%d
+
+3) PHPUnit\TestFixture\OutcomesAndIssuesTest::testErrorWithNotice
+This test did not perform any assertions
+
+%sOutcomesAndIssuesTest.php:%d
+
+4) PHPUnit\TestFixture\OutcomesAndIssuesTest::testErrorWithWarning
+This test did not perform any assertions
+
+%sOutcomesAndIssuesTest.php:%d
+
 --
 
 There were 3 incomplete tests:
@@ -203,4 +218,4 @@ deprecation message
 %sOutcomesAndIssuesTest.php:%d
 
 ERRORS!
-Tests: 17, Assertions: 7, Errors: 3, Failures: 3, Warnings: 5, Deprecations: 5, Notices: 5, Skipped: 3, Incomplete: 3, Risky: 1.
+Tests: 17, Assertions: 7, Errors: 3, Failures: 3, Warnings: 5, Deprecations: 5, Notices: 5, Skipped: 3, Incomplete: 3, Risky: 4.
diff --git a/tests/end-to-end/generic/transform-exception-hook-method.phpt b/tests/end-to-end/generic/transform-exception-hook-method.phpt
@@ -3,6 +3,7 @@ https://github.com/sebastianbergmann/phpunit/issues/5300
 --FILE--
 <?php declare(strict_types=1);
 $_SERVER['argv'][] = '--do-not-cache-result';
+$_SERVER['argv'][] = '--dont-report-useless-tests';
 $_SERVER['argv'][] = '--configuration';
 $_SERVER['argv'][] = __DIR__ . '/../_files/transform-exception-hook-method';
 

diff --git a/tests/end-to-end/logging/log-junit-with-progress-with-errors.phpt b/tests/end-to-end/logging/log-junit-with-progress-with-errors.phpt
@@ -11,6 +11,7 @@ $logfile = tempnam(sys_get_temp_dir(), __FILE__);
 
 $_SERVER['argv'][] = '--do-not-cache-result';
 $_SERVER['argv'][] = '--no-configuration';
+$_SERVER['argv'][] = '--dont-report-useless-tests';
 $_SERVER['argv'][] = '--log-junit';
 $_SERVER['argv'][] = $logfile;
 $_SERVER['argv'][] = __DIR__ . '/_files/TypeErrorTest.php';

diff --git a/tests/end-to-end/logging/log-teamcity.phpt b/tests/end-to-end/logging/log-teamcity.phpt
@@ -31,6 +31,8 @@ require_once __DIR__ . '/../../bootstrap.php';
 
 ##teamcity[testFailed name='testError' message='RuntimeException' details='%sStatusTest.php:%d|n' duration='%d' flowId='%d']
 
+##teamcity[testFailed name='testError' message='This test did not perform any assertions' details='' duration='%d' flowId='%d']
+
 ##teamcity[testFinished name='testError' duration='%d' flowId='%d']
 
 ##teamcity[testStarted name='testIncomplete' locationHint='php_qn://%sStatusTest.php::\PHPUnit\SelfTest\Basic\StatusTest::testIncomplete' flowId='%d']
@@ -65,6 +67,8 @@ require_once __DIR__ . '/../../bootstrap.php';
 
 ##teamcity[testFailed name='testErrorWithMessage' message='RuntimeException: error with custom message' details='%sStatusTest.php:%d|n' duration='%d' flowId='%d']
 
+##teamcity[testFailed name='testErrorWithMessage' message='This test did not perform any assertions' details='' duration='%d' flowId='%d']
+
 ##teamcity[testFinished name='testErrorWithMessage' duration='%d' flowId='%d']
 
 ##teamcity[testStarted name='testIncompleteWithMessage' locationHint='php_qn://%sStatusTest.php::\PHPUnit\SelfTest\Basic\StatusTest::testIncompleteWithMessage' flowId='%d']

diff --git a/tests/end-to-end/logging/teamcity-directory.phpt b/tests/end-to-end/logging/teamcity-directory.phpt
@@ -26,8 +26,12 @@ Runtime: %s
 
 ##teamcity[testFailed name='testOneWithSetUpException' message='RuntimeException: throw exception in setUp' details='%stests/end-to-end/_files/basic/unit/SetUpTest.php:%d|n' duration='%d' flowId='%d']
 
+##teamcity[testFailed name='testOneWithSetUpException' message='This test did not perform any assertions' details='' duration='%d' flowId='%d']
+
 ##teamcity[testFailed name='testTwoWithSetUpException' message='RuntimeException: throw exception in setUp' details='%stests/end-to-end/_files/basic/unit/SetUpTest.php:%d|n' duration='%d' flowId='%d']
 
+##teamcity[testFailed name='testTwoWithSetUpException' message='This test did not perform any assertions' details='' duration='%d' flowId='%d']
+
 ##teamcity[testSuiteFinished name='PHPUnit\SelfTest\Basic\SetUpTest' flowId='%d']
 
 ##teamcity[testSuiteStarted name='PHPUnit\SelfTest\Basic\StatusTest' locationHint='php_qn://%stests/end-to-end/_files/basic/unit/StatusTest.php::\PHPUnit\SelfTest\Basic\StatusTest' flowId='%d']
@@ -38,14 +42,16 @@ Runtime: %s
 
 ##teamcity[testStarted name='testFailure' locationHint='php_qn://%stests/end-to-end/_files/basic/unit/StatusTest.php::\PHPUnit\SelfTest\Basic\StatusTest::testFailure' flowId='%d']
 
-##teamcity[testFailed name='testFailure' message='Failed asserting that false is true.' details='%stests/end-to-end/_files/basic/unit/StatusTest.php:%d|n' duration='%d' flowId='%d']
+##teamcity[testFailed name='testFailure' message='Failed asserting that false is true.' details='%stests/end-to-end/_files/basic/unit/StatusTest.php:%d|n' duration='1' flowId='%d']
 
 ##teamcity[testFinished name='testFailure' duration='%d' flowId='%d']
 
 ##teamcity[testStarted name='testError' locationHint='php_qn://%stests/end-to-end/_files/basic/unit/StatusTest.php::\PHPUnit\SelfTest\Basic\StatusTest::testError' flowId='%d']
 
 ##teamcity[testFailed name='testError' message='RuntimeException' details='%stests/end-to-end/_files/basic/unit/StatusTest.php:%d|n' duration='%d' flowId='%d']
 
+##teamcity[testFailed name='testError' message='This test did not perform any assertions' details='' duration='%d' flowId='%d']
+
 ##teamcity[testFinished name='testError' duration='%d' flowId='%d']
 
 ##teamcity[testStarted name='testIncomplete' locationHint='php_qn://%stests/end-to-end/_files/basic/unit/StatusTest.php::\PHPUnit\SelfTest\Basic\StatusTest::testIncomplete' flowId='%d']
@@ -80,6 +86,8 @@ Runtime: %s
 
 ##teamcity[testFailed name='testErrorWithMessage' message='RuntimeException: error with custom message' details='%stests/end-to-end/_files/basic/unit/StatusTest.php:%d|n' duration='%d' flowId='%d']
 
+##teamcity[testFailed name='testErrorWithMessage' message='This test did not perform any assertions' details='' duration='%d' flowId='%d']
+
 ##teamcity[testFinished name='testErrorWithMessage' duration='%d' flowId='%d']
 
 ##teamcity[testStarted name='testIncompleteWithMessage' locationHint='php_qn://%stests/end-to-end/_files/basic/unit/StatusTest.php::\PHPUnit\SelfTest\Basic\StatusTest::testIncompleteWithMessage' flowId='%d']
@@ -163,17 +171,37 @@ Failed asserting that false is true.
 
 --
 
-There were 2 risky tests:
+There were 6 risky tests:
+
+1) PHPUnit\SelfTest\Basic\SetUpTest::testOneWithSetUpException
+This test did not perform any assertions
 
-1) PHPUnit\SelfTest\Basic\StatusTest::testRisky
+%s/SetUpTest.php:%d
+
+2) PHPUnit\SelfTest\Basic\SetUpTest::testTwoWithSetUpException
+This test did not perform any assertions
+
+%s/SetUpTest.php:%d
+
+3) PHPUnit\SelfTest\Basic\StatusTest::testError
+This test did not perform any assertions
+
+%s/StatusTest.php:%d
+
+4) PHPUnit\SelfTest\Basic\StatusTest::testRisky
+This test did not perform any assertions
+
+%s/StatusTest.php:%d
+
+5) PHPUnit\SelfTest\Basic\StatusTest::testErrorWithMessage
 This test did not perform any assertions
 
 %s/StatusTest.php:%d
 
-2) PHPUnit\SelfTest\Basic\StatusTest::testRiskyWithMessage
+6) PHPUnit\SelfTest\Basic\StatusTest::testRiskyWithMessage
 This test did not perform any assertions
 
 %s/StatusTest.php:%d
 
 ERRORS!
-Tests: 18, Assertions: 6, Errors: 5, Failures: 2, Skipped: 3, Incomplete: 2, Risky: 2.
+Tests: 18, Assertions: 6, Errors: 5, Failures: 2, Skipped: 3, Incomplete: 2, Risky: 6.
diff --git a/tests/end-to-end/logging/teamcity-file.phpt b/tests/end-to-end/logging/teamcity-file.phpt
@@ -34,6 +34,8 @@ Runtime: %s
 
 ##teamcity[testFailed name='testError' message='RuntimeException' details='%sStatusTest.php:%d|n' duration='%d' flowId='%d']
 
+##teamcity[testFailed name='testError' message='This test did not perform any assertions' details='' duration='%d' flowId='%d']
+
 ##teamcity[testFinished name='testError' duration='%d' flowId='%d']
 
 ##teamcity[testStarted name='testIncomplete' locationHint='php_qn://%sStatusTest.php::\PHPUnit\SelfTest\Basic\StatusTest::testIncomplete' flowId='%d']
@@ -68,6 +70,8 @@ Runtime: %s
 
 ##teamcity[testFailed name='testErrorWithMessage' message='RuntimeException: error with custom message' details='%sStatusTest.php:%d|n' duration='%d' flowId='%d']
 
+##teamcity[testFailed name='testErrorWithMessage' message='This test did not perform any assertions' details='' duration='%d' flowId='%d']
+
 ##teamcity[testFinished name='testErrorWithMessage' duration='%d' flowId='%d']
 
 ##teamcity[testStarted name='testIncompleteWithMessage' locationHint='php_qn://%sStatusTest.php::\PHPUnit\SelfTest\Basic\StatusTest::testIncompleteWithMessage' flowId='%d']
@@ -122,17 +126,27 @@ Failed asserting that false is true.
 
 --
 
-There were 2 risky tests:
+There were 4 risky tests:
+
+1) PHPUnit\SelfTest\Basic\StatusTest::testError
+This test did not perform any assertions
+
+%s/StatusTest.php:%d
+
+2) PHPUnit\SelfTest\Basic\StatusTest::testRisky
+This test did not perform any assertions
+
+%s/StatusTest.php:%d
 
-1) PHPUnit\SelfTest\Basic\StatusTest::testRisky
+3) PHPUnit\SelfTest\Basic\StatusTest::testErrorWithMessage
 This test did not perform any assertions
 
 %s/StatusTest.php:%d
 
-2) PHPUnit\SelfTest\Basic\StatusTest::testRiskyWithMessage
+4) PHPUnit\SelfTest\Basic\StatusTest::testRiskyWithMessage
 This test did not perform any assertions
 
 %s/StatusTest.php:%d
 
 ERRORS!
-Tests: 13, Assertions: 4, Errors: 2, Failures: 2, Skipped: 3, Incomplete: 2, Risky: 2.
+Tests: 13, Assertions: 4, Errors: 2, Failures: 2, Skipped: 3, Incomplete: 2, Risky: 4.
diff --git a/tests/end-to-end/logging/teamcity-inner-exceptions.phpt b/tests/end-to-end/logging/teamcity-inner-exceptions.phpt
@@ -4,6 +4,7 @@ phpunit --log-teamcity php://stdout ../../_files/ExceptionStackTest.php
 <?php declare(strict_types=1);
 $_SERVER['argv'][] = '--do-not-cache-result';
 $_SERVER['argv'][] = '--no-configuration';
+$_SERVER['argv'][] = '--dont-report-useless-tests';
 $_SERVER['argv'][] = '--no-output';
 $_SERVER['argv'][] = '--log-teamcity';
 $_SERVER['argv'][] = 'php://stdout';

diff --git a/tests/end-to-end/regression/4376.phpt b/tests/end-to-end/regression/4376.phpt
@@ -3,6 +3,7 @@ https://github.com/sebastianbergmann/phpunit/issues/4376
 --FILE--
 <?php declare(strict_types=1);
 $_SERVER['argv'][] = '--do-not-cache-result';
+$_SERVER['argv'][] = '--dont-report-useless-tests';
 $_SERVER['argv'][] = '--configuration';
 $_SERVER['argv'][] = __DIR__ . '/4376/';
 

diff --git a/tests/end-to-end/regression/5340.phpt b/tests/end-to-end/regression/5340.phpt
@@ -0,0 +1,48 @@
+--TEST--
+https://github.com/sebastianbergmann/phpunit/issues/5340
+--FILE--
+<?php declare(strict_types=1);
+$_SERVER['argv'][] = '--do-not-cache-result';
+$_SERVER['argv'][] = '--no-configuration';
+$_SERVER['argv'][] = '--disallow-test-output';
+$_SERVER['argv'][] = __DIR__ . '/5340/Issue5340Test.php';
+
+require __DIR__ . '/../../bootstrap.php';
+
+(new PHPUnit\TextUI\Application)->run($_SERVER['argv']);
+--EXPECTF--
+PHPUnit %s by Sebastian Bergmann and contributors.
+
+%s
+
+output printed from passing test
+R
+output printed from failing test
+F                                                                  2 / 2 (100%)
+
+Time: %s, Memory: %s MB
+
+There was 1 failure:
+
+1) Issue5340Test::testTwo
+Failed asserting that false is true.
+
+%s
+
+--
+
+There were 2 risky tests:
+
+1) Issue5340Test::testOne
+This test printed output: output printed from passing test
+
+%s/Issue5340Test.php:%d
+
+2) Issue5340Test::testTwo
+This test printed output: 
+output printed from failing test
+
+%s/Issue5340Test.php:%d
+
+FAILURES!
+Tests: 2, Assertions: 2, Failures: 1, Risky: 2.
diff --git a/tests/end-to-end/regression/5340/Issue5340Test.php b/tests/end-to-end/regression/5340/Issue5340Test.php
@@ -0,0 +1,27 @@
+<?php declare(strict_types=1);
+/*
+ * This file is part of PHPUnit.
+ *
+ * (c) Sebastian Bergmann <sebastian@phpunit.de>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+use PHPUnit\Framework\TestCase;
+
+final class Issue5340Test extends TestCase
+{
+    public function testOne(): void
+    {
+        print 'output printed from passing test' . PHP_EOL;
+
+        $this->assertTrue(true);
+    }
+
+    public function testTwo(): void
+    {
+        print PHP_EOL . 'output printed from failing test' . PHP_EOL;
+
+        $this->assertTrue(false);
+    }
+}
diff --git a/tests/end-to-end/regression/74.phpt b/tests/end-to-end/regression/74.phpt
@@ -4,6 +4,7 @@ GH-74: catchable fatal error in 3.5
 <?php declare(strict_types=1);
 $_SERVER['argv'][] = '--do-not-cache-result';
 $_SERVER['argv'][] = '--no-configuration';
+$_SERVER['argv'][] = '--dont-report-useless-tests';
 $_SERVER['argv'][] = '--process-isolation';
 $_SERVER['argv'][] = __DIR__ . '/74/Issue74Test.php';