Skip to content

Commit

Permalink
Merge 079c69f into 44c506b
Browse files Browse the repository at this point in the history
  • Loading branch information
@yoanm
yoanm committed Apr 2, 2023
2 parents 44c506b + 079c69f commit e42749d
Show file tree
Hide file tree
Showing 7 changed files with 146 additions and 108 deletions.
8 changes: 6 additions & 2 deletions .github/workflows/CI.yml
View file
Expand Up @@ -90,6 +90,7 @@ jobs:
run: |
composer require -W ${{ env.COMPOSER_OPTIONS }} ${{ matrix.composer-flag }} \
phpunit/phpunit:^${{ matrix.phpunit-version }} \
phpunit/php-code-coverage:^${{ matrix.phpunit-version }} \
&& composer update ${{ matrix.composer-flag }} \
&& make build
Expand Down Expand Up @@ -155,7 +156,7 @@ jobs:
run: make build

- name: ComposerRequireChecker
uses: docker://webfactory/composer-require-checker:3.2.0
uses: docker://webfactory/composer-require-checker:4.5.0

- name: Dependencies check
if: ${{ github.event_name == 'pull_request' }}
Expand Down Expand Up @@ -232,8 +233,11 @@ jobs:

- name: Build
run: |
composer require -W ${{ env.COMPOSER_OPTIONS }} \
composer config minimum-stability dev \
&& composer require -W ${{ env.COMPOSER_OPTIONS }} \
phpunit/phpunit:^${{ matrix.phpunit-version }} \
phpunit/php-code-coverage:^${{ matrix.phpunit-version }} \
phpspec/prophecy-phpunit:">=2" \
&& composer update \
&& make build
Expand Down
2 changes: 1 addition & 1 deletion composer.json
View file
Expand Up @@ -29,7 +29,7 @@
"require": {
"php": "^8.0",
"phpunit/phpunit": "^9.0",
"phpunit/php-code-coverage": "^9.2.4"
"phpunit/php-code-coverage": "^9.0"
},
"require-dev": {
"squizlabs/php_codesniffer": "^3.5",
Expand Down
11 changes: 10 additions & 1 deletion features/bootstrap/FeatureContext.php
View file
Expand Up @@ -61,6 +61,11 @@ public function iRunPhpunitTestSuite()
$this->process = new Process($argList, null, $env);

$this->process->run();

//var_dump(['env' => $env]);
//var_dump(['argList' => $argList]);
//var_dump(['output' => $this->process->getOutput()]);
//var_dump(['error' => $this->process->getErrorOutput()]);
}

/**
Expand All @@ -70,7 +75,11 @@ public function iShouldHaveXFailures($expectedFailureCount)
{
$output = $this->process->getOutput();

$failureCount = preg_match_all(sprintf('#There was %s failure#', $expectedFailureCount), $output);
$matched = preg_match_all('#There (?:was|were) (\d+) failure#', $output, $matches);
if ($matched === false) {
throw new \Exception('Error when executing preg_match_all');
}
$failureCount = $matches[1][0];
if ($failureCount != $expectedFailureCount) {
throw new \Exception(sprintf('Found %d failure, but %d expected', $failureCount, $expectedFailureCount));
}
Expand Down
3 changes: 3 additions & 0 deletions features/demo_app/tests/RiskyOutputTest.php
View file
@@ -1,5 +1,8 @@
<?php

/**
* @coversNothing
*/
class RiskyOutputTest extends \PHPUnit\Framework\TestCase
{
/**
Expand Down
77 changes: 41 additions & 36 deletions phpunit.xml.dist
View file
@@ -1,40 +1,45 @@
<?xml version="1.0" encoding="UTF-8"?>

<!-- https://phpunit.de/manual/current/en/appendixes.configuration.html -->
<phpunit xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:noNamespaceSchemaLocation="http://schema.phpunit.de/4.8/phpunit.xsd"
backupGlobals="true"
colors="true"
convertErrorsToExceptions="true"
convertNoticesToExceptions="true"
convertWarningsToExceptions="true"
stopOnFailure="true"
stopOnRisky="true"
beStrictAboutTestsThatDoNotTestAnything="true"
checkForUnintentionallyCoveredCode="true"
beStrictAboutOutputDuringTests="true"
beStrictAboutChangesToGlobalState="true"
forceCoversAnnotation="true"
bootstrap="vendor/autoload.php"
<phpunit xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="https://schema.phpunit.de/9.3/phpunit.xsd"
backupGlobals="true"
backupStaticAttributes="false"
colors="true"
processIsolation="true"

convertErrorsToExceptions="true"
convertNoticesToExceptions="true"
convertWarningsToExceptions="true"

stopOnError="true"
stopOnFailure="true"
stopOnRisky="true"

beStrictAboutTestsThatDoNotTestAnything="true"
checkForUnintentionallyCoveredCode="true"
beStrictAboutOutputDuringTests="true"
beStrictAboutChangesToGlobalState="true"

forceCoversAnnotation="true"

bootstrap="vendor/autoload.php"
>
<listeners>
<listener class="Yoanm\PhpUnitExtended\Listener\YoanmTestsStrategyListener"/>
</listeners>


<testsuites>
<testsuite name="technical">
<directory>tests/Technical/Unit/*</directory> <!-- launch unit before => faster than integration -->
<directory>tests/Technical/Integration/*</directory>
</testsuite>
<testsuite name="functional"> <!-- defined functional after technical => longer than technical -->
<directory>tests/Functional/*</directory>
</testsuite>
</testsuites>

<filter>
<whitelist>
<directory>src</directory>
</whitelist>
</filter>

<listeners>
<listener class="Yoanm\PhpUnitExtended\Listener\YoanmTestsStrategyListener"/>
</listeners>

<testsuites>
<testsuite name="technical">
<directory>tests/Technical</directory>
</testsuite>
<testsuite name="functional">
<directory>tests/Functional</directory>
</testsuite>
</testsuites>

<coverage>
<include>
<directory>src</directory>
</include>
</coverage>
</phpunit>
101 changes: 53 additions & 48 deletions src/Listener/RiskyToFailedListener.php
View file
Expand Up @@ -2,11 +2,15 @@
namespace Yoanm\PhpUnitExtended\Listener;

use PHPUnit\Framework\AssertionFailedError;
use PHPUnit\Framework\CoveredCodeNotExecutedException;
use PHPUnit\Framework\InvalidCoversTargetException;
use PHPUnit\Framework\MissingCoversAnnotationException;
use PHPUnit\Framework\OutputError;
use PHPUnit\Framework\Test;
use PHPUnit\Framework\TestCase;
use PHPUnit\Framework\TestListener;
use PHPUnit\Framework\TestListenerDefaultImplementation;
use PHPUnit\Framework\TestResult;
use PHPUnit\Framework\UnintentionallyCoveredCodeError;
use PHPUnit\Framework\Warning;

Expand Down Expand Up @@ -34,66 +38,67 @@ public function addRiskyTest(Test $test, \Throwable $e, float $time) : void
}

/**
* @param Test $test
* @param \Throwable $e
* @param Test $test
* @param \Throwable $exception
* @param $time
*/
protected function addErrorIfNeeded(Test $test, \Throwable $e, $time)
protected function addErrorIfNeeded(Test $test, \Throwable $exception, $time)
{
/* Must be TestCase instance to have access to "getTestResultObject" method */
if ($test instanceof TestCase && $test->getTestResultObject() !== null) {
$reason = $this->getErrorReason($e);
if (null !== $reason) {
$test->getTestResultObject()->addFailure(
$test,
new AssertionFailedError(
sprintf(
"Strict mode - %s :\n%s",
$reason,
$e->getMessage()
)
),
$time
);
if ($test instanceof TestCase) {
if (!$test->getTestResultObject()) {
$test->setTestResultObject(new TestResult());
}
$test->getTestResultObject()->addFailure(
$test,
new AssertionFailedError(
sprintf(
"Strict mode - %s :\n%s",
$this->getErrorReason($exception),
$exception->getMessage()
)
),
$time
);
}
}

/**
* @param \Throwable $e
*
* @return null|string
*/
protected function getErrorReason(\Throwable $e)
protected function getErrorReason(\Throwable $exception): string
{
$reason = null;
switch (true) {
if ($exception instanceof OutputError) {
/* beStrictAboutOutputDuringTests="true" */
case $e instanceof OutputError:
$reason = 'No output during test';
break;
return 'No output during test';
} elseif ($exception instanceof UnintentionallyCoveredCodeError
|| $exception instanceof InvalidCoversTargetException
) {
/* checkForUnintentionallyCoveredCode="true" */
case $e instanceof UnintentionallyCoveredCodeError:
$reason = 'Executed code must be defined with @covers and @uses annotations';
break;
default:
if (preg_match('#\-\-\- Global variables before the test#', $e->getMessage())) {
/* beStrictAboutChangesToGlobalState="true" (no specific exception) for globals */
$reason = 'No global variable manipulation during test';
} elseif (preg_match('#\-\-\- Static attributes before the test#', $e->getMessage())) {
/* beStrictAboutChangesToGlobalState="true" (no specific exception) for static var */
/* Only when beStrictAboutChangesToGlobalState="true" */
$reason = 'No static attribute manipulation during test';
} elseif (preg_match('#This test did not perform any assertions#', $e->getMessage())) {
/* beStrictAboutTestsThatDoNotTestAnything="true" (no specific exception) */
$reason = 'No test that do not test anything';
} elseif (preg_match('#"@covers [^"]+" is invalid#', $e->getMessage())) {
/* forceCoversAnnotation="true" (no specific exception) */
$reason = 'Only executed code must be defined with @covers and @uses annotations';
}
break;
return 'Executed code must be defined with @covers and @uses annotations';
} elseif (str_contains($exception->getMessage(), '--- Global variables before the test')) {
/* beStrictAboutChangesToGlobalState="true" (no specific exception) for globals */
return 'No global variable manipulation during test';
} elseif (str_contains($exception->getMessage(), '--- Static attributes before the test')) {
/* beStrictAboutChangesToGlobalState="true" (no specific exception) for static var */
/* Only when beStrictAboutChangesToGlobalState="true" */
return 'No static attribute manipulation during test';
} elseif (str_contains($exception->getMessage(), 'This test did not perform any assertions')) {
/* beStrictAboutTestsThatDoNotTestAnything="true" (no specific exception) */
return 'No test that do not test anything';
} elseif ($exception instanceof CoveredCodeNotExecutedException
|| preg_match('#"@covers [^"]+" is invalid#', $exception->getMessage())
) {
/* forceCoversAnnotation="true" (no specific exception) */
return 'Only executed code must be defined with @covers and @uses annotations';
} elseif ($exception instanceof MissingCoversAnnotationException
|| str_contains(
$exception->getMessage(),
'This test does not have a @covers annotation but is expected to have one'
)
) {
/* forceCoversAnnotation="true" (no specific exception) */
return 'Missing @covers or @coversNothing annotation';
}

return $reason;
// Always return an error even if it's not a known/managed error
return $exception->getMessage();
}
}
52 changes: 32 additions & 20 deletions tests/Functional/Listener/RiskyToFailedListenerTest.php
View file
@@ -1,6 +1,9 @@
<?php
namespace Tests\Functional\Listener;

use PHPUnit\Framework\CoveredCodeNotExecutedException;
use PHPUnit\Framework\InvalidCoversTargetException;
use PHPUnit\Framework\MissingCoversAnnotationException;
use PHPUnit\Framework\OutputError;
use PHPUnit\Framework\RiskyTestError;
use PHPUnit\Framework\TestCase;
Expand Down Expand Up @@ -50,32 +53,24 @@ public function testShouldHandleBadCoverageTagWarning()
*
* @param $exceptionClass
* @param $expectedReason
* @param bool|true $called
* @param null $exceptionMessage
*/
public function testShouldHandleRiskyTestWith(
$exceptionClass,
$expectedReason,
$called,
$exceptionMessage = 'my default exception message'
) {
$time = 0.3;
$test = new TestCaseMock();
$exception = new $exceptionClass($exceptionMessage);

$test->setTestResultObject(new TestResult());

$this->listener->addRiskyTest($test, $exception, $time);

$failures = $test->getTestResultObject()->failures();
if ($called) {
$this->assertCount(1, $failures);
$failure = array_shift($failures);
$this->assertInstanceOf(TestFailure::class, $failure);
$this->assertStringContainsString($exceptionMessage, $failure->getExceptionAsString());
} else {
$this->assertCount(0, $failures);
}
$this->assertCount(1, $failures);
$failure = array_shift($failures);
$this->assertInstanceOf(TestFailure::class, $failure);
$this->assertStringContainsString($exceptionMessage, $failure->getExceptionAsString());
}

/**
Expand All @@ -87,35 +82,52 @@ public function getExceptionMessageProvider()
'Output exception' => [
'exceptionClass' => OutputError::class,
'expectedMessage' => 'No output during test',
'called' => true,
],
'Coverage exception' => [
'Coverage overflow - UnintentionallyCoveredCodeError' => [
'exceptionClass' => UnintentionallyCoveredCodeError::class,
'expectedMessage' => 'Executed code must be defined with @covers and @uses annotations',
'called' => true,
],
'Coverage overflow - InvalidCoversTargetException instance' => [
'exceptionClass' => InvalidCoversTargetException::class,
'expectedMessage' => 'Executed code must be defined with @covers and @uses annotations',
],
'Globals manipulation - globals' => [
'exceptionClass' => RiskyTestError::class,
'expectedMessage' => 'No global variable manipulation during test',
'called' => true,
'exceptionMessage' => '--- Global variables before the test',
],
'Globals manipulation - static' => [
'exceptionClass' => RiskyTestError::class,
'expectedMessage' => 'No static attribute manipulation during test',
'called' => true,
'exceptionMessage' => '--- Static attributes before the test',
],
'Test nothing' => [
'exceptionClass' => RiskyTestError::class,
'expectedMessage' => 'No test that do not test anything',
'called' => true,
'exceptionMessage' => 'This test did not perform any assertions',
],
'Coverage exception - CoveredCodeNotExecutedException instance' => [
'exceptionClass' => CoveredCodeNotExecutedException::class,
'expectedMessage' => 'Only executed code must be defined with @covers and @uses annotations',
],
'Coverage exception - by message' => [
'exceptionClass' => RiskyTestError::class,
'expectedMessage' => 'Only executed code must be defined with @covers and @uses annotations',
'exceptionMessage' => '"@covers A\Class" is invalid'
],
'Missing @covers - MissingCoversAnnotationException instance' => [
'exceptionClass' => MissingCoversAnnotationException::class,
'expectedMessage' => 'Missing @covers or @coversNothing annotation',
],
'Missing @covers - by message' => [
'exceptionClass' => RiskyTestError::class,
'expectedMessage' => 'Missing @covers or @coversNothing annotation',
'exceptionMessage' => '"@covers A\Class" is invalid'
],
'other exceptions' => [
'exceptionClass' => \Exception::class,
'expectedMessage' => 'Risky test',
'called' => false,
'expectedMessage' => 'Arghh !',
'exceptionMessage' => 'Arghh !',
],
];
}
Expand Down

0 comments on commit e42749d

Please sign in to comment.