From 99b3a654844d0d49433cf237feb6996bb41d6733 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20M=C3=B6ller?= Date: Mon, 22 Jun 2020 10:42:43 +0200 Subject: [PATCH] Enhancement: Allow failing tests when test suite is empty --- phpunit.xsd | 1 + src/TextUI/CliArguments/Builder.php | 8 +++++++ src/TextUI/CliArguments/Configuration.php | 22 ++++++++++++++++++- src/TextUI/CliArguments/Mapper.php | 4 ++++ src/TextUI/TestRunner.php | 5 +++++ src/TextUI/XmlConfiguration/Loader.php | 1 + .../XmlConfiguration/PHPUnit/PHPUnit.php | 13 ++++++++++- tests/unit/TextUI/XmlConfigurationTest.php | 2 ++ 8 files changed, 54 insertions(+), 2 deletions(-) diff --git a/phpunit.xsd b/phpunit.xsd index 278469872ef..72cd8f09dcc 100644 --- a/phpunit.xsd +++ b/phpunit.xsd @@ -228,6 +228,7 @@ + diff --git a/src/TextUI/CliArguments/Builder.php b/src/TextUI/CliArguments/Builder.php index e7f79ed1574..0f34b1c17e6 100644 --- a/src/TextUI/CliArguments/Builder.php +++ b/src/TextUI/CliArguments/Builder.php @@ -91,6 +91,7 @@ final class Builder 'stop-on-incomplete', 'stop-on-risky', 'stop-on-skipped', + 'fail-on-empty-test-suite', 'fail-on-warning', 'fail-on-risky', 'strict-coverage', @@ -162,6 +163,7 @@ public function fromParameters(array $parameters, array $additionalLongOptions): $executionOrderDefects = null; $extensions = []; $unavailableExtensions = []; + $failOnEmptyTestSuite = null; $failOnIncomplete = null; $failOnRisky = null; $failOnSkipped = null; @@ -496,6 +498,11 @@ public function fromParameters(array $parameters, array $additionalLongOptions): break; + case '--fail-on-empty-test-suite': + $failOnEmptyTestSuite = true; + + break; + case '--fail-on-incomplete': $failOnIncomplete = true; @@ -764,6 +771,7 @@ public function fromParameters(array $parameters, array $additionalLongOptions): $executionOrderDefects, $extensions, $unavailableExtensions, + $failOnEmptyTestSuite, $failOnIncomplete, $failOnRisky, $failOnSkipped, diff --git a/src/TextUI/CliArguments/Configuration.php b/src/TextUI/CliArguments/Configuration.php index 56c2f607df9..d9b0d9da4dd 100644 --- a/src/TextUI/CliArguments/Configuration.php +++ b/src/TextUI/CliArguments/Configuration.php @@ -182,6 +182,11 @@ final class Configuration */ private $unavailableExtensions; + /** + * @var ?bool + */ + private $failOnEmptyTestSuite; + /** * @var ?bool */ @@ -435,7 +440,7 @@ final class Configuration /** * @param null|int|string $columns */ - public function __construct(?string $argument, ?string $atLeastVersion, ?bool $backupGlobals, ?bool $backupStaticAttributes, ?bool $beStrictAboutChangesToGlobalState, ?bool $beStrictAboutResourceUsageDuringSmallTests, ?string $bootstrap, ?bool $cacheResult, ?string $cacheResultFile, ?bool $checkVersion, ?string $colors, $columns, ?string $configuration, ?string $coverageClover, ?string $coverageCrap4J, ?string $coverageHtml, ?string $coveragePhp, ?string $coverageText, ?bool $coverageTextShowUncoveredFiles, ?bool $coverageTextShowOnlySummary, ?string $coverageXml, ?bool $debug, ?int $defaultTimeLimit, ?bool $disableCodeCoverageIgnore, ?bool $disallowTestOutput, ?bool $disallowTodoAnnotatedTests, ?bool $enforceTimeLimit, ?array $excludeGroups, ?int $executionOrder, ?int $executionOrderDefects, ?array $extensions, ?array $unavailableExtensions, ?bool $failOnIncomplete, ?bool $failOnRisky, ?bool $failOnSkipped, ?bool $failOnWarning, ?string $filter, ?bool $generateConfiguration, ?array $groups, ?bool $help, ?string $includePath, ?array $iniSettings, ?string $junitLogfile, ?bool $listGroups, ?bool $listSuites, ?bool $listTests, ?string $listTestsXml, ?string $loader, ?bool $noCoverage, ?bool $noExtensions, ?bool $noInteraction, ?bool $noLogging, ?string $printer, ?bool $processIsolation, ?int $randomOrderSeed, ?int $repeat, ?bool $reportUselessTests, ?bool $resolveDependencies, ?bool $reverseList, ?bool $stderr, ?bool $strictCoverage, ?bool $stopOnDefect, ?bool $stopOnError, ?bool $stopOnFailure, ?bool $stopOnIncomplete, ?bool $stopOnRisky, ?bool $stopOnSkipped, ?bool $stopOnWarning, ?string $teamcityLogfile, ?array $testdoxExcludeGroups, ?array $testdoxGroups, ?string $testdoxHtmlFile, ?string $testdoxTextFile, ?string $testdoxXmlFile, ?array $testSuffixes, ?string $testSuite, array $unrecognizedOptions, ?string $unrecognizedOrderBy, ?bool $useDefaultConfiguration, ?bool $verbose, ?bool $version, ?array $coverageFilter, ?string $xdebugFilterFile) + public function __construct(?string $argument, ?string $atLeastVersion, ?bool $backupGlobals, ?bool $backupStaticAttributes, ?bool $beStrictAboutChangesToGlobalState, ?bool $beStrictAboutResourceUsageDuringSmallTests, ?string $bootstrap, ?bool $cacheResult, ?string $cacheResultFile, ?bool $checkVersion, ?string $colors, $columns, ?string $configuration, ?string $coverageClover, ?string $coverageCrap4J, ?string $coverageHtml, ?string $coveragePhp, ?string $coverageText, ?bool $coverageTextShowUncoveredFiles, ?bool $coverageTextShowOnlySummary, ?string $coverageXml, ?bool $debug, ?int $defaultTimeLimit, ?bool $disableCodeCoverageIgnore, ?bool $disallowTestOutput, ?bool $disallowTodoAnnotatedTests, ?bool $enforceTimeLimit, ?array $excludeGroups, ?int $executionOrder, ?int $executionOrderDefects, ?array $extensions, ?array $unavailableExtensions, ?bool $failOnEmptyTestSuite, ?bool $failOnIncomplete, ?bool $failOnRisky, ?bool $failOnSkipped, ?bool $failOnWarning, ?string $filter, ?bool $generateConfiguration, ?array $groups, ?bool $help, ?string $includePath, ?array $iniSettings, ?string $junitLogfile, ?bool $listGroups, ?bool $listSuites, ?bool $listTests, ?string $listTestsXml, ?string $loader, ?bool $noCoverage, ?bool $noExtensions, ?bool $noInteraction, ?bool $noLogging, ?string $printer, ?bool $processIsolation, ?int $randomOrderSeed, ?int $repeat, ?bool $reportUselessTests, ?bool $resolveDependencies, ?bool $reverseList, ?bool $stderr, ?bool $strictCoverage, ?bool $stopOnDefect, ?bool $stopOnError, ?bool $stopOnFailure, ?bool $stopOnIncomplete, ?bool $stopOnRisky, ?bool $stopOnSkipped, ?bool $stopOnWarning, ?string $teamcityLogfile, ?array $testdoxExcludeGroups, ?array $testdoxGroups, ?string $testdoxHtmlFile, ?string $testdoxTextFile, ?string $testdoxXmlFile, ?array $testSuffixes, ?string $testSuite, array $unrecognizedOptions, ?string $unrecognizedOrderBy, ?bool $useDefaultConfiguration, ?bool $verbose, ?bool $version, ?array $coverageFilter, ?string $xdebugFilterFile) { $this->argument = $argument; $this->atLeastVersion = $atLeastVersion; @@ -470,6 +475,7 @@ public function __construct(?string $argument, ?string $atLeastVersion, ?bool $b $this->executionOrderDefects = $executionOrderDefects; $this->extensions = $extensions; $this->unavailableExtensions = $unavailableExtensions; + $this->failOnEmptyTestSuite = $failOnEmptyTestSuite; $this->failOnIncomplete = $failOnIncomplete; $this->failOnRisky = $failOnRisky; $this->failOnSkipped = $failOnSkipped; @@ -956,6 +962,20 @@ public function executionOrderDefects(): int return $this->executionOrderDefects; } + public function hasFailOnEmptyTestSuite(): bool + { + return $this->failOnEmptyTestSuite !== null; + } + + public function failOnEmptyTestSuite(): bool + { + if ($this->failOnEmptyTestSuite === null) { + throw new Exception; + } + + return $this->failOnEmptyTestSuite; + } + public function hasFailOnIncomplete(): bool { return $this->failOnIncomplete !== null; diff --git a/src/TextUI/CliArguments/Mapper.php b/src/TextUI/CliArguments/Mapper.php index 864ca38d226..4242ed129b2 100644 --- a/src/TextUI/CliArguments/Mapper.php +++ b/src/TextUI/CliArguments/Mapper.php @@ -209,6 +209,10 @@ public function mapToLegacyArray(Configuration $arguments): array $result['stopOnSkipped'] = $arguments->stopOnSkipped(); } + if ($arguments->hasFailOnEmptyTestSuite()) { + $result['failOnEmptyTestSuite'] = $arguments->failOnEmptyTestSuite(); + } + if ($arguments->hasFailOnIncomplete()) { $result['failOnIncomplete'] = $arguments->failOnIncomplete(); } diff --git a/src/TextUI/TestRunner.php b/src/TextUI/TestRunner.php index 14e0e41330f..ac9c3082dcd 100644 --- a/src/TextUI/TestRunner.php +++ b/src/TextUI/TestRunner.php @@ -804,6 +804,10 @@ public function run(TestSuite $suite, array $arguments = [], array $warnings = [ } if ($exit) { + if (isset($arguments['failOnEmptyTestSuite']) && $arguments['failOnEmptyTestSuite'] === true && \count($result) === 0) { + exit(self::FAILURE_EXIT); + } + if ($result->wasSuccessfulIgnoringWarnings()) { if ($arguments['failOnRisky'] && !$result->allHarmless()) { exit(self::FAILURE_EXIT); @@ -959,6 +963,7 @@ private function handleConfiguration(array &$arguments): void $arguments['stopOnIncomplete'] = $arguments['stopOnIncomplete'] ?? $phpunitConfiguration->stopOnIncomplete(); $arguments['stopOnRisky'] = $arguments['stopOnRisky'] ?? $phpunitConfiguration->stopOnRisky(); $arguments['stopOnSkipped'] = $arguments['stopOnSkipped'] ?? $phpunitConfiguration->stopOnSkipped(); + $arguments['failOnEmptyTestSuite'] = $arguments['failOnEmptyTestSuite'] ?? $phpunitConfiguration->failOnEmptyTestSuite(); $arguments['failOnIncomplete'] = $arguments['failOnIncomplete'] ?? $phpunitConfiguration->failOnIncomplete(); $arguments['failOnRisky'] = $arguments['failOnRisky'] ?? $phpunitConfiguration->failOnRisky(); $arguments['failOnSkipped'] = $arguments['failOnSkipped'] ?? $phpunitConfiguration->failOnSkipped(); diff --git a/src/TextUI/XmlConfiguration/Loader.php b/src/TextUI/XmlConfiguration/Loader.php index 41833f9d2c4..ffd2a8400bc 100644 --- a/src/TextUI/XmlConfiguration/Loader.php +++ b/src/TextUI/XmlConfiguration/Loader.php @@ -1027,6 +1027,7 @@ private function phpunit(string $filename, DOMDocument $document): PHPUnit $this->getBooleanAttribute($document->documentElement, 'forceCoversAnnotation', false), $bootstrap, $this->getBooleanAttribute($document->documentElement, 'processIsolation', false), + $this->getBooleanAttribute($document->documentElement, 'failOnEmptyTestSuite', false), $this->getBooleanAttribute($document->documentElement, 'failOnIncomplete', false), $this->getBooleanAttribute($document->documentElement, 'failOnRisky', false), $this->getBooleanAttribute($document->documentElement, 'failOnSkipped', false), diff --git a/src/TextUI/XmlConfiguration/PHPUnit/PHPUnit.php b/src/TextUI/XmlConfiguration/PHPUnit/PHPUnit.php index 46e9eac05e7..9efe1dd2929 100644 --- a/src/TextUI/XmlConfiguration/PHPUnit/PHPUnit.php +++ b/src/TextUI/XmlConfiguration/PHPUnit/PHPUnit.php @@ -90,6 +90,11 @@ final class PHPUnit */ private $processIsolation; + /** + * @var bool + */ + private $failOnEmptyTestSuite; + /** * @var bool */ @@ -269,7 +274,7 @@ final class PHPUnit */ private $conflictBetweenPrinterClassAndTestdox; - public function __construct(bool $cacheResult, ?string $cacheResultFile, $columns, string $colors, bool $stderr, bool $noInteraction, bool $verbose, bool $reverseDefectList, bool $convertDeprecationsToExceptions, bool $convertErrorsToExceptions, bool $convertNoticesToExceptions, bool $convertWarningsToExceptions, bool $forceCoversAnnotation, ?string $bootstrap, bool $processIsolation, bool $failOnIncomplete, bool $failOnRisky, bool $failOnSkipped, bool $failOnWarning, bool $stopOnDefect, bool $stopOnError, bool $stopOnFailure, bool $stopOnWarning, bool $stopOnIncomplete, bool $stopOnRisky, bool $stopOnSkipped, ?string $extensionsDirectory, ?string $testSuiteLoaderClass, ?string $testSuiteLoaderFile, ?string $printerClass, ?string $printerFile, bool $beStrictAboutChangesToGlobalState, bool $beStrictAboutOutputDuringTests, bool $beStrictAboutResourceUsageDuringSmallTests, bool $beStrictAboutTestsThatDoNotTestAnything, bool $beStrictAboutTodoAnnotatedTests, bool $beStrictAboutCoversAnnotation, bool $enforceTimeLimit, int $defaultTimeLimit, int $timeoutForSmallTests, int $timeoutForMediumTests, int $timeoutForLargeTests, ?string $defaultTestSuite, int $executionOrder, bool $resolveDependencies, bool $defectsFirst, bool $backupGlobals, bool $backupStaticAttributes, bool $registerMockObjectsFromTestArgumentsRecursively, bool $conflictBetweenPrinterClassAndTestdox) + public function __construct(bool $cacheResult, ?string $cacheResultFile, $columns, string $colors, bool $stderr, bool $noInteraction, bool $verbose, bool $reverseDefectList, bool $convertDeprecationsToExceptions, bool $convertErrorsToExceptions, bool $convertNoticesToExceptions, bool $convertWarningsToExceptions, bool $forceCoversAnnotation, ?string $bootstrap, bool $processIsolation, bool $failOnEmptyTestSuite, bool $failOnIncomplete, bool $failOnRisky, bool $failOnSkipped, bool $failOnWarning, bool $stopOnDefect, bool $stopOnError, bool $stopOnFailure, bool $stopOnWarning, bool $stopOnIncomplete, bool $stopOnRisky, bool $stopOnSkipped, ?string $extensionsDirectory, ?string $testSuiteLoaderClass, ?string $testSuiteLoaderFile, ?string $printerClass, ?string $printerFile, bool $beStrictAboutChangesToGlobalState, bool $beStrictAboutOutputDuringTests, bool $beStrictAboutResourceUsageDuringSmallTests, bool $beStrictAboutTestsThatDoNotTestAnything, bool $beStrictAboutTodoAnnotatedTests, bool $beStrictAboutCoversAnnotation, bool $enforceTimeLimit, int $defaultTimeLimit, int $timeoutForSmallTests, int $timeoutForMediumTests, int $timeoutForLargeTests, ?string $defaultTestSuite, int $executionOrder, bool $resolveDependencies, bool $defectsFirst, bool $backupGlobals, bool $backupStaticAttributes, bool $registerMockObjectsFromTestArgumentsRecursively, bool $conflictBetweenPrinterClassAndTestdox) { $this->cacheResult = $cacheResult; $this->cacheResultFile = $cacheResultFile; @@ -286,6 +291,7 @@ public function __construct(bool $cacheResult, ?string $cacheResultFile, $column $this->forceCoversAnnotation = $forceCoversAnnotation; $this->bootstrap = $bootstrap; $this->processIsolation = $processIsolation; + $this->failOnEmptyTestSuite = $failOnEmptyTestSuite; $this->failOnIncomplete = $failOnIncomplete; $this->failOnRisky = $failOnRisky; $this->failOnSkipped = $failOnSkipped; @@ -422,6 +428,11 @@ public function processIsolation(): bool return $this->processIsolation; } + public function failOnEmptyTestSuite(): bool + { + return $this->failOnEmptyTestSuite; + } + public function failOnIncomplete(): bool { return $this->failOnIncomplete; diff --git a/tests/unit/TextUI/XmlConfigurationTest.php b/tests/unit/TextUI/XmlConfigurationTest.php index 9cb3be75d6b..11cdde7a1a9 100644 --- a/tests/unit/TextUI/XmlConfigurationTest.php +++ b/tests/unit/TextUI/XmlConfigurationTest.php @@ -127,6 +127,7 @@ public function configurationRootOptionsProvider(): array 'stopOnIncomplete' => ['stopOnIncomplete', 'true', true], 'stopOnRisky' => ['stopOnRisky', 'true', true], 'stopOnSkipped' => ['stopOnSkipped', 'true', true], + 'failOnEmptyTestSuite' => ['failOnEmptyTestSuite', 'true', true], 'failOnWarning' => ['failOnWarning', 'true', true], 'failOnRisky' => ['failOnRisky', 'true', true], 'processIsolation' => ['processIsolation', 'true', true], @@ -648,6 +649,7 @@ public function testPHPUnitConfigurationIsReadCorrectly(): void $this->assertSame(60, $phpunit->timeoutForLargeTests()); $this->assertFalse($phpunit->beStrictAboutResourceUsageDuringSmallTests()); $this->assertFalse($phpunit->beStrictAboutTodoAnnotatedTests()); + $this->assertFalse($phpunit->failOnEmptyTestSuite()); $this->assertFalse($phpunit->failOnIncomplete()); $this->assertFalse($phpunit->failOnRisky()); $this->assertFalse($phpunit->failOnSkipped());