Enhancement: Allow failing tests when test suite is empty

sebastianbergmann · Jun 23, 2020 · 99b3a65 · 99b3a65
1 parent 25712ea
commit 99b3a65
Show file tree

Hide file tree

Showing 8 changed files with 54 additions and 2 deletions.
diff --git a/phpunit.xsd b/phpunit.xsd
@@ -228,6 +228,7 @@
         <xs:attribute name="stopOnIncomplete" type="xs:boolean" default="false"/>
         <xs:attribute name="stopOnRisky" type="xs:boolean" default="false"/>
         <xs:attribute name="stopOnSkipped" type="xs:boolean" default="false"/>
+        <xs:attribute name="failOnEmptyTestSuite" type="xs:boolean" default="false"/>
         <xs:attribute name="failOnIncomplete" type="xs:boolean" default="false"/>
         <xs:attribute name="failOnRisky" type="xs:boolean" default="false"/>
         <xs:attribute name="failOnSkipped" type="xs:boolean" default="false"/>

diff --git a/src/TextUI/CliArguments/Builder.php b/src/TextUI/CliArguments/Builder.php
@@ -91,6 +91,7 @@ final class Builder
         'stop-on-incomplete',
         'stop-on-risky',
         'stop-on-skipped',
+        'fail-on-empty-test-suite',
         'fail-on-warning',
         'fail-on-risky',
         'strict-coverage',
@@ -162,6 +163,7 @@ public function fromParameters(array $parameters, array $additionalLongOptions):
         $executionOrderDefects                      = null;
         $extensions                                 = [];
         $unavailableExtensions                      = [];
+        $failOnEmptyTestSuite                       = null;
         $failOnIncomplete                           = null;
         $failOnRisky                                = null;
         $failOnSkipped                              = null;
@@ -496,6 +498,11 @@ public function fromParameters(array $parameters, array $additionalLongOptions):
 
                     break;
 
+                case '--fail-on-empty-test-suite':
+                    $failOnEmptyTestSuite = true;
+
+                    break;
+
                 case '--fail-on-incomplete':
                     $failOnIncomplete = true;
 
@@ -764,6 +771,7 @@ public function fromParameters(array $parameters, array $additionalLongOptions):
             $executionOrderDefects,
             $extensions,
             $unavailableExtensions,
+            $failOnEmptyTestSuite,
             $failOnIncomplete,
             $failOnRisky,
             $failOnSkipped,

diff --git a/src/TextUI/CliArguments/Configuration.php b/src/TextUI/CliArguments/Configuration.php
@@ -182,6 +182,11 @@ final class Configuration
      */
     private $unavailableExtensions;
 
+    /**
+     * @var ?bool
+     */
+    private $failOnEmptyTestSuite;
+
     /**
      * @var ?bool
      */
@@ -435,7 +440,7 @@ final class Configuration
     /**
      * @param null|int|string $columns
      */
-    public function __construct(?string $argument, ?string $atLeastVersion, ?bool $backupGlobals, ?bool $backupStaticAttributes, ?bool $beStrictAboutChangesToGlobalState, ?bool $beStrictAboutResourceUsageDuringSmallTests, ?string $bootstrap, ?bool $cacheResult, ?string $cacheResultFile, ?bool $checkVersion, ?string $colors, $columns, ?string $configuration, ?string $coverageClover, ?string $coverageCrap4J, ?string $coverageHtml, ?string $coveragePhp, ?string $coverageText, ?bool $coverageTextShowUncoveredFiles, ?bool $coverageTextShowOnlySummary, ?string $coverageXml, ?bool $debug, ?int $defaultTimeLimit, ?bool $disableCodeCoverageIgnore, ?bool $disallowTestOutput, ?bool $disallowTodoAnnotatedTests, ?bool $enforceTimeLimit, ?array $excludeGroups, ?int $executionOrder, ?int $executionOrderDefects, ?array $extensions, ?array $unavailableExtensions, ?bool $failOnIncomplete, ?bool $failOnRisky, ?bool $failOnSkipped, ?bool $failOnWarning, ?string $filter, ?bool $generateConfiguration, ?array $groups, ?bool $help, ?string $includePath, ?array $iniSettings, ?string $junitLogfile, ?bool $listGroups, ?bool $listSuites, ?bool $listTests, ?string $listTestsXml, ?string $loader, ?bool $noCoverage, ?bool $noExtensions, ?bool $noInteraction, ?bool $noLogging, ?string $printer, ?bool $processIsolation, ?int $randomOrderSeed, ?int $repeat, ?bool $reportUselessTests, ?bool $resolveDependencies, ?bool $reverseList, ?bool $stderr, ?bool $strictCoverage, ?bool $stopOnDefect, ?bool $stopOnError, ?bool $stopOnFailure, ?bool $stopOnIncomplete, ?bool $stopOnRisky, ?bool $stopOnSkipped, ?bool $stopOnWarning, ?string $teamcityLogfile, ?array $testdoxExcludeGroups, ?array $testdoxGroups, ?string $testdoxHtmlFile, ?string $testdoxTextFile, ?string $testdoxXmlFile, ?array $testSuffixes, ?string $testSuite, array $unrecognizedOptions, ?string $unrecognizedOrderBy, ?bool $useDefaultConfiguration, ?bool $verbose, ?bool $version, ?array $coverageFilter, ?string $xdebugFilterFile)
+    public function __construct(?string $argument, ?string $atLeastVersion, ?bool $backupGlobals, ?bool $backupStaticAttributes, ?bool $beStrictAboutChangesToGlobalState, ?bool $beStrictAboutResourceUsageDuringSmallTests, ?string $bootstrap, ?bool $cacheResult, ?string $cacheResultFile, ?bool $checkVersion, ?string $colors, $columns, ?string $configuration, ?string $coverageClover, ?string $coverageCrap4J, ?string $coverageHtml, ?string $coveragePhp, ?string $coverageText, ?bool $coverageTextShowUncoveredFiles, ?bool $coverageTextShowOnlySummary, ?string $coverageXml, ?bool $debug, ?int $defaultTimeLimit, ?bool $disableCodeCoverageIgnore, ?bool $disallowTestOutput, ?bool $disallowTodoAnnotatedTests, ?bool $enforceTimeLimit, ?array $excludeGroups, ?int $executionOrder, ?int $executionOrderDefects, ?array $extensions, ?array $unavailableExtensions, ?bool $failOnEmptyTestSuite, ?bool $failOnIncomplete, ?bool $failOnRisky, ?bool $failOnSkipped, ?bool $failOnWarning, ?string $filter, ?bool $generateConfiguration, ?array $groups, ?bool $help, ?string $includePath, ?array $iniSettings, ?string $junitLogfile, ?bool $listGroups, ?bool $listSuites, ?bool $listTests, ?string $listTestsXml, ?string $loader, ?bool $noCoverage, ?bool $noExtensions, ?bool $noInteraction, ?bool $noLogging, ?string $printer, ?bool $processIsolation, ?int $randomOrderSeed, ?int $repeat, ?bool $reportUselessTests, ?bool $resolveDependencies, ?bool $reverseList, ?bool $stderr, ?bool $strictCoverage, ?bool $stopOnDefect, ?bool $stopOnError, ?bool $stopOnFailure, ?bool $stopOnIncomplete, ?bool $stopOnRisky, ?bool $stopOnSkipped, ?bool $stopOnWarning, ?string $teamcityLogfile, ?array $testdoxExcludeGroups, ?array $testdoxGroups, ?string $testdoxHtmlFile, ?string $testdoxTextFile, ?string $testdoxXmlFile, ?array $testSuffixes, ?string $testSuite, array $unrecognizedOptions, ?string $unrecognizedOrderBy, ?bool $useDefaultConfiguration, ?bool $verbose, ?bool $version, ?array $coverageFilter, ?string $xdebugFilterFile)
     {
         $this->argument                                   = $argument;
         $this->atLeastVersion                             = $atLeastVersion;
@@ -470,6 +475,7 @@ public function __construct(?string $argument, ?string $atLeastVersion, ?bool $b
         $this->executionOrderDefects                      = $executionOrderDefects;
         $this->extensions                                 = $extensions;
         $this->unavailableExtensions                      = $unavailableExtensions;
+        $this->failOnEmptyTestSuite                       = $failOnEmptyTestSuite;
         $this->failOnIncomplete                           = $failOnIncomplete;
         $this->failOnRisky                                = $failOnRisky;
         $this->failOnSkipped                              = $failOnSkipped;
@@ -956,6 +962,20 @@ public function executionOrderDefects(): int
         return $this->executionOrderDefects;
     }
 
+    public function hasFailOnEmptyTestSuite(): bool
+    {
+        return $this->failOnEmptyTestSuite !== null;
+    }
+
+    public function failOnEmptyTestSuite(): bool
+    {
+        if ($this->failOnEmptyTestSuite === null) {
+            throw new Exception;
+        }
+
+        return $this->failOnEmptyTestSuite;
+    }
+
     public function hasFailOnIncomplete(): bool
     {
         return $this->failOnIncomplete !== null;

diff --git a/src/TextUI/CliArguments/Mapper.php b/src/TextUI/CliArguments/Mapper.php
@@ -209,6 +209,10 @@ public function mapToLegacyArray(Configuration $arguments): array
             $result['stopOnSkipped'] = $arguments->stopOnSkipped();
         }
 
+        if ($arguments->hasFailOnEmptyTestSuite()) {
+            $result['failOnEmptyTestSuite'] = $arguments->failOnEmptyTestSuite();
+        }
+
         if ($arguments->hasFailOnIncomplete()) {
             $result['failOnIncomplete'] = $arguments->failOnIncomplete();
         }

diff --git a/src/TextUI/TestRunner.php b/src/TextUI/TestRunner.php
@@ -804,6 +804,10 @@ public function run(TestSuite $suite, array $arguments = [], array $warnings = [
         }
 
         if ($exit) {
+            if (isset($arguments['failOnEmptyTestSuite']) && $arguments['failOnEmptyTestSuite'] === true && \count($result) === 0) {
+                exit(self::FAILURE_EXIT);
+            }
+
             if ($result->wasSuccessfulIgnoringWarnings()) {
                 if ($arguments['failOnRisky'] && !$result->allHarmless()) {
                     exit(self::FAILURE_EXIT);
@@ -959,6 +963,7 @@ private function handleConfiguration(array &$arguments): void
             $arguments['stopOnIncomplete']                                = $arguments['stopOnIncomplete'] ?? $phpunitConfiguration->stopOnIncomplete();
             $arguments['stopOnRisky']                                     = $arguments['stopOnRisky'] ?? $phpunitConfiguration->stopOnRisky();
             $arguments['stopOnSkipped']                                   = $arguments['stopOnSkipped'] ?? $phpunitConfiguration->stopOnSkipped();
+            $arguments['failOnEmptyTestSuite']                            = $arguments['failOnEmptyTestSuite'] ?? $phpunitConfiguration->failOnEmptyTestSuite();
             $arguments['failOnIncomplete']                                = $arguments['failOnIncomplete'] ?? $phpunitConfiguration->failOnIncomplete();
             $arguments['failOnRisky']                                     = $arguments['failOnRisky'] ?? $phpunitConfiguration->failOnRisky();
             $arguments['failOnSkipped']                                   = $arguments['failOnSkipped'] ?? $phpunitConfiguration->failOnSkipped();

diff --git a/src/TextUI/XmlConfiguration/Loader.php b/src/TextUI/XmlConfiguration/Loader.php
@@ -1027,6 +1027,7 @@ private function phpunit(string $filename, DOMDocument $document): PHPUnit
             $this->getBooleanAttribute($document->documentElement, 'forceCoversAnnotation', false),
             $bootstrap,
             $this->getBooleanAttribute($document->documentElement, 'processIsolation', false),
+            $this->getBooleanAttribute($document->documentElement, 'failOnEmptyTestSuite', false),
             $this->getBooleanAttribute($document->documentElement, 'failOnIncomplete', false),
             $this->getBooleanAttribute($document->documentElement, 'failOnRisky', false),
             $this->getBooleanAttribute($document->documentElement, 'failOnSkipped', false),

diff --git a/src/TextUI/XmlConfiguration/PHPUnit/PHPUnit.php b/src/TextUI/XmlConfiguration/PHPUnit/PHPUnit.php
@@ -90,6 +90,11 @@ final class PHPUnit
      */
     private $processIsolation;
 
+    /**
+     * @var bool
+     */
+    private $failOnEmptyTestSuite;
+
     /**
      * @var bool
      */
@@ -269,7 +274,7 @@ final class PHPUnit
      */
     private $conflictBetweenPrinterClassAndTestdox;
 
-    public function __construct(bool $cacheResult, ?string $cacheResultFile, $columns, string $colors, bool $stderr, bool $noInteraction, bool $verbose, bool $reverseDefectList, bool $convertDeprecationsToExceptions, bool $convertErrorsToExceptions, bool $convertNoticesToExceptions, bool $convertWarningsToExceptions, bool $forceCoversAnnotation, ?string $bootstrap, bool $processIsolation, bool $failOnIncomplete, bool $failOnRisky, bool $failOnSkipped, bool $failOnWarning, bool $stopOnDefect, bool $stopOnError, bool $stopOnFailure, bool $stopOnWarning, bool $stopOnIncomplete, bool $stopOnRisky, bool $stopOnSkipped, ?string $extensionsDirectory, ?string $testSuiteLoaderClass, ?string $testSuiteLoaderFile, ?string $printerClass, ?string $printerFile, bool $beStrictAboutChangesToGlobalState, bool $beStrictAboutOutputDuringTests, bool $beStrictAboutResourceUsageDuringSmallTests, bool $beStrictAboutTestsThatDoNotTestAnything, bool $beStrictAboutTodoAnnotatedTests, bool $beStrictAboutCoversAnnotation, bool $enforceTimeLimit, int $defaultTimeLimit, int $timeoutForSmallTests, int $timeoutForMediumTests, int $timeoutForLargeTests, ?string $defaultTestSuite, int $executionOrder, bool $resolveDependencies, bool $defectsFirst, bool $backupGlobals, bool $backupStaticAttributes, bool $registerMockObjectsFromTestArgumentsRecursively, bool $conflictBetweenPrinterClassAndTestdox)
+    public function __construct(bool $cacheResult, ?string $cacheResultFile, $columns, string $colors, bool $stderr, bool $noInteraction, bool $verbose, bool $reverseDefectList, bool $convertDeprecationsToExceptions, bool $convertErrorsToExceptions, bool $convertNoticesToExceptions, bool $convertWarningsToExceptions, bool $forceCoversAnnotation, ?string $bootstrap, bool $processIsolation, bool $failOnEmptyTestSuite, bool $failOnIncomplete, bool $failOnRisky, bool $failOnSkipped, bool $failOnWarning, bool $stopOnDefect, bool $stopOnError, bool $stopOnFailure, bool $stopOnWarning, bool $stopOnIncomplete, bool $stopOnRisky, bool $stopOnSkipped, ?string $extensionsDirectory, ?string $testSuiteLoaderClass, ?string $testSuiteLoaderFile, ?string $printerClass, ?string $printerFile, bool $beStrictAboutChangesToGlobalState, bool $beStrictAboutOutputDuringTests, bool $beStrictAboutResourceUsageDuringSmallTests, bool $beStrictAboutTestsThatDoNotTestAnything, bool $beStrictAboutTodoAnnotatedTests, bool $beStrictAboutCoversAnnotation, bool $enforceTimeLimit, int $defaultTimeLimit, int $timeoutForSmallTests, int $timeoutForMediumTests, int $timeoutForLargeTests, ?string $defaultTestSuite, int $executionOrder, bool $resolveDependencies, bool $defectsFirst, bool $backupGlobals, bool $backupStaticAttributes, bool $registerMockObjectsFromTestArgumentsRecursively, bool $conflictBetweenPrinterClassAndTestdox)
     {
         $this->cacheResult                                     = $cacheResult;
         $this->cacheResultFile                                 = $cacheResultFile;
@@ -286,6 +291,7 @@ public function __construct(bool $cacheResult, ?string $cacheResultFile, $column
         $this->forceCoversAnnotation                           = $forceCoversAnnotation;
         $this->bootstrap                                       = $bootstrap;
         $this->processIsolation                                = $processIsolation;
+        $this->failOnEmptyTestSuite                            = $failOnEmptyTestSuite;
         $this->failOnIncomplete                                = $failOnIncomplete;
         $this->failOnRisky                                     = $failOnRisky;
         $this->failOnSkipped                                   = $failOnSkipped;
@@ -422,6 +428,11 @@ public function processIsolation(): bool
         return $this->processIsolation;
     }
 
+    public function failOnEmptyTestSuite(): bool
+    {
+        return $this->failOnEmptyTestSuite;
+    }
+
     public function failOnIncomplete(): bool
     {
         return $this->failOnIncomplete;

diff --git a/tests/unit/TextUI/XmlConfigurationTest.php b/tests/unit/TextUI/XmlConfigurationTest.php
@@ -127,6 +127,7 @@ public function configurationRootOptionsProvider(): array
             'stopOnIncomplete'                               => ['stopOnIncomplete', 'true', true],
             'stopOnRisky'                                    => ['stopOnRisky', 'true', true],
             'stopOnSkipped'                                  => ['stopOnSkipped', 'true', true],
+            'failOnEmptyTestSuite'                           => ['failOnEmptyTestSuite', 'true', true],
             'failOnWarning'                                  => ['failOnWarning', 'true', true],
             'failOnRisky'                                    => ['failOnRisky', 'true', true],
             'processIsolation'                               => ['processIsolation', 'true', true],
@@ -648,6 +649,7 @@ public function testPHPUnitConfigurationIsReadCorrectly(): void
         $this->assertSame(60, $phpunit->timeoutForLargeTests());
         $this->assertFalse($phpunit->beStrictAboutResourceUsageDuringSmallTests());
         $this->assertFalse($phpunit->beStrictAboutTodoAnnotatedTests());
+        $this->assertFalse($phpunit->failOnEmptyTestSuite());
         $this->assertFalse($phpunit->failOnIncomplete());
         $this->assertFalse($phpunit->failOnRisky());
         $this->assertFalse($phpunit->failOnSkipped());