Add --only-covering-test-cases option to enable/disable filtering b…

…y covering test cases By default, Infection runs all tests cases from **files** that contain at least 1 test case that cover mutated line Using `--only-covering-test-cases`, we can enable filtering and run only those test cases from files, that cover mutated line. For very fast test suites this decreases performance (see #1539). For slow test suites (like functional / integration tests) - this can dramatically improve performance of Mutation Testing / Infection
infection · Jul 25, 2021 · 50f972a · 50f972a
1 parent 0fa6025
commit 50f972a
Show file tree

Hide file tree

Showing 13 changed files with 116 additions and 34 deletions.
diff --git a/src/Command/RunCommand.php b/src/Command/RunCommand.php
@@ -116,6 +116,8 @@ final class RunCommand extends BaseCommand
 
     private const OPTION_USE_NOOP_MUTATORS = 'noop';
 
+    private const OPTION_EXECUTE_ONLY_COVERING_TEST_CASES = 'only-covering-test-cases';
+
     /** @var string */
     private const OPTION_MIN_MSI = 'min-msi';
 
@@ -257,6 +259,12 @@ protected function configure(): void
                 InputOption::VALUE_NONE,
                 'Use noop mutators that do not change AST. For debugging purposes.',
             )
+            ->addOption(
+                self::OPTION_EXECUTE_ONLY_COVERING_TEST_CASES,
+                null,
+                InputOption::VALUE_NONE,
+                'Execute only those test cases that cover mutated line, not the whole file with covering test cases. Can dramatically speed up Mutation Testing for slow test suites. For PHPUnit / Pest it uses `--filter` option',
+            )
             ->addOption(
                 self::OPTION_MIN_MSI,
                 null,
@@ -444,7 +452,8 @@ private function createContainer(IO $io, LoggerInterface $logger): Container
             $gitDiffFilter,
             $gitDiffBase,
             (bool) $input->getOption(self::OPTION_LOGGER_GITHUB),
-            (bool) $input->getOption(self::OPTION_USE_NOOP_MUTATORS)
+            (bool) $input->getOption(self::OPTION_USE_NOOP_MUTATORS),
+            (bool) $input->getOption(self::OPTION_EXECUTE_ONLY_COVERING_TEST_CASES)
         );
     }
 

diff --git a/src/Configuration/Configuration.php b/src/Configuration/Configuration.php
@@ -87,6 +87,7 @@ class Configuration
     private bool $dryRun;
     /** @var array<string, array<int, string>> */
     private array $ignoreSourceCodeMutatorsMap;
+    private bool $executeOnlyCoveringTestCases;
 
     /**
      * @param string[] $sourceDirectories
@@ -123,7 +124,8 @@ public function __construct(
         int $msiPrecision,
         int $threadCount,
         bool $dryRun,
-        array $ignoreSourceCodeMutatorsMap
+        array $ignoreSourceCodeMutatorsMap,
+        bool $executeOnlyCoveringTestCases
     ) {
         Assert::nullOrGreaterThanEq($timeout, 0);
         Assert::allString($sourceDirectories);
@@ -161,6 +163,7 @@ public function __construct(
         $this->threadCount = $threadCount;
         $this->dryRun = $dryRun;
         $this->ignoreSourceCodeMutatorsMap = $ignoreSourceCodeMutatorsMap;
+        $this->executeOnlyCoveringTestCases = $executeOnlyCoveringTestCases;
     }
 
     public function getProcessTimeout(): float
@@ -317,4 +320,9 @@ public function getIgnoreSourceCodeMutatorsMap(): array
     {
         return $this->ignoreSourceCodeMutatorsMap;
     }
+
+    public function getExecuteOnlyCoveringTestCases(): bool
+    {
+        return $this->executeOnlyCoveringTestCases;
+    }
 }
diff --git a/src/Configuration/ConfigurationFactory.php b/src/Configuration/ConfigurationFactory.php
@@ -118,7 +118,8 @@ public function create(
         ?string $gitDiffFilter,
         ?string $gitDiffBase,
         bool $useGitHubLogger,
-        bool $useNoopMutators
+        bool $useNoopMutators,
+        bool $executeOnlyCoveringTestCases
     ): Configuration {
         $configDir = dirname($schema->getFile());
 
@@ -170,7 +171,8 @@ public function create(
             $msiPrecision,
             $threadCount,
             $dryRun,
-            $ignoreSourceCodeMutatorsMap
+            $ignoreSourceCodeMutatorsMap,
+            $executeOnlyCoveringTestCases
         );
     }
 

diff --git a/src/Container.php b/src/Container.php
@@ -166,6 +166,7 @@ final class Container
     public const DEFAULT_GIT_DIFF_BASE = null;
     public const DEFAULT_USE_GITHUB_LOGGER = false;
     public const DEFAULT_USE_NOOP_MUTATORS = false;
+    public const DEFAULT_EXECUTE_ONLY_COVERING_TEST_CASES = false;
     public const DEFAULT_NO_PROGRESS = false;
     public const DEFAULT_FORCE_PROGRESS = false;
     public const DEFAULT_EXISTING_COVERAGE_PATH = null;
@@ -684,7 +685,8 @@ public static function create(): self
             self::DEFAULT_GIT_DIFF_FILTER,
             self::DEFAULT_GIT_DIFF_BASE,
             self::DEFAULT_USE_GITHUB_LOGGER,
-            self::DEFAULT_USE_NOOP_MUTATORS
+            self::DEFAULT_USE_NOOP_MUTATORS,
+            self::DEFAULT_EXECUTE_ONLY_COVERING_TEST_CASES
         );
     }
 
@@ -715,7 +717,8 @@ public function withValues(
         ?string $gitDiffFilter,
         ?string $gitDiffBase,
         bool $useGitHubLogger,
-        bool $useNoopMutators
+        bool $useNoopMutators,
+        bool $executeOnlyCoveringTestCases
     ): self {
         $clone = clone $this;
 
@@ -790,7 +793,8 @@ static function (self $container) use (
                 $gitDiffFilter,
                 $gitDiffBase,
                 $useGitHubLogger,
-                $useNoopMutators
+                $useNoopMutators,
+                $executeOnlyCoveringTestCases
             ): Configuration {
                 return $container->getConfigurationFactory()->create(
                     $container->getSchemaConfiguration(),
@@ -815,7 +819,8 @@ static function (self $container) use (
                     $gitDiffFilter,
                     $gitDiffBase,
                     $useGitHubLogger,
-                    $useNoopMutators
+                    $useNoopMutators,
+                    $executeOnlyCoveringTestCases
                 );
             }
         );

diff --git a/src/TestFramework/Factory.php b/src/TestFramework/Factory.php
@@ -102,7 +102,8 @@ public function create(string $adapterName, bool $skipCoverage): TestFrameworkAd
                 $this->jUnitFilePath,
                 $this->projectDir,
                 $this->infectionConfig->getSourceDirectories(),
-                $skipCoverage
+                $skipCoverage,
+                $this->infectionConfig->getExecuteOnlyCoveringTestCases()
             );
         }
 
@@ -120,7 +121,8 @@ public function create(string $adapterName, bool $skipCoverage): TestFrameworkAd
                 $this->jUnitFilePath,
                 $this->projectDir,
                 $this->infectionConfig->getSourceDirectories(),
-                $skipCoverage
+                $skipCoverage,
+                $this->infectionConfig->getExecuteOnlyCoveringTestCases()
             );
         }
 

diff --git a/src/TestFramework/PhpUnit/Adapter/PestAdapterFactory.php b/src/TestFramework/PhpUnit/Adapter/PestAdapterFactory.php
@@ -64,7 +64,8 @@ public static function create(
         string $jUnitFilePath,
         string $projectDir,
         array $sourceDirectories,
-        bool $skipCoverage
+        bool $skipCoverage,
+        bool $executeOnlyCoveringTestCases = false
     ): TestFrameworkAdapter {
         Assert::string($testFrameworkConfigDir, 'Config dir is not allowed to be `null` for the Pest adapter');
 
@@ -97,7 +98,7 @@ public static function create(
                 $projectDir,
                 new JUnitTestCaseSorter()
             ),
-            new ArgumentsAndOptionsBuilder(),
+            new ArgumentsAndOptionsBuilder($executeOnlyCoveringTestCases),
             new VersionParser(),
             new CommandLineBuilder()
         );

diff --git a/src/TestFramework/PhpUnit/Adapter/PhpUnitAdapterFactory.php b/src/TestFramework/PhpUnit/Adapter/PhpUnitAdapterFactory.php
@@ -67,7 +67,8 @@ public static function create(
         string $jUnitFilePath,
         string $projectDir,
         array $sourceDirectories,
-        bool $skipCoverage
+        bool $skipCoverage,
+        bool $executeOnlyCoveringTestCases = false
     ): TestFrameworkAdapter {
         Assert::string($testFrameworkConfigDir, 'Config dir is not allowed to be `null` for the Pest adapter');
 
@@ -100,7 +101,7 @@ public static function create(
                 $projectDir,
                 new JUnitTestCaseSorter()
             ),
-            new ArgumentsAndOptionsBuilder(),
+            new ArgumentsAndOptionsBuilder($executeOnlyCoveringTestCases),
             new VersionParser(),
             new CommandLineBuilder()
         );

diff --git a/src/TestFramework/PhpUnit/CommandLine/ArgumentsAndOptionsBuilder.php b/src/TestFramework/PhpUnit/CommandLine/ArgumentsAndOptionsBuilder.php
@@ -51,6 +51,13 @@
  */
 final class ArgumentsAndOptionsBuilder implements CommandLineArgumentsAndOptionsBuilder
 {
+    private bool $executeOnlyCoveringTestCases;
+
+    public function __construct(bool $executeOnlyCoveringTestCases)
+    {
+        $this->executeOnlyCoveringTestCases = $executeOnlyCoveringTestCases;
+    }
+
     public function buildForInitialTestsRun(string $configPath, string $extraOptions): array
     {
         $options = [
@@ -77,7 +84,7 @@ public function buildForMutant(string $configPath, string $extraOptions, array $
     {
         $options = $this->buildForInitialTestsRun($configPath, $extraOptions);
 
-        if (count($tests) > 0) {
+        if ($this->executeOnlyCoveringTestCases && count($tests) > 0) {
             $filterString = '/';
             $usedTestCases = [];
 

diff --git a/tests/phpunit/Configuration/ConfigurationAssertions.php b/tests/phpunit/Configuration/ConfigurationAssertions.php
@@ -83,7 +83,8 @@ private function assertConfigurationStateIs(
         int $expectedMsiPrecision,
         int $expectedThreadCount,
         bool $expectedDryRyn,
-        array $expectedIgnoreSourceCodeMutatorsMap
+        array $expectedIgnoreSourceCodeMutatorsMap,
+        bool $expectedExecuteOnlyCoveringTestCases
     ): void {
         $this->assertSame($expectedTimeout, $configuration->getProcessTimeout());
         $this->assertSame($expectedSourceDirectories, $configuration->getSourceDirectories());
@@ -132,6 +133,7 @@ private function assertConfigurationStateIs(
         $this->assertSame($expectedThreadCount, $configuration->getThreadCount());
         $this->assertSame($expectedDryRyn, $configuration->isDryRun());
         $this->assertSame($expectedIgnoreSourceCodeMutatorsMap, $configuration->getIgnoreSourceCodeMutatorsMap());
+        $this->assertSame($expectedExecuteOnlyCoveringTestCases, $configuration->getExecuteOnlyCoveringTestCases());
     }
 
     /**