feat: Ability to run Fixer with parallel runner 🎉 (#7777)

Co-authored-by: Dariusz Rumiński <dariusz.ruminski@gmail.com> Co-authored-by: Julien Falque <julien.falque@gmail.com>
PHP-CS-Fixer · May 15, 2024 · 5c90224 · 5c90224
1 parent c2289bc
commit 5c90224
Show file tree

Hide file tree

Showing 54 changed files with 2,935 additions and 91 deletions.
diff --git a/.php-cs-fixer.dist.php b/.php-cs-fixer.dist.php
@@ -14,8 +14,10 @@
 
 use PhpCsFixer\Config;
 use PhpCsFixer\Finder;
+use PhpCsFixer\Runner\Parallel\ParallelConfigFactory;
 
 return (new Config())
+    ->setParallelConfig(ParallelConfigFactory::detect()) // @TODO 4.0 no need to call this manually
     ->setRiskyAllowed(true)
     ->setRules([
         '@PHP74Migration' => true,

diff --git a/composer.json b/composer.json
@@ -24,8 +24,15 @@
         "ext-filter": "*",
         "ext-json": "*",
         "ext-tokenizer": "*",
+        "clue/ndjson-react": "^1.0",
         "composer/semver": "^3.4",
         "composer/xdebug-handler": "^3.0.3",
+        "fidry/cpu-core-counter": "^1.0",
+        "react/child-process": "^0.6.5",
+        "react/event-loop": "^1.0",
+        "react/promise": "^2.0 || ^3.0",
+        "react/socket": "^1.0",
+        "react/stream": "^1.0",
         "sebastian/diff": "^4.0 || ^5.0 || ^6.0",
         "symfony/console": "^5.4 || ^6.0 || ^7.0",
         "symfony/event-dispatcher": "^5.4 || ^6.0 || ^7.0",
@@ -87,7 +94,10 @@
         ],
         "cs:check": "@php php-cs-fixer check --verbose --diff",
         "cs:fix": "@php php-cs-fixer fix",
-        "cs:fix:parallel": "echo '🔍 Will run in batches of 50 files.'; if [[ -f .php-cs-fixer.php ]]; then FIXER_CONFIG=.php-cs-fixer.php; else FIXER_CONFIG=.php-cs-fixer.dist.php; fi; php php-cs-fixer list-files --config=$FIXER_CONFIG | xargs -n 50 -P 8 php php-cs-fixer fix --config=$FIXER_CONFIG --path-mode intersection 2> /dev/null",
+        "cs:fix:parallel": [
+            "echo '⚠️ This script is deprecated! Utilise built-in parallelisation instead.';",
+            "@cs:fix"
+        ],
         "docs": "@php dev-tools/doc.php",
         "infection": "@test:mutation",
         "install-tools": "@composer --working-dir=dev-tools install",
@@ -156,7 +166,7 @@
         "auto-review": "Execute Auto-review",
         "cs:check": "Check coding standards",
         "cs:fix": "Fix coding standards",
-        "cs:fix:parallel": "Fix coding standards in naive parallel mode (using xargs)",
+        "cs:fix:parallel": "⚠️DEPRECATED! Use cs:fix with proper parallel config",
         "docs": "Regenerate docs",
         "infection": "Alias for 'test:mutation'",
         "install-tools": "Install DEV tools",

diff --git a/doc/usage.rst b/doc/usage.rst
@@ -21,11 +21,25 @@ If you do not have config file, you can run following command to fix non-hidden,
 
     php php-cs-fixer.phar fix .
 
-With some magic of tools provided by your OS, you can also fix files in parallel:
+You can also fix files in parallel, utilising more CPU cores. You can do this by using config class that implements ``PhpCsFixer\Runner\Parallel\ParallelConfig\ParallelAwareConfigInterface``, and use ``setParallelConfig()`` method. Recommended way is to utilise auto-detecting parallel configuration:
 
-.. code-block:: console
+.. code-block:: php
 
-    php php-cs-fixer.phar list-files --config=.php-cs-fixer.dist.php | xargs -n 50 -P 8 php php-cs-fixer.phar fix --config=.php-cs-fixer.dist.php --path-mode intersection -v
+    <?php
+
+    return (new PhpCsFixer\Config())
+        ->setParallelConfig(ParallelConfigFactory::detect())
+    ;
+
+However, in some case you may want to fine-tune parallelisation with explicit values (e.g. in environments where auto-detection does not work properly and suggests more cores than it should):
+
+.. code-block:: php
+
+    <?php
+
+    return (new PhpCsFixer\Config())
+        ->setParallelConfig(new ParallelConfig(4, 20))
+    ;
 
 You can limit process to given file or files in a given directory and its subdirectories:
 
@@ -98,6 +112,8 @@ Complete configuration for rules can be supplied using a ``json`` formatted stri
 
 The ``--dry-run`` flag will run the fixer without making changes to your files (implicitly set when you use ``check`` command).
 
+The ``--sequential`` flag will enforce sequential analysis even if parallel config is provided.
+
 The ``--diff`` flag can be used to let the fixer output all the changes it makes in ``udiff`` format.
 
 The ``--allow-risky`` option (pass ``yes`` or ``no``) allows you to set whether risky rules may run. Default value is taken from config file.

diff --git a/src/Config.php b/src/Config.php
@@ -15,13 +15,15 @@
 namespace PhpCsFixer;
 
 use PhpCsFixer\Fixer\FixerInterface;
+use PhpCsFixer\Runner\Parallel\ParallelConfig;
+use PhpCsFixer\Runner\Parallel\ParallelConfigFactory;
 
 /**
  * @author Fabien Potencier <fabien@symfony.com>
  * @author Katsuhiro Ogawa <ko.fivestar@gmail.com>
  * @author Dariusz Rumiński <dariusz.ruminski@gmail.com>
  */
-class Config implements ConfigInterface
+class Config implements ConfigInterface, ParallelAwareConfigInterface
 {
     private string $cacheFile = '.php-cs-fixer.cache';
 
@@ -47,6 +49,8 @@ class Config implements ConfigInterface
 
     private string $name;
 
+    private ParallelConfig $parallelConfig;
+
     /**
      * @var null|string
      */
@@ -71,6 +75,13 @@ public function __construct(string $name = 'default')
             $this->name = $name;
             $this->rules = ['@PSR12' => true];
         }
+
+        // @TODO 4.0 cleanup
+        if (Utils::isFutureModeEnabled() || filter_var(getenv('PHP_CS_FIXER_PARALLEL'), FILTER_VALIDATE_BOOL)) {
+            $this->parallelConfig = ParallelConfigFactory::detect();
+        } else {
+            $this->parallelConfig = ParallelConfigFactory::sequential();
+        }
     }
 
     public function getCacheFile(): string
@@ -118,6 +129,11 @@ public function getName(): string
         return $this->name;
     }
 
+    public function getParallelConfig(): ParallelConfig
+    {
+        return $this->parallelConfig;
+    }
+
     public function getPhpExecutable(): ?string
     {
         return $this->phpExecutable;
@@ -189,6 +205,13 @@ public function setLineEnding(string $lineEnding): ConfigInterface
         return $this;
     }
 
+    public function setParallelConfig(ParallelConfig $config): ConfigInterface
+    {
+        $this->parallelConfig = $config;
+
+        return $this;
+    }
+
     public function setPhpExecutable(?string $phpExecutable): ConfigInterface
     {
         $this->phpExecutable = $phpExecutable;

diff --git a/src/Console/Application.php b/src/Console/Application.php
@@ -21,12 +21,15 @@
 use PhpCsFixer\Console\Command\ListFilesCommand;
 use PhpCsFixer\Console\Command\ListSetsCommand;
 use PhpCsFixer\Console\Command\SelfUpdateCommand;
+use PhpCsFixer\Console\Command\WorkerCommand;
 use PhpCsFixer\Console\SelfUpdate\GithubClient;
 use PhpCsFixer\Console\SelfUpdate\NewVersionChecker;
 use PhpCsFixer\PharChecker;
+use PhpCsFixer\Runner\Parallel\WorkerException;
 use PhpCsFixer\ToolInfo;
 use PhpCsFixer\Utils;
 use Symfony\Component\Console\Application as BaseApplication;
+use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Command\ListCommand;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\ConsoleOutputInterface;
@@ -45,6 +48,7 @@ final class Application extends BaseApplication
     public const VERSION_CODENAME = '15 Keys Accelerate';
 
     private ToolInfo $toolInfo;
+    private ?Command $executedCommand = null;
 
     public function __construct()
     {
@@ -63,6 +67,7 @@ public function __construct()
             $this->toolInfo,
             new PharChecker()
         ));
+        $this->add(new WorkerCommand($this->toolInfo));
     }
 
     public static function getMajorVersion(): int
@@ -160,4 +165,45 @@ protected function getDefaultCommands(): array
     {
         return [new HelpCommand(), new ListCommand()];
     }
+
+    /**
+     * @throws \Throwable
+     */
+    protected function doRunCommand(Command $command, InputInterface $input, OutputInterface $output): int
+    {
+        $this->executedCommand = $command;
+
+        return parent::doRunCommand($command, $input, $output);
+    }
+
+    protected function doRenderThrowable(\Throwable $e, OutputInterface $output): void
+    {
+        // Since parallel analysis utilises child processes, and they have their own output,
+        // we need to capture the output of the child process to determine it there was an exception.
+        // Default render format is not machine-friendly, so we need to override it for `worker` command,
+        // in order to be able to easily parse exception data for further displaying on main process' side.
+        if ($this->executedCommand instanceof WorkerCommand) {
+            $output->writeln(WorkerCommand::ERROR_PREFIX.json_encode(
+                [
+                    'class' => \get_class($e),
+                    'message' => $e->getMessage(),
+                    'file' => $e->getFile(),
+                    'line' => $e->getLine(),
+                    'code' => $e->getCode(),
+                    'trace' => $e->getTraceAsString(),
+                ]
+            ));
+
+            return;
+        }
+
+        parent::doRenderThrowable($e, $output);
+
+        if ($output->isVeryVerbose() && $e instanceof WorkerException) {
+            $output->writeln('<comment>Original trace from worker:</comment>');
+            $output->writeln('');
+            $output->writeln($e->getOriginalTraceAsString());
+            $output->writeln('');
+        }
+    }
 }
diff --git a/src/Console/Command/FixCommand.php b/src/Console/Command/FixCommand.php
@@ -30,6 +30,7 @@
 use PhpCsFixer\ToolInfoInterface;
 use Symfony\Component\Console\Attribute\AsCommand;
 use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Formatter\OutputFormatter;
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Input\InputOption;
@@ -150,6 +151,8 @@ public function getHelp(): string
 
             The <comment>--dry-run</comment> flag will run the fixer without making changes to your files.
 
+            The <comment>--sequential</comment> flag will enforce sequential analysis even if parallel config is provided.
+
             The <comment>--diff</comment> flag can be used to let the fixer output all the changes it makes.
 
             The <comment>--allow-risky</comment> option (pass `yes` or `no`) allows you to set whether risky rules may run. Default value is taken from config file.
@@ -206,12 +209,13 @@ protected function configure(): void
                 new InputOption('config', '', InputOption::VALUE_REQUIRED, 'The path to a config file.'),
                 new InputOption('dry-run', '', InputOption::VALUE_NONE, 'Only shows which files would have been modified.'),
                 new InputOption('rules', '', InputOption::VALUE_REQUIRED, 'List of rules that should be run against configured paths.'),
-                new InputOption('using-cache', '', InputOption::VALUE_REQUIRED, 'Does cache should be used (can be `yes` or `no`).'),
+                new InputOption('using-cache', '', InputOption::VALUE_REQUIRED, 'Should cache be used (can be `yes` or `no`).'),
                 new InputOption('cache-file', '', InputOption::VALUE_REQUIRED, 'The path to the cache file.'),
                 new InputOption('diff', '', InputOption::VALUE_NONE, 'Prints diff for each file.'),
                 new InputOption('format', '', InputOption::VALUE_REQUIRED, 'To output results in other formats.'),
                 new InputOption('stop-on-violation', '', InputOption::VALUE_NONE, 'Stop execution on first violation.'),
                 new InputOption('show-progress', '', InputOption::VALUE_REQUIRED, 'Type of progress indicator (none, dots).'),
+                new InputOption('sequential', '', InputOption::VALUE_NONE, 'Enforce sequential analysis.'),
             ]
         );
     }
@@ -243,6 +247,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
                 'stop-on-violation' => $input->getOption('stop-on-violation'),
                 'verbosity' => $verbosity,
                 'show-progress' => $input->getOption('show-progress'),
+                'sequential' => $input->getOption('sequential'),
             ],
             getcwd(),
             $this->toolInfo
@@ -256,6 +261,31 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 
         if (null !== $stdErr) {
             $stdErr->writeln(Application::getAboutWithRuntime(true));
+            $isParallel = $resolver->getParallelConfig()->getMaxProcesses() > 1;
+
+            $stdErr->writeln(sprintf(
+                'Running analysis on %d core%s.',
+                $resolver->getParallelConfig()->getMaxProcesses(),
+                $isParallel ? sprintf(
+                    's with %d file%s per process',
+                    $resolver->getParallelConfig()->getFilesPerProcess(),
+                    $resolver->getParallelConfig()->getFilesPerProcess() > 1 ? 's' : ''
+                ) : ' sequentially'
+            ));
+
+            /** @TODO v4 remove warnings related to parallel runner */
+            $usageDocs = 'https://cs.symfony.com/doc/usage.html';
+            $stdErr->writeln(sprintf(
+                $stdErr->isDecorated() ? '<bg=yellow;fg=black;>%s</>' : '%s',
+                $isParallel
+                    ? 'Parallel runner is an experimental feature and may be unstable, use it at your own risk. Feedback highly appreciated!'
+                    : sprintf(
+                        'You can enable parallel runner and speed up the analysis! Please see %s for more information.',
+                        $stdErr->isDecorated()
+                            ? sprintf('<href=%s;bg=yellow;fg=red;bold>usage docs</>', OutputFormatter::escape($usageDocs))
+                            : $usageDocs
+                    )
+            ));
 
             $configFile = $resolver->getConfigFile();
             $stdErr->writeln(sprintf('Loaded config <comment>%s</comment>%s.', $resolver->getConfig()->getName(), null === $configFile ? '' : ' from "'.$configFile.'"'));
@@ -297,7 +327,10 @@ protected function execute(InputInterface $input, OutputInterface $output): int
             $resolver->isDryRun(),
             $resolver->getCacheManager(),
             $resolver->getDirectory(),
-            $resolver->shouldStopOnViolation()
+            $resolver->shouldStopOnViolation(),
+            $resolver->getParallelConfig(),
+            $input,
+            $resolver->getConfigFile()
         );
 
         $this->eventDispatcher->addListener(FixerFileProcessedEvent::NAME, [$progressOutput, 'onFixerFileProcessed']);

diff --git a/src/Console/Command/FixCommandExitStatusCalculator.php b/src/Console/Command/FixCommandExitStatusCalculator.php
@@ -28,8 +28,13 @@ final class FixCommandExitStatusCalculator
     public const EXIT_STATUS_FLAG_HAS_INVALID_FIXER_CONFIG = 32;
     public const EXIT_STATUS_FLAG_EXCEPTION_IN_APP = 64;
 
-    public function calculate(bool $isDryRun, bool $hasChangedFiles, bool $hasInvalidErrors, bool $hasExceptionErrors, bool $hasLintErrorsAfterFixing): int
-    {
+    public function calculate(
+        bool $isDryRun,
+        bool $hasChangedFiles,
+        bool $hasInvalidErrors,
+        bool $hasExceptionErrors,
+        bool $hasLintErrorsAfterFixing
+    ): int {
         $exitStatus = 0;
 
         if ($isDryRun) {