From f48325ba03ec5d89c57208d8ddb926c3edd1d081 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Fri, 12 Jan 2024 11:06:41 +0100 Subject: [PATCH 01/77] Register hidden `WorkerCommand` Empty for now, because logic for running fixers must be moved here. Basically it's a PoC of command being hidden - works on PHP 7.4 and 8.3 with latest Composer packages on each version. Command's arguments/options are more or less what we need in the worker, but it may be changed later. --- src/Console/Application.php | 2 + src/Console/Command/WorkerCommand.php | 82 +++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 src/Console/Command/WorkerCommand.php diff --git a/src/Console/Application.php b/src/Console/Application.php index 0df441e38a0..aed73079787 100644 --- a/src/Console/Application.php +++ b/src/Console/Application.php @@ -21,6 +21,7 @@ use PhpCsFixer\Console\Command\ListFilesCommand; use PhpCsFixer\Console\Command\ListSetsCommand; use PhpCsFixer\Console\Command\SelfUpdateCommand; +use PhpCsFixer\Console\Command\WorkerCommand; use PhpCsFixer\Console\SelfUpdate\GithubClient; use PhpCsFixer\Console\SelfUpdate\NewVersionChecker; use PhpCsFixer\PharChecker; @@ -63,6 +64,7 @@ public function __construct() $this->toolInfo, new PharChecker() )); + $this->add(new WorkerCommand()); } public static function getMajorVersion(): int diff --git a/src/Console/Command/WorkerCommand.php b/src/Console/Command/WorkerCommand.php new file mode 100644 index 00000000000..c3d084d4c0c --- /dev/null +++ b/src/Console/Command/WorkerCommand.php @@ -0,0 +1,82 @@ + + * Dariusz Rumiński + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace PhpCsFixer\Console\Command; + +use Symfony\Component\Console\Attribute\AsCommand; +use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Input\InputArgument; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Input\InputOption; +use Symfony\Component\Console\Output\OutputInterface; + +/** + * @author Greg Korba + * + * @internal + */ +#[AsCommand(name: 'worker', description: 'Internal command for running fixers in parallel', hidden: true)] +final class WorkerCommand extends Command +{ + /** @var string */ + protected static $defaultName = 'worker'; + + /** @var string */ + protected static $defaultDescription = 'Internal command for running fixers in parallel'; + + public function __construct(string $name = null) + { + parent::__construct($name); + + $this->setHidden(true); + } + + protected function configure(): void + { + $this->setDefinition( + [ + new InputArgument( + 'paths', + InputArgument::IS_ARRAY, + 'The path(s) that rules will be run against (each path can be a file or directory).' + ), + new InputOption( + 'allow-risky', + '', + InputOption::VALUE_REQUIRED, + 'Are risky fixers allowed (can be `yes` or `no`).' + ), + new InputOption('config', '', InputOption::VALUE_REQUIRED, 'The path to a config file.'), + new InputOption( + 'rules', + '', + InputOption::VALUE_REQUIRED, + 'List of rules that should be run against configured paths.' + ), + new InputOption( + 'using-cache', + '', + InputOption::VALUE_REQUIRED, + 'Does cache should be used (can be `yes` or `no`).' + ), + new InputOption('cache-file', '', InputOption::VALUE_REQUIRED, 'The path to the cache file.'), + ] + ); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + return Command::SUCCESS; + } +} From 186bec416f9d54535f890bf4339f577a53b9b182 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Fri, 12 Jan 2024 17:03:44 +0100 Subject: [PATCH 02/77] Configuration flow for parallel run --- src/Config.php | 19 ++++++- src/Console/Command/FixCommand.php | 10 ++-- src/Console/ConfigurationResolver.php | 11 +++++ src/ParallelRunnerConfigInterface.php | 29 +++++++++++ src/Runner/Parallel/ParallelConfig.php | 55 +++++++++++++++++++++ src/Runner/Runner.php | 41 ++++++++++----- src/Runner/RunnerConfig.php | 49 ++++++++++++++++++ tests/Console/ConfigurationResolverTest.php | 23 ++++++++- tests/Runner/RunnerTest.php | 17 +++---- tests/Test/AbstractIntegrationTestCase.php | 6 ++- 10 files changed, 232 insertions(+), 28 deletions(-) create mode 100644 src/ParallelRunnerConfigInterface.php create mode 100644 src/Runner/Parallel/ParallelConfig.php create mode 100644 src/Runner/RunnerConfig.php diff --git a/src/Config.php b/src/Config.php index 59e5462cf5e..58cb3a2ee4a 100644 --- a/src/Config.php +++ b/src/Config.php @@ -15,13 +15,14 @@ namespace PhpCsFixer; use PhpCsFixer\Fixer\FixerInterface; +use PhpCsFixer\Runner\Parallel\ParallelConfig; /** * @author Fabien Potencier * @author Katsuhiro Ogawa * @author Dariusz Rumiński */ -class Config implements ConfigInterface +class Config implements ConfigInterface, ParallelRunnerConfigInterface { private string $cacheFile = '.php-cs-fixer.cache'; @@ -47,6 +48,8 @@ class Config implements ConfigInterface private string $name; + private ParallelConfig $parallelRunnerConfig; + /** * @var null|string */ @@ -63,6 +66,8 @@ class Config implements ConfigInterface public function __construct(string $name = 'default') { + $this->parallelRunnerConfig = ParallelConfig::detect(); + // @TODO 4.0 cleanup if (Utils::isFutureModeEnabled()) { $this->name = $name.' (future mode)'; @@ -118,6 +123,11 @@ public function getName(): string return $this->name; } + public function getParallelConfig(): ParallelConfig + { + return $this->parallelRunnerConfig; + } + public function getPhpExecutable(): ?string { return $this->phpExecutable; @@ -189,6 +199,13 @@ public function setLineEnding(string $lineEnding): ConfigInterface return $this; } + public function setParallelConfig(ParallelConfig $config): ConfigInterface + { + $this->parallelRunnerConfig = $config; + + return $this; + } + public function setPhpExecutable(?string $phpExecutable): ConfigInterface { $this->phpExecutable = $phpExecutable; diff --git a/src/Console/Command/FixCommand.php b/src/Console/Command/FixCommand.php index 7db27641e5f..65a9e511ad5 100644 --- a/src/Console/Command/FixCommand.php +++ b/src/Console/Command/FixCommand.php @@ -27,6 +27,7 @@ use PhpCsFixer\Error\ErrorsManager; use PhpCsFixer\FixerFileProcessedEvent; use PhpCsFixer\Runner\Runner; +use PhpCsFixer\Runner\RunnerConfig; use PhpCsFixer\ToolInfoInterface; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; @@ -288,16 +289,19 @@ protected function execute(InputInterface $input, OutputInterface $output): int ); $runner = new Runner( + new RunnerConfig( + $resolver->isDryRun(), + $resolver->shouldStopOnViolation(), + $resolver->getParallelConfig() + ), $finder, $resolver->getFixers(), $resolver->getDiffer(), ProgressOutputType::NONE !== $progressType ? $this->eventDispatcher : null, $this->errorsManager, $resolver->getLinter(), - $resolver->isDryRun(), $resolver->getCacheManager(), - $resolver->getDirectory(), - $resolver->shouldStopOnViolation() + $resolver->getDirectory() ); $this->eventDispatcher->addListener(FixerFileProcessedEvent::NAME, [$progressOutput, 'onFixerFileProcessed']); diff --git a/src/Console/ConfigurationResolver.php b/src/Console/ConfigurationResolver.php index 2a895371491..2fab4118157 100644 --- a/src/Console/ConfigurationResolver.php +++ b/src/Console/ConfigurationResolver.php @@ -35,8 +35,10 @@ use PhpCsFixer\FixerFactory; use PhpCsFixer\Linter\Linter; use PhpCsFixer\Linter\LinterInterface; +use PhpCsFixer\ParallelRunnerConfigInterface; use PhpCsFixer\RuleSet\RuleSet; use PhpCsFixer\RuleSet\RuleSetInterface; +use PhpCsFixer\Runner\Parallel\ParallelConfig; use PhpCsFixer\StdinFileInfo; use PhpCsFixer\ToolInfoInterface; use PhpCsFixer\Utils; @@ -274,6 +276,15 @@ public function getConfig(): ConfigInterface return $this->config; } + public function getParallelConfig(): ParallelConfig + { + $config = $this->getConfig(); + + return $config instanceof ParallelRunnerConfigInterface + ? $config->getParallelConfig() + : ParallelConfig::detect(); + } + public function getConfigFile(): ?string { if (null === $this->configFile) { diff --git a/src/ParallelRunnerConfigInterface.php b/src/ParallelRunnerConfigInterface.php new file mode 100644 index 00000000000..f6a0ce1c837 --- /dev/null +++ b/src/ParallelRunnerConfigInterface.php @@ -0,0 +1,29 @@ + + * Dariusz Rumiński + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace PhpCsFixer; + +use PhpCsFixer\Runner\Parallel\ParallelConfig; + +/** + * @author Greg Korba + * + * @TODO 4.0 Include parallel runner config in main config + */ +interface ParallelRunnerConfigInterface extends ConfigInterface +{ + public function getParallelConfig(): ParallelConfig; + + public function setParallelConfig(ParallelConfig $config): ConfigInterface; +} diff --git a/src/Runner/Parallel/ParallelConfig.php b/src/Runner/Parallel/ParallelConfig.php new file mode 100644 index 00000000000..1d50a315cfd --- /dev/null +++ b/src/Runner/Parallel/ParallelConfig.php @@ -0,0 +1,55 @@ + + * Dariusz Rumiński + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace PhpCsFixer\Runner\Parallel; + +/** + * @author Greg Korba + */ +final class ParallelConfig +{ + private int $filesPerProcess; + private int $maxProcesses; + private int $processTimeout; + + public function __construct(int $maxProcesses = 1, int $filesPerProcess = 10, int $processTimeout = 0) + { + $this->maxProcesses = $maxProcesses; + $this->filesPerProcess = $filesPerProcess; + $this->processTimeout = $processTimeout; + } + + public function getFilesPerProcess(): int + { + return $this->filesPerProcess; + } + + public function getMaxProcesses(): int + { + return $this->maxProcesses; + } + + public function getProcessTimeout(): int + { + return $this->processTimeout; + } + + /** + * @TODO Automatic detection of available cores + */ + public static function detect(): self + { + return new self(); + } +} diff --git a/src/Runner/Runner.php b/src/Runner/Runner.php index 89a65d0f68f..f822da632e8 100644 --- a/src/Runner/Runner.php +++ b/src/Runner/Runner.php @@ -34,9 +34,14 @@ /** * @author Dariusz Rumiński + * @author Greg Korba + * + * @phpstan-type _RunResult array, diff: string}> */ final class Runner { + private RunnerConfig $runnerConfig; + private DifferInterface $differ; private ?DirectoryInterface $directory; @@ -47,8 +52,6 @@ final class Runner private CacheManagerInterface $cacheManager; - private bool $isDryRun; - private LinterInterface $linter; /** @@ -61,40 +64,54 @@ final class Runner */ private array $fixers; - private bool $stopOnViolation; - /** * @param \Traversable<\SplFileInfo> $finder * @param list $fixers */ public function __construct( + RunnerConfig $runnerConfig, \Traversable $finder, array $fixers, DifferInterface $differ, ?EventDispatcherInterface $eventDispatcher, ErrorsManager $errorsManager, LinterInterface $linter, - bool $isDryRun, CacheManagerInterface $cacheManager, - ?DirectoryInterface $directory = null, - bool $stopOnViolation = false + ?DirectoryInterface $directory = null ) { + $this->runnerConfig = $runnerConfig; $this->finder = $finder; $this->fixers = $fixers; $this->differ = $differ; $this->eventDispatcher = $eventDispatcher; $this->errorsManager = $errorsManager; $this->linter = $linter; - $this->isDryRun = $isDryRun; $this->cacheManager = $cacheManager; $this->directory = $directory ?? new Directory(''); - $this->stopOnViolation = $stopOnViolation; } /** - * @return array, diff: string}> + * @return _RunResult */ public function fix(): array + { + return $this->runnerConfig->getParallelConfig()->getMaxProcesses() > 1 + ? $this->fixParallel() + : $this->fixSequential(); + } + + /** + * @return _RunResult + */ + private function fixParallel(): array + { + throw new \RuntimeException('NOT IMPLEMENTED YET'); + } + + /** + * @return _RunResult + */ + private function fixSequential(): array { $changed = []; @@ -120,7 +137,7 @@ public function fix(): array $name = $this->directory->getRelativePathTo($file->__toString()); $changed[$name] = $fixInfo; - if ($this->stopOnViolation) { + if ($this->runnerConfig->shouldStopOnViolation()) { break; } } @@ -223,7 +240,7 @@ private function fixFile(\SplFileInfo $file, LintingResultInterface $lintingResu return null; } - if (!$this->isDryRun) { + if (!$this->runnerConfig->isDryRun()) { $fileName = $file->getRealPath(); if (!file_exists($fileName)) { diff --git a/src/Runner/RunnerConfig.php b/src/Runner/RunnerConfig.php new file mode 100644 index 00000000000..f1d88ee323e --- /dev/null +++ b/src/Runner/RunnerConfig.php @@ -0,0 +1,49 @@ + + * Dariusz Rumiński + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace PhpCsFixer\Runner; + +use PhpCsFixer\Runner\Parallel\ParallelConfig; + +/** + * @author Greg Korba + */ +final class RunnerConfig +{ + private bool $isDryRun = false; + private bool $stopOnViolation = false; + private ParallelConfig $parallelConfig; + + public function __construct(bool $isDryRun, bool $stopOnViolation, ParallelConfig $parallelConfig) + { + $this->isDryRun = $isDryRun; + $this->stopOnViolation = $stopOnViolation; + $this->parallelConfig = $parallelConfig; + } + + public function isDryRun(): bool + { + return $this->isDryRun; + } + + public function shouldStopOnViolation(): bool + { + return $this->stopOnViolation; + } + + public function getParallelConfig(): ParallelConfig + { + return $this->parallelConfig; + } +} diff --git a/tests/Console/ConfigurationResolverTest.php b/tests/Console/ConfigurationResolverTest.php index 13ca5a9c37c..d5f8d8eed7e 100644 --- a/tests/Console/ConfigurationResolverTest.php +++ b/tests/Console/ConfigurationResolverTest.php @@ -17,6 +17,7 @@ use PhpCsFixer\AbstractFixer; use PhpCsFixer\Cache\NullCacheManager; use PhpCsFixer\Config; +use PhpCsFixer\ConfigInterface; use PhpCsFixer\ConfigurationException\InvalidConfigurationException; use PhpCsFixer\Console\Command\FixCommand; use PhpCsFixer\Console\ConfigurationResolver; @@ -30,6 +31,7 @@ use PhpCsFixer\FixerConfiguration\FixerConfigurationResolverInterface; use PhpCsFixer\FixerConfiguration\FixerOptionBuilder; use PhpCsFixer\FixerDefinition\FixerDefinitionInterface; +use PhpCsFixer\Runner\Parallel\ParallelConfig; use PhpCsFixer\Tests\TestCase; use PhpCsFixer\Tokenizer\Tokens; use PhpCsFixer\ToolInfoInterface; @@ -46,6 +48,25 @@ */ final class ConfigurationResolverTest extends TestCase { + public function testResolveParallelConfig(): void + { + $parallelConfig = new ParallelConfig(); + $config = (new Config())->setParallelConfig($parallelConfig); + $resolver = $this->createConfigurationResolver([], $config); + + self::assertSame($parallelConfig, $resolver->getParallelConfig()); + } + + public function testResolveDefaultParallelConfig(): void + { + $parallelConfig = $this->createConfigurationResolver([])->getParallelConfig(); + $defaultParallelConfig = ParallelConfig::detect(); + + self::assertSame($parallelConfig->getMaxProcesses(), $defaultParallelConfig->getMaxProcesses()); + self::assertSame($parallelConfig->getFilesPerProcess(), $defaultParallelConfig->getFilesPerProcess()); + self::assertSame($parallelConfig->getProcessTimeout(), $defaultParallelConfig->getProcessTimeout()); + } + public function testSetOptionWithUndefinedOption(): void { $this->expectException(InvalidConfigurationException::class); @@ -1378,7 +1399,7 @@ private static function getFixtureDir(): string */ private function createConfigurationResolver( array $options, - ?Config $config = null, + ?ConfigInterface $config = null, string $cwdPath = '', ?ToolInfoInterface $toolInfo = null ): ConfigurationResolver { diff --git a/tests/Runner/RunnerTest.php b/tests/Runner/RunnerTest.php index 4181237bb98..b766a818197 100644 --- a/tests/Runner/RunnerTest.php +++ b/tests/Runner/RunnerTest.php @@ -25,7 +25,9 @@ use PhpCsFixer\Linter\Linter; use PhpCsFixer\Linter\LinterInterface; use PhpCsFixer\Linter\LintingResultInterface; +use PhpCsFixer\Runner\Parallel\ParallelConfig; use PhpCsFixer\Runner\Runner; +use PhpCsFixer\Runner\RunnerConfig; use PhpCsFixer\Tests\TestCase; use Symfony\Component\Finder\Finder; @@ -56,16 +58,15 @@ public function testThatFixSuccessfully(): void $path = __DIR__.\DIRECTORY_SEPARATOR.'..'.\DIRECTORY_SEPARATOR.'Fixtures'.\DIRECTORY_SEPARATOR.'FixerTest'.\DIRECTORY_SEPARATOR.'fix'; $runner = new Runner( + new RunnerConfig(true, false, new ParallelConfig()), Finder::create()->in($path), $fixers, new NullDiffer(), null, new ErrorsManager(), $linter, - true, new NullCacheManager(), - new Directory($path), - false + new Directory($path) ); $changed = $runner->fix(); @@ -76,16 +77,15 @@ public function testThatFixSuccessfully(): void $path = __DIR__.\DIRECTORY_SEPARATOR.'..'.\DIRECTORY_SEPARATOR.'Fixtures'.\DIRECTORY_SEPARATOR.'FixerTest'.\DIRECTORY_SEPARATOR.'fix'; $runner = new Runner( + new RunnerConfig(true, true, new ParallelConfig()), Finder::create()->in($path), $fixers, new NullDiffer(), null, new ErrorsManager(), $linter, - true, new NullCacheManager(), new Directory($path), - true ); $changed = $runner->fix(); @@ -104,6 +104,7 @@ public function testThatFixInvalidFileReportsToErrorManager(): void $path = realpath(__DIR__.\DIRECTORY_SEPARATOR.'..').\DIRECTORY_SEPARATOR.'Fixtures'.\DIRECTORY_SEPARATOR.'FixerTest'.\DIRECTORY_SEPARATOR.'invalid'; $runner = new Runner( + new RunnerConfig(true, false, new ParallelConfig()), Finder::create()->in($path), [ new Fixer\ClassNotation\VisibilityRequiredFixer(), @@ -113,7 +114,6 @@ public function testThatFixInvalidFileReportsToErrorManager(): void null, $errorsManager, new Linter(), - true, new NullCacheManager() ); $changed = $runner->fix(); @@ -144,16 +144,15 @@ public function testThatDiffedFileIsPassedToDiffer(): void ]; $runner = new Runner( + new RunnerConfig(true, true, new ParallelConfig()), Finder::create()->in($path), $fixers, $differ, null, new ErrorsManager(), new Linter(), - true, new NullCacheManager(), - new Directory($path), - true + new Directory($path) ); $runner->fix(); diff --git a/tests/Test/AbstractIntegrationTestCase.php b/tests/Test/AbstractIntegrationTestCase.php index 1c984703342..3c786440888 100644 --- a/tests/Test/AbstractIntegrationTestCase.php +++ b/tests/Test/AbstractIntegrationTestCase.php @@ -26,7 +26,9 @@ use PhpCsFixer\Linter\LinterInterface; use PhpCsFixer\Linter\ProcessLinter; use PhpCsFixer\PhpunitConstraintIsIdenticalString\Constraint\IsIdenticalString; +use PhpCsFixer\Runner\Parallel\ParallelConfig; use PhpCsFixer\Runner\Runner; +use PhpCsFixer\Runner\RunnerConfig; use PhpCsFixer\Tests\TestCase; use PhpCsFixer\Tokenizer\Tokens; use PhpCsFixer\WhitespacesFixerConfig; @@ -255,13 +257,13 @@ protected function doTest(IntegrationCase $case): void $errorsManager = new ErrorsManager(); $fixers = self::createFixers($case); $runner = new Runner( + new RunnerConfig(false, false, new ParallelConfig()), new \ArrayIterator([new \SplFileInfo($tmpFile)]), $fixers, new UnifiedDiffer(), null, $errorsManager, $this->linter, - false, new NullCacheManager() ); @@ -315,13 +317,13 @@ protected function doTest(IntegrationCase $case): void } $runner = new Runner( + new RunnerConfig(false, false, new ParallelConfig()), new \ArrayIterator([new \SplFileInfo($tmpFile)]), array_reverse($fixers), new UnifiedDiffer(), null, $errorsManager, $this->linter, - false, new NullCacheManager() ); From eebe4efb3154c32ef873a2c0217e30aab167e518 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Mon, 22 Jan 2024 01:12:37 +0100 Subject: [PATCH 03/77] Extract factory method for linting aware file iterator The same collection of files has to be determined for both sequential and parallel run. See PR7777#discussion_r1526648174 --- ...php => FileCachingLintingFileIterator.php} | 16 ++------ ...ngIterator.php => LintingFileIterator.php} | 2 +- ...intingResultAwareFileIteratorInterface.php | 29 ++++++++++++++ src/Runner/Runner.php | 28 ++++++++------ .../Runner/FileCachingLintingIteratorTest.php | 10 ++--- ...orTest.php => LintingFileIteratorTest.php} | 38 +++++++++---------- 6 files changed, 74 insertions(+), 49 deletions(-) rename src/Runner/{FileCachingLintingIterator.php => FileCachingLintingFileIterator.php} (84%) rename src/Runner/{FileLintingIterator.php => LintingFileIterator.php} (93%) create mode 100644 src/Runner/LintingResultAwareFileIteratorInterface.php rename tests/Runner/{FileLintingIteratorTest.php => LintingFileIteratorTest.php} (69%) diff --git a/src/Runner/FileCachingLintingIterator.php b/src/Runner/FileCachingLintingFileIterator.php similarity index 84% rename from src/Runner/FileCachingLintingIterator.php rename to src/Runner/FileCachingLintingFileIterator.php index c07dbbd27e3..368dd8d7c49 100644 --- a/src/Runner/FileCachingLintingIterator.php +++ b/src/Runner/FileCachingLintingFileIterator.php @@ -24,19 +24,11 @@ * * @extends \CachingIterator> */ -final class FileCachingLintingIterator extends \CachingIterator +final class FileCachingLintingFileIterator extends \CachingIterator implements LintingResultAwareFileIteratorInterface { private LinterInterface $linter; - - /** - * @var LintingResultInterface - */ - private $currentResult; - - /** - * @var LintingResultInterface - */ - private $nextResult; + private ?LintingResultInterface $currentResult; + private ?LintingResultInterface $nextResult; /** * @param \Iterator $iterator @@ -48,7 +40,7 @@ public function __construct(\Iterator $iterator, LinterInterface $linter) $this->linter = $linter; } - public function currentLintingResult(): LintingResultInterface + public function currentLintingResult(): ?LintingResultInterface { return $this->currentResult; } diff --git a/src/Runner/FileLintingIterator.php b/src/Runner/LintingFileIterator.php similarity index 93% rename from src/Runner/FileLintingIterator.php rename to src/Runner/LintingFileIterator.php index 09cec8da099..9b93ed1d6f1 100644 --- a/src/Runner/FileLintingIterator.php +++ b/src/Runner/LintingFileIterator.php @@ -24,7 +24,7 @@ * * @extends \IteratorIterator> */ -final class FileLintingIterator extends \IteratorIterator +final class LintingFileIterator extends \IteratorIterator implements LintingResultAwareFileIteratorInterface { /** * @var null|LintingResultInterface diff --git a/src/Runner/LintingResultAwareFileIteratorInterface.php b/src/Runner/LintingResultAwareFileIteratorInterface.php new file mode 100644 index 00000000000..1c05bf29d6a --- /dev/null +++ b/src/Runner/LintingResultAwareFileIteratorInterface.php @@ -0,0 +1,29 @@ + + * Dariusz Rumiński + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace PhpCsFixer\Runner; + +use PhpCsFixer\Linter\LintingResultInterface; + +/** + * @author Greg Korba + * + * @internal + * + * @extends \Iterator + */ +interface LintingResultAwareFileIteratorInterface extends \Iterator +{ + public function currentLintingResult(): ?LintingResultInterface; +} diff --git a/src/Runner/Runner.php b/src/Runner/Runner.php index f822da632e8..f0462b6e41c 100644 --- a/src/Runner/Runner.php +++ b/src/Runner/Runner.php @@ -114,18 +114,7 @@ private function fixParallel(): array private function fixSequential(): array { $changed = []; - - $finder = $this->finder; - $finderIterator = $finder instanceof \IteratorAggregate ? $finder->getIterator() : $finder; - $fileFilteredFileIterator = new FileFilterIterator( - $finderIterator, - $this->eventDispatcher, - $this->cacheManager - ); - - $collection = $this->linter->isAsync() - ? new FileCachingLintingIterator($fileFilteredFileIterator, $this->linter) - : new FileLintingIterator($fileFilteredFileIterator, $this->linter); + $collection = $this->getFileIterator(); foreach ($collection as $file) { $fixInfo = $this->fixFile($file, $collection->currentLintingResult()); @@ -314,4 +303,19 @@ private function dispatchEvent(string $name, Event $event): void $this->eventDispatcher->dispatch($event, $name); } + + private function getFileIterator(): LintingResultAwareFileIteratorInterface + { + $finder = $this->finder; + $finderIterator = $finder instanceof \IteratorAggregate ? $finder->getIterator() : $finder; + $fileFilteredFileIterator = new FileFilterIterator( + $finderIterator, + $this->eventDispatcher, + $this->cacheManager + ); + + return $this->linter->isAsync() + ? new FileCachingLintingFileIterator($fileFilteredFileIterator, $this->linter) + : new LintingFileIterator($fileFilteredFileIterator, $this->linter); + } } diff --git a/tests/Runner/FileCachingLintingIteratorTest.php b/tests/Runner/FileCachingLintingIteratorTest.php index c08de02744d..15939b459d6 100644 --- a/tests/Runner/FileCachingLintingIteratorTest.php +++ b/tests/Runner/FileCachingLintingIteratorTest.php @@ -16,13 +16,13 @@ use PhpCsFixer\Linter\LinterInterface; use PhpCsFixer\Linter\LintingResultInterface; -use PhpCsFixer\Runner\FileCachingLintingIterator; +use PhpCsFixer\Runner\FileCachingLintingFileIterator; use PhpCsFixer\Tests\TestCase; /** * @internal * - * @covers \PhpCsFixer\Runner\FileCachingLintingIterator + * @covers \PhpCsFixer\Runner\FileCachingLintingFileIterator */ final class FileCachingLintingIteratorTest extends TestCase { @@ -30,7 +30,7 @@ public function testLintingEmpty(): void { $iterator = new \ArrayIterator([]); - $fileCachingLintingIterator = new FileCachingLintingIterator( + $fileCachingLintingIterator = new FileCachingLintingFileIterator( $iterator, $this->createLinterDouble() ); @@ -57,7 +57,7 @@ public function check(): void $iterator = new \ArrayIterator($files); - $fileCachingLintingIterator = new FileCachingLintingIterator( + $fileCachingLintingIterator = new FileCachingLintingFileIterator( $iterator, $this->createLinterDouble($lintingResult) ); @@ -68,7 +68,7 @@ public function check(): void } private static function assertLintingIteratorIteration( - FileCachingLintingIterator $fileCachingLintingIterator, + FileCachingLintingFileIterator $fileCachingLintingIterator, LintingResultInterface $lintingResultInterface, \SplFileInfo ...$files ): void { diff --git a/tests/Runner/FileLintingIteratorTest.php b/tests/Runner/LintingFileIteratorTest.php similarity index 69% rename from tests/Runner/FileLintingIteratorTest.php rename to tests/Runner/LintingFileIteratorTest.php index 19f5df1cd0b..2207bc55e40 100644 --- a/tests/Runner/FileLintingIteratorTest.php +++ b/tests/Runner/LintingFileIteratorTest.php @@ -16,29 +16,29 @@ use PhpCsFixer\Linter\LinterInterface; use PhpCsFixer\Linter\LintingResultInterface; -use PhpCsFixer\Runner\FileLintingIterator; +use PhpCsFixer\Runner\LintingFileIterator; use PhpCsFixer\Tests\TestCase; /** * @internal * - * @covers \PhpCsFixer\Runner\FileLintingIterator + * @covers \PhpCsFixer\Runner\LintingFileIterator */ -final class FileLintingIteratorTest extends TestCase +final class LintingFileIteratorTest extends TestCase { public function testFileLintingIteratorEmpty(): void { $iterator = new \ArrayIterator([]); - $fileLintingIterator = new FileLintingIterator( + $lintingFileIterator = new LintingFileIterator( $iterator, $this->createLinterDouble() ); - self::assertNull($fileLintingIterator->current()); - self::assertNull($fileLintingIterator->currentLintingResult()); - self::assertSame($iterator, $fileLintingIterator->getInnerIterator()); - self::assertFalse($fileLintingIterator->valid()); + self::assertNull($lintingFileIterator->current()); + self::assertNull($lintingFileIterator->currentLintingResult()); + self::assertSame($iterator, $lintingFileIterator->getInnerIterator()); + self::assertFalse($lintingFileIterator->valid()); } public function testFileLintingIterator(): void @@ -54,44 +54,44 @@ public function check(): void $iterator = new \ArrayIterator([$file]); - $fileLintingIterator = new FileLintingIterator( + $lintingFileIterator = new LintingFileIterator( $iterator, $this->createLinterDouble($lintingResult) ); // test when not touched current is null - self::assertNull($fileLintingIterator->currentLintingResult()); + self::assertNull($lintingFileIterator->currentLintingResult()); // test iterating - $this->fileLintingIteratorIterationTest($fileLintingIterator, $file, $lintingResult); + $this->lintingFileIteratorIterationTest($lintingFileIterator, $file, $lintingResult); // rewind and test again - $fileLintingIterator->rewind(); + $lintingFileIterator->rewind(); - $this->fileLintingIteratorIterationTest($fileLintingIterator, $file, $lintingResult); + $this->lintingFileIteratorIterationTest($lintingFileIterator, $file, $lintingResult); } - private function fileLintingIteratorIterationTest( - FileLintingIterator $fileLintingIterator, + private function lintingFileIteratorIterationTest( + LintingFileIterator $lintingFileIterator, \SplFileInfo $file, LintingResultInterface $lintingResultInterface ): void { $iterations = 0; - foreach ($fileLintingIterator as $lintedFile) { + foreach ($lintingFileIterator as $lintedFile) { self::assertSame($file, $lintedFile); - self::assertSame($lintingResultInterface, $fileLintingIterator->currentLintingResult()); + self::assertSame($lintingResultInterface, $lintingFileIterator->currentLintingResult()); ++$iterations; } self::assertSame(1, $iterations); - $fileLintingIterator->next(); + $lintingFileIterator->next(); - self::assertNull($fileLintingIterator->currentLintingResult()); + self::assertNull($lintingFileIterator->currentLintingResult()); } private function createLinterDouble(?LintingResultInterface $lintingResult = null): LinterInterface From b788803fb4364f341e716fa19acb36b5b4937d5e Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Fri, 26 Jan 2024 00:00:51 +0100 Subject: [PATCH 04/77] =?UTF-8?q?Initial=20implementation=20of=20parallel?= =?UTF-8?q?=20analysis=20=F0=9F=8E=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It's working, but needs polishing here and there. It speeds up analysis significantly (like 5x for this codebase on my computer). --- composer.json | 5 + src/Console/Application.php | 2 +- src/Console/Command/FixCommand.php | 3 +- src/Console/Command/WorkerCommand.php | 182 ++++++++++++- src/Error/ErrorsManager.php | 10 + src/Runner/Parallel/ParallelConfig.php | 13 +- .../Parallel/ParallelisationException.php | 30 +++ src/Runner/Parallel/Process.php | 254 ++++++++++++++++++ src/Runner/Parallel/ProcessIdentifier.php | 53 ++++ src/Runner/Parallel/ProcessPool.php | 87 ++++++ src/Runner/Runner.php | 184 ++++++++++++- src/Runner/RunnerConfig.php | 17 +- 12 files changed, 813 insertions(+), 27 deletions(-) create mode 100644 src/Runner/Parallel/ParallelisationException.php create mode 100644 src/Runner/Parallel/Process.php create mode 100644 src/Runner/Parallel/ProcessIdentifier.php create mode 100644 src/Runner/Parallel/ProcessPool.php diff --git a/composer.json b/composer.json index 0434e876016..4e98ee57333 100644 --- a/composer.json +++ b/composer.json @@ -24,8 +24,13 @@ "ext-filter": "*", "ext-json": "*", "ext-tokenizer": "*", + "clue/ndjson-react": "^1.0", "composer/semver": "^3.4", "composer/xdebug-handler": "^3.0.3", + "react/child-process": "^0.6.5", + "react/event-loop": "^1.5", + "react/promise": "^3.1", + "react/socket": "^1.15", "sebastian/diff": "^4.0 || ^5.0 || ^6.0", "symfony/console": "^5.4 || ^6.0 || ^7.0", "symfony/event-dispatcher": "^5.4 || ^6.0 || ^7.0", diff --git a/src/Console/Application.php b/src/Console/Application.php index aed73079787..b9c53e14d33 100644 --- a/src/Console/Application.php +++ b/src/Console/Application.php @@ -64,7 +64,7 @@ public function __construct() $this->toolInfo, new PharChecker() )); - $this->add(new WorkerCommand()); + $this->add(new WorkerCommand($this->toolInfo)); } public static function getMajorVersion(): int diff --git a/src/Console/Command/FixCommand.php b/src/Console/Command/FixCommand.php index 65a9e511ad5..d91e078eceb 100644 --- a/src/Console/Command/FixCommand.php +++ b/src/Console/Command/FixCommand.php @@ -292,7 +292,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int new RunnerConfig( $resolver->isDryRun(), $resolver->shouldStopOnViolation(), - $resolver->getParallelConfig() + $resolver->getParallelConfig(), + $resolver->getConfigFile() ), $finder, $resolver->getFixers(), diff --git a/src/Console/Command/WorkerCommand.php b/src/Console/Command/WorkerCommand.php index c3d084d4c0c..2a3abaa3774 100644 --- a/src/Console/Command/WorkerCommand.php +++ b/src/Console/Command/WorkerCommand.php @@ -14,12 +14,28 @@ namespace PhpCsFixer\Console\Command; +use Clue\React\NDJson\Decoder; +use Clue\React\NDJson\Encoder; +use PhpCsFixer\Config; +use PhpCsFixer\Console\ConfigurationResolver; +use PhpCsFixer\Error\Error; +use PhpCsFixer\Error\ErrorsManager; +use PhpCsFixer\FixerFileProcessedEvent; +use PhpCsFixer\Runner\Parallel\ParallelConfig; +use PhpCsFixer\Runner\Runner; +use PhpCsFixer\Runner\RunnerConfig; +use PhpCsFixer\ToolInfoInterface; +use React\EventLoop\StreamSelectLoop; +use React\Socket\ConnectionInterface; +use React\Socket\TcpConnector; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; -use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; +use Symfony\Component\Console\Output\ConsoleOutputInterface; use Symfony\Component\Console\Output\OutputInterface; +use Symfony\Component\EventDispatcher\EventDispatcher; +use Symfony\Component\EventDispatcher\EventDispatcherInterface; /** * @author Greg Korba @@ -35,21 +51,39 @@ final class WorkerCommand extends Command /** @var string */ protected static $defaultDescription = 'Internal command for running fixers in parallel'; - public function __construct(string $name = null) + private ToolInfoInterface $toolInfo; + private ConfigurationResolver $configurationResolver; + private ErrorsManager $errorsManager; + private EventDispatcherInterface $eventDispatcher; + + /** @var array */ + private array $events; + + public function __construct(ToolInfoInterface $toolInfo) { - parent::__construct($name); + parent::__construct(); $this->setHidden(true); + $this->toolInfo = $toolInfo; + $this->errorsManager = new ErrorsManager(); + $this->eventDispatcher = new EventDispatcher(); } protected function configure(): void { $this->setDefinition( [ - new InputArgument( - 'paths', - InputArgument::IS_ARRAY, - 'The path(s) that rules will be run against (each path can be a file or directory).' + new InputOption( + 'port', + null, + InputOption::VALUE_REQUIRED, + 'Specifies parallelisation server\'s port.' + ), + new InputOption( + 'identifier', + null, + InputOption::VALUE_REQUIRED, + 'Specifies parallelisation process\' identifier.' ), new InputOption( 'allow-risky', @@ -58,6 +92,12 @@ protected function configure(): void 'Are risky fixers allowed (can be `yes` or `no`).' ), new InputOption('config', '', InputOption::VALUE_REQUIRED, 'The path to a config file.'), + new InputOption( + 'dry-run', + '', + InputOption::VALUE_NONE, + 'Only shows which files would have been modified.' + ), new InputOption( 'rules', '', @@ -71,12 +111,140 @@ protected function configure(): void 'Does cache should be used (can be `yes` or `no`).' ), new InputOption('cache-file', '', InputOption::VALUE_REQUIRED, 'The path to the cache file.'), + new InputOption('diff', '', InputOption::VALUE_NONE, 'Prints diff for each file.'), ] ); } protected function execute(InputInterface $input, OutputInterface $output): int { + $verbosity = $output->getVerbosity(); + $errorOutput = $output instanceof ConsoleOutputInterface ? $output->getErrorOutput() : $output; + $identifier = $input->getOption('identifier'); + $port = $input->getOption('port'); + + if (null === $identifier || !is_numeric($port)) { + $errorOutput->writeln('Missing parallelisation options'); + + return Command::FAILURE; + } + + try { + $runner = $this->createRunner($input); + } catch (\Throwable $e) { + $errorOutput->writeln($e->getMessage()); + + return Command::FAILURE; + } + + $loop = new StreamSelectLoop(); + $tcpConnector = new TcpConnector($loop); + $tcpConnector + ->connect(sprintf('127.0.0.1:%d', $port)) + ->then(function (ConnectionInterface $connection) use ($runner, $identifier): void { + $jsonInvalidUtf8Ignore = \defined('JSON_INVALID_UTF8_IGNORE') ? JSON_INVALID_UTF8_IGNORE : 0; + $out = new Encoder($connection, $jsonInvalidUtf8Ignore); + $in = new Decoder($connection, true, 512, $jsonInvalidUtf8Ignore); + + // [REACT] Initialise connection with the parallelisation operator + $out->write(['action' => 'hello', 'identifier' => $identifier]); + + $handleError = static function (\Throwable $error): void { + // @TODO Handle communication errors + }; + $out->on('error', $handleError); + $in->on('error', $handleError); + + // [REACT] Listen for messages from the parallelisation operator (analysis requests) + $in->on('data', function (array $json) use ($runner, $out): void { + if ('run' !== $json['action']) { + return; + } + + /** @var iterable $files */ + $files = $json['files']; + + // Reset events because we want to collect only those coming from analysed files chunk + $this->events = []; + $runner->setFileIterator(new \ArrayIterator( + array_map(static fn (string $path) => new \SplFileInfo($path), $files) + )); + $analysisResult = $runner->fix(); + + $result = []; + foreach ($files as $i => $absolutePath) { + $relativePath = $this->configurationResolver->getDirectory()->getRelativePathTo($absolutePath); + + // @phpstan-ignore-next-line False-positive caused by assigning empty array to $events property + $result[$relativePath]['status'] = isset($this->events[$i]) + ? $this->events[$i]->getStatus() + : null; + $result[$relativePath]['fixInfo'] = $analysisResult[$relativePath] ?? null; + // @TODO consider serialising whole Error, so it can be deserialised on server side and passed to error manager + $result[$relativePath]['errors'] = array_map( + static fn (Error $error): array => [ + 'type' => $error->getType(), + 'error_message' => null !== $error->getSource() ? $error->getSource()->getMessage() : null, + ], + $this->errorsManager->forPath($absolutePath) + ); + } + + $out->write(['action' => 'result', 'result' => $result]); + }); + }) + ; + + $loop->run(); + return Command::SUCCESS; } + + private function createRunner(InputInterface $input): Runner + { + $passedConfig = $input->getOption('config'); + $passedRules = $input->getOption('rules'); + + if (null !== $passedConfig && null !== $passedRules) { + throw new \RuntimeException('Passing both `--config` and `--rules` options is not allowed'); + } + + // There's no one single source of truth when it comes to fixing single file, we need to collect statuses from events. + $this->eventDispatcher->addListener(FixerFileProcessedEvent::NAME, function (FixerFileProcessedEvent $event): void { + $this->events[] = $event; + }); + + $this->configurationResolver = new ConfigurationResolver( + new Config(), + [ + 'allow-risky' => $input->getOption('allow-risky'), + 'config' => $passedConfig, + 'dry-run' => $input->getOption('dry-run'), + 'rules' => $passedRules, + 'path' => [], + 'path-mode' => ConfigurationResolver::PATH_MODE_OVERRIDE, + 'using-cache' => $input->getOption('using-cache'), + 'cache-file' => $input->getOption('cache-file'), + 'diff' => $input->getOption('diff'), + ], + getcwd(), + $this->toolInfo + ); + + return new Runner( + new RunnerConfig( + $this->configurationResolver->isDryRun(), + false, // @TODO Pass this option to the runner + ParallelConfig::sequential() // IMPORTANT! Worker must run in sequential mode + ), + null, // Paths are known when parallelisation server requests new chunk, not now + $this->configurationResolver->getFixers(), + $this->configurationResolver->getDiffer(), + $this->eventDispatcher, + $this->errorsManager, + $this->configurationResolver->getLinter(), + $this->configurationResolver->getCacheManager(), + $this->configurationResolver->getDirectory() + ); + } } diff --git a/src/Error/ErrorsManager.php b/src/Error/ErrorsManager.php index d1d9e0cfc04..e6333db80f1 100644 --- a/src/Error/ErrorsManager.php +++ b/src/Error/ErrorsManager.php @@ -58,6 +58,16 @@ public function getLintErrors(): array return array_filter($this->errors, static fn (Error $error): bool => Error::TYPE_LINT === $error->getType()); } + /** + * Returns errors reported for specified path. + * + * @return list + */ + public function forPath(string $path): array + { + return array_values(array_filter($this->errors, static fn (Error $error): bool => $path === $error->getFilePath())); + } + /** * Returns true if no errors were reported. */ diff --git a/src/Runner/Parallel/ParallelConfig.php b/src/Runner/Parallel/ParallelConfig.php index 1d50a315cfd..8e8c3e91124 100644 --- a/src/Runner/Parallel/ParallelConfig.php +++ b/src/Runner/Parallel/ParallelConfig.php @@ -23,8 +23,12 @@ final class ParallelConfig private int $maxProcesses; private int $processTimeout; - public function __construct(int $maxProcesses = 1, int $filesPerProcess = 10, int $processTimeout = 0) + public function __construct(int $maxProcesses = 1, int $filesPerProcess = 10, int $processTimeout = 120) { + if ($maxProcesses <= 0 || $filesPerProcess <= 0 || $processTimeout <= 0) { + throw new ParallelisationException('Invalid parallelisation configuration: only positive integers are allowed'); + } + $this->maxProcesses = $maxProcesses; $this->filesPerProcess = $filesPerProcess; $this->processTimeout = $processTimeout; @@ -45,11 +49,16 @@ public function getProcessTimeout(): int return $this->processTimeout; } + public static function sequential(): self + { + return new self(1); + } + /** * @TODO Automatic detection of available cores */ public static function detect(): self { - return new self(); + return self::sequential(); } } diff --git a/src/Runner/Parallel/ParallelisationException.php b/src/Runner/Parallel/ParallelisationException.php new file mode 100644 index 00000000000..5460519dd62 --- /dev/null +++ b/src/Runner/Parallel/ParallelisationException.php @@ -0,0 +1,30 @@ + + * Dariusz Rumiński + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace PhpCsFixer\Runner\Parallel; + +/** + * Common exception for all the errors related to parallelisation. + * + * @author Greg Korba + * + * @internal + */ +final class ParallelisationException extends \RuntimeException +{ + public static function forUnknownIdentifier(ProcessIdentifier $identifier): self + { + return new self('Unknown process identifier: '.(string) $identifier); + } +} diff --git a/src/Runner/Parallel/Process.php b/src/Runner/Parallel/Process.php new file mode 100644 index 00000000000..b551b3d49f5 --- /dev/null +++ b/src/Runner/Parallel/Process.php @@ -0,0 +1,254 @@ + + * Dariusz Rumiński + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace PhpCsFixer\Runner\Parallel; + +use PhpCsFixer\Console\Command\FixCommand; +use PhpCsFixer\Runner\RunnerConfig; +use PhpCsFixer\ToolInfo; +use React\ChildProcess\Process as ReactProcess; +use React\EventLoop\LoopInterface; +use React\EventLoop\TimerInterface; +use React\Stream\ReadableStreamInterface; +use React\Stream\WritableStreamInterface; +use Symfony\Component\Console\Application; +use Symfony\Component\Console\Input\ArgvInput; + +/** + * Represents single process that is handled within parallel run. + * Inspired by: + * - https://github.com/phpstan/phpstan-src/blob/9ce425bca5337039fb52c0acf96a20a2b8ace490/src/Parallel/Process.php + * - https://github.com/phpstan/phpstan-src/blob/1477e752b4b5893f323b6d2c43591e68b3d85003/src/Process/ProcessHelper.php. + * + * @author Greg Korba + * + * @internal + */ +final class Process +{ + // Properties required for process instantiation + private string $command; + private LoopInterface $loop; + private int $timeoutSeconds; + + // Properties required for process execution + private ReactProcess $process; + private ?WritableStreamInterface $in = null; + + /** @var false|resource */ + private $stdErr; + + /** @var false|resource */ + private $stdOut; + + /** @var callable(mixed[]): void */ + private $onData; + + /** @var callable(\Throwable): void */ + private $onError; + + private ?TimerInterface $timer = null; + + public function __construct(string $command, LoopInterface $loop, int $timeoutSeconds) + { + $this->command = $command; + $this->loop = $loop; + $this->timeoutSeconds = $timeoutSeconds; + } + + public static function create( + LoopInterface $loop, + RunnerConfig $runnerConfig, + ProcessIdentifier $identifier, + int $serverPort + ): self { + $input = self::getArgvInput(); + + $commandArgs = [ + 'php', + escapeshellarg($_SERVER['argv'][0]), + 'worker', + '--port', + (string) $serverPort, + '--identifier', + escapeshellarg((string) $identifier), + ]; + + if ($runnerConfig->isDryRun()) { + $commandArgs[] = '--dry-run'; + } + + if (filter_var($input->getOption('diff'), FILTER_VALIDATE_BOOLEAN)) { + $commandArgs[] = '--diff'; + } + + foreach (['allow-risky', 'config', 'rules', 'using-cache', 'cache-file'] as $option) { + $optionValue = $input->getOption($option); + + if (null !== $optionValue) { + $commandArgs[] = "--{$option}"; + $commandArgs[] = escapeshellarg($optionValue); + } + } + + return new self( + implode(' ', $commandArgs), + $loop, + $runnerConfig->getParallelConfig()->getProcessTimeout() + ); + } + + /** + * @param callable(mixed[] $json): void $onData callback to be called when data is received from the parallelisation operator + * @param callable(\Throwable $exception): void $onError callback to be called when an exception occurs + * @param callable(?int $exitCode, string $output): void $onExit callback to be called when the process exits + */ + public function start(callable $onData, callable $onError, callable $onExit): void + { + $this->stdOut = tmpfile(); + if (false === $this->stdOut) { + throw new ParallelisationException('Failed creating temp file for stdOut.'); + } + + $this->stdErr = tmpfile(); + if (false === $this->stdErr) { + throw new ParallelisationException('Failed creating temp file for stdErr.'); + } + + $this->onData = $onData; + $this->onError = $onError; + + $this->process = new ReactProcess($this->command, null, null, [ + 1 => $this->stdOut, + 2 => $this->stdErr, + ]); + $this->process->start($this->loop); + $this->process->on('exit', function ($exitCode) use ($onExit): void { + $this->cancelTimer(); + + $output = ''; + rewind($this->stdOut); + $stdOut = stream_get_contents($this->stdOut); + if (\is_string($stdOut)) { + $output .= $stdOut; + } + + rewind($this->stdErr); + $stdErr = stream_get_contents($this->stdErr); + if (\is_string($stdErr)) { + $output .= $stdErr; + } + + $onExit($exitCode, $output); + + fclose($this->stdOut); + fclose($this->stdErr); + }); + } + + /** + * Handles requests from parallelisation operator to its worker (spawned process). + * + * @param mixed[] $data + */ + public function request(array $data): void + { + $this->cancelTimer(); // Configured process timeout actually means "chunk timeout" (each request resets timer) + + if (null === $this->in) { + throw new ParallelisationException( + 'Process not connected with parallelisation operator, ensure `bindConnection()` was called' + ); + } + + $this->in->write($data); + $this->timer = $this->loop->addTimer($this->timeoutSeconds, function (): void { + $onError = $this->onError; + $onError( + new \Exception( + sprintf( + 'Child process timed out after %d seconds. Try making it longer using `ParallelConfig`.', + $this->timeoutSeconds + ) + ) + ); + }); + } + + public function quit(): void + { + $this->cancelTimer(); + if (!$this->process->isRunning()) { + return; + } + + foreach ($this->process->pipes as $pipe) { + $pipe->close(); + } + + if (null === $this->in) { + return; + } + + $this->in->end(); + } + + public function bindConnection(ReadableStreamInterface $out, WritableStreamInterface $in): void + { + $this->in = $in; + + $in->on('error', function (\Throwable $error): void { + ($this->onError)($error); + }); + + $out->on('data', function (array $json): void { + $this->cancelTimer(); + if ('result' !== $json['action']) { + return; + } + + ($this->onData)($json['result']); + }); + $out->on('error', function (\Throwable $error): void { + ($this->onError)($error); + }); + } + + /** + * Probably we should pass the input from the fix/check command explicitly, so it does not have to be re-created, + * but for now it's good enough to simulate it here. It works as expected and we don't need to refactor the full + * path from the CLI command, through Runner, up to this class. + */ + private static function getArgvInput(): ArgvInput + { + $fixCommand = new FixCommand(new ToolInfo()); + $application = new Application(); + $application->add($fixCommand); + + // In order to have full list of options supported by the command (e.g. `--verbose`) + $fixCommand->mergeApplicationDefinition(false); + + return new ArgvInput(null, $fixCommand->getDefinition()); + } + + private function cancelTimer(): void + { + if (null === $this->timer) { + return; + } + + $this->loop->cancelTimer($this->timer); + $this->timer = null; + } +} diff --git a/src/Runner/Parallel/ProcessIdentifier.php b/src/Runner/Parallel/ProcessIdentifier.php new file mode 100644 index 00000000000..03bb11df433 --- /dev/null +++ b/src/Runner/Parallel/ProcessIdentifier.php @@ -0,0 +1,53 @@ + + * Dariusz Rumiński + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace PhpCsFixer\Runner\Parallel; + +/** + * Represents identifier of single process that is handled within parallel run. + * + * @author Greg Korba + * + * @internal + */ +final class ProcessIdentifier implements \Stringable +{ + private const IDENTIFIER_PREFIX = 'php-cs-fixer_parallel_'; + + private string $identifier; + + private function __construct(string $identifier) + { + $this->identifier = $identifier; + } + + public function __toString(): string + { + return $this->identifier; + } + + public static function create(): self + { + return new self(uniqid(self::IDENTIFIER_PREFIX, true)); + } + + public static function fromRaw(string $identifier): self + { + if (!str_starts_with($identifier, self::IDENTIFIER_PREFIX)) { + throw new ParallelisationException(sprintf('Invalid process identifier "%s".', $identifier)); + } + + return new self($identifier); + } +} diff --git a/src/Runner/Parallel/ProcessPool.php b/src/Runner/Parallel/ProcessPool.php new file mode 100644 index 00000000000..f0caa10823b --- /dev/null +++ b/src/Runner/Parallel/ProcessPool.php @@ -0,0 +1,87 @@ + + * Dariusz Rumiński + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace PhpCsFixer\Runner\Parallel; + +use React\Socket\TcpServer; + +/** + * Represents collection of active processes that are being run in parallel. + * Inspired by {@see https://github.com/phpstan/phpstan-src/blob/ed68345a82992775112acc2c2bd639d1bd3a1a02/src/Parallel/ProcessPool.php}. + * + * @author Greg Korba + * + * @internal + */ +final class ProcessPool +{ + private TcpServer $server; + + /** @var null|(callable(): void) */ + private $onServerClose; + + /** @var array */ + private array $processes = []; + + public function __construct(TcpServer $server, ?callable $onServerClose = null) + { + $this->server = $server; + $this->onServerClose = $onServerClose; + } + + public function getProcess(ProcessIdentifier $identifier): Process + { + if (!\array_key_exists((string) $identifier, $this->processes)) { + throw ParallelisationException::forUnknownIdentifier($identifier); + } + + return $this->processes[(string) $identifier]; + } + + public function addProcess(ProcessIdentifier $identifier, Process $process): void + { + $this->processes[(string) $identifier] = $process; + } + + public function endProcessIfKnown(ProcessIdentifier $identifier): void + { + if (!\array_key_exists((string) $identifier, $this->processes)) { + return; + } + + $this->endProcess($identifier); + } + + public function endAll(): void + { + foreach (array_keys($this->processes) as $identifier) { + $this->endProcess(ProcessIdentifier::fromRaw($identifier)); + } + } + + private function endProcess(ProcessIdentifier $identifier): void + { + $this->getProcess($identifier)->quit(); + + unset($this->processes[(string) $identifier]); + + if (0 === \count($this->processes)) { + $this->server->close(); + + if (null !== $this->onServerClose) { + ($this->onServerClose)(); + } + } + } +} diff --git a/src/Runner/Runner.php b/src/Runner/Runner.php index f0462b6e41c..53ef51f86d7 100644 --- a/src/Runner/Runner.php +++ b/src/Runner/Runner.php @@ -14,6 +14,8 @@ namespace PhpCsFixer\Runner; +use Clue\React\NDJson\Decoder; +use Clue\React\NDJson\Encoder; use PhpCsFixer\AbstractFixer; use PhpCsFixer\Cache\CacheManagerInterface; use PhpCsFixer\Cache\Directory; @@ -27,7 +29,14 @@ use PhpCsFixer\Linter\LinterInterface; use PhpCsFixer\Linter\LintingException; use PhpCsFixer\Linter\LintingResultInterface; +use PhpCsFixer\Runner\Parallel\ParallelisationException; +use PhpCsFixer\Runner\Parallel\Process; +use PhpCsFixer\Runner\Parallel\ProcessIdentifier; +use PhpCsFixer\Runner\Parallel\ProcessPool; use PhpCsFixer\Tokenizer\Tokens; +use React\EventLoop\StreamSelectLoop; +use React\Socket\ConnectionInterface; +use React\Socket\TcpServer; use Symfony\Component\EventDispatcher\EventDispatcherInterface; use Symfony\Component\Filesystem\Exception\IOException; use Symfony\Contracts\EventDispatcher\Event; @@ -37,6 +46,8 @@ * @author Greg Korba * * @phpstan-type _RunResult array, diff: string}> + * + * @internal */ final class Runner { @@ -55,9 +66,11 @@ final class Runner private LinterInterface $linter; /** - * @var \Traversable<\SplFileInfo> + * @var ?iterable<\SplFileInfo> */ - private $finder; + private $fileIterator; + + private int $fileCount; /** * @var list @@ -65,12 +78,12 @@ final class Runner private array $fixers; /** - * @param \Traversable<\SplFileInfo> $finder - * @param list $fixers + * @param null|iterable<\SplFileInfo> $fileIterator + * @param list $fixers */ public function __construct( RunnerConfig $runnerConfig, - \Traversable $finder, + ?iterable $fileIterator, array $fixers, DifferInterface $differ, ?EventDispatcherInterface $eventDispatcher, @@ -80,7 +93,7 @@ public function __construct( ?DirectoryInterface $directory = null ) { $this->runnerConfig = $runnerConfig; - $this->finder = $finder; + $this->fileIterator = $fileIterator; $this->fixers = $fixers; $this->differ = $differ; $this->eventDispatcher = $eventDispatcher; @@ -90,6 +103,14 @@ public function __construct( $this->directory = $directory ?? new Directory(''); } + /** + * @param iterable<\SplFileInfo> $fileIterator + */ + public function setFileIterator(iterable $fileIterator): void + { + $this->fileIterator = $fileIterator; + } + /** * @return _RunResult */ @@ -101,11 +122,136 @@ public function fix(): array } /** + * Heavily inspired by {@see https://github.com/phpstan/phpstan-src/blob/9ce425bca5337039fb52c0acf96a20a2b8ace490/src/Parallel/ParallelAnalyser.php}. + * * @return _RunResult */ private function fixParallel(): array { - throw new \RuntimeException('NOT IMPLEMENTED YET'); + $changed = []; + $streamSelectLoop = new StreamSelectLoop(); + $server = new TcpServer('127.0.0.1:0', $streamSelectLoop); + $serverPort = parse_url($server->getAddress() ?? '', PHP_URL_PORT); + + if (!is_numeric($serverPort)) { + throw new ParallelisationException(sprintf( + 'Unable to parse server port from "%s"', + $server->getAddress() ?? '' + )); + } + + $processPool = new ProcessPool($server); + $fileIterator = $this->getFileIterator(); + $fileIterator->rewind(); + + $fileChunk = function () use ($fileIterator): array { + $files = []; + + while (\count($files) < $this->runnerConfig->getParallelConfig()->getFilesPerProcess()) { + $current = $fileIterator->current(); + + if (null === $current) { + break; + } + + $files[] = $current->getRealPath(); + + $fileIterator->next(); + } + + return $files; + }; + + // [REACT] Handle worker's handshake (init connection) + $server->on('connection', static function (ConnectionInterface $connection) use ($processPool, $fileChunk): void { + $jsonInvalidUtf8Ignore = \defined('JSON_INVALID_UTF8_IGNORE') ? JSON_INVALID_UTF8_IGNORE : 0; + $decoder = new Decoder($connection, true, 512, $jsonInvalidUtf8Ignore); + $encoder = new Encoder($connection, $jsonInvalidUtf8Ignore); + + // [REACT] Bind connection when worker's process requests "hello" action (enables 2-way communication) + $decoder->on('data', static function (array $data) use ($processPool, $fileChunk, $decoder, $encoder): void { + if ('hello' !== $data['action']) { + return; + } + + $identifier = ProcessIdentifier::fromRaw($data['identifier']); + $process = $processPool->getProcess($identifier); + $process->bindConnection($decoder, $encoder); + $job = $fileChunk(); + + if (0 === \count($job)) { + $processPool->endProcessIfKnown($identifier); + + return; + } + + $process->request(['action' => 'run', 'files' => $job]); + }); + }); + + $processesToSpawn = min( + $this->runnerConfig->getParallelConfig()->getMaxProcesses(), + (int) ceil($this->fileCount / $this->runnerConfig->getParallelConfig()->getFilesPerProcess()) + ); + + for ($i = 0; $i < $processesToSpawn; ++$i) { + $identifier = ProcessIdentifier::create(); + $process = Process::create( + $streamSelectLoop, + $this->runnerConfig, + $identifier, + $serverPort, + ); + $processPool->addProcess($identifier, $process); + $process->start( + // [REACT] Handle worker's "result" action (analysis report) + function (array $analysisResult) use ($processPool, $process, $identifier, $fileChunk, &$changed): void { + foreach ($analysisResult as $file => $result) { + // Pass-back information about applied changes (only if there are any) + if (isset($result['fixInfo'])) { + $changed[$file] = $result['fixInfo']; + } + // Dispatch an event for each file processed and dispatch its status (required for progress output) + $this->dispatchEvent(FixerFileProcessedEvent::NAME, new FixerFileProcessedEvent($result['status'])); + + // @TODO Pass reported errors to the error manager + } + + // Request another chunk of files, if still available + $job = $fileChunk(); + + if (0 === \count($job)) { + $processPool->endProcessIfKnown($identifier); + + return; + } + + $process->request(['action' => 'run', 'files' => $job]); + }, + + // [REACT] Handle errors encountered during worker's execution + static function (\Throwable $error) use ($processPool): void { + // @TODO Pass-back error to the main process so it can be displayed to the user + + $processPool->endAll(); + }, + + // [REACT] Handle worker's shutdown + static function ($exitCode, string $output) use ($processPool, $identifier): void { + $processPool->endProcessIfKnown($identifier); + + if (0 === $exitCode || null === $exitCode) { + return; + } + + // @TODO Handle output string for non-zero exit codes + } + ); + } + + $streamSelectLoop->run(); + + return $changed; } /** @@ -306,16 +452,28 @@ private function dispatchEvent(string $name, Event $event): void private function getFileIterator(): LintingResultAwareFileIteratorInterface { - $finder = $this->finder; - $finderIterator = $finder instanceof \IteratorAggregate ? $finder->getIterator() : $finder; - $fileFilteredFileIterator = new FileFilterIterator( - $finderIterator, + if (null === $this->fileIterator) { + throw new \RuntimeException('File iterator is not configured. Pass paths during Runner initialisation or set them after with `setFileIterator()`.'); + } + + $fileIterator = new \ArrayIterator( + $this->fileIterator instanceof \IteratorAggregate + ? $this->fileIterator->getIterator() + : $this->fileIterator + ); + + // In order to determine the amount of required workers, we need to know how many files we need to analyse + $this->fileCount = \count(iterator_to_array($fileIterator)); + $fileIterator->rewind(); // Important! Without this 0 files would be analysed + + $fileFilterIterator = new FileFilterIterator( + $fileIterator, $this->eventDispatcher, $this->cacheManager ); return $this->linter->isAsync() - ? new FileCachingLintingFileIterator($fileFilteredFileIterator, $this->linter) - : new LintingFileIterator($fileFilteredFileIterator, $this->linter); + ? new FileCachingLintingFileIterator($fileFilterIterator, $this->linter) + : new LintingFileIterator($fileFilterIterator, $this->linter); } } diff --git a/src/Runner/RunnerConfig.php b/src/Runner/RunnerConfig.php index f1d88ee323e..18128dec0d9 100644 --- a/src/Runner/RunnerConfig.php +++ b/src/Runner/RunnerConfig.php @@ -24,12 +24,18 @@ final class RunnerConfig private bool $isDryRun = false; private bool $stopOnViolation = false; private ParallelConfig $parallelConfig; - - public function __construct(bool $isDryRun, bool $stopOnViolation, ParallelConfig $parallelConfig) - { + private ?string $configFile = null; + + public function __construct( + bool $isDryRun, + bool $stopOnViolation, + ParallelConfig $parallelConfig, + ?string $configFile = null + ) { $this->isDryRun = $isDryRun; $this->stopOnViolation = $stopOnViolation; $this->parallelConfig = $parallelConfig; + $this->configFile = $configFile; } public function isDryRun(): bool @@ -46,4 +52,9 @@ public function getParallelConfig(): ParallelConfig { return $this->parallelConfig; } + + public function getConfigFile(): ?string + { + return $this->configFile; + } } From c12f95f1b935a45f6f007b29802df921650538b9 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Sat, 27 Jan 2024 23:52:46 +0100 Subject: [PATCH 05/77] Use sequential runner by default Parallel runner must be introduced slowly, and start as an opt-in. When it's stable enough, we can change it to be default one. --- src/Config.php | 2 +- src/Console/ConfigurationResolver.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Config.php b/src/Config.php index 58cb3a2ee4a..beead176910 100644 --- a/src/Config.php +++ b/src/Config.php @@ -66,7 +66,7 @@ class Config implements ConfigInterface, ParallelRunnerConfigInterface public function __construct(string $name = 'default') { - $this->parallelRunnerConfig = ParallelConfig::detect(); + $this->parallelRunnerConfig = ParallelConfig::sequential(); // @TODO 4.0 cleanup if (Utils::isFutureModeEnabled()) { diff --git a/src/Console/ConfigurationResolver.php b/src/Console/ConfigurationResolver.php index 2fab4118157..91cc7bb8388 100644 --- a/src/Console/ConfigurationResolver.php +++ b/src/Console/ConfigurationResolver.php @@ -282,7 +282,7 @@ public function getParallelConfig(): ParallelConfig return $config instanceof ParallelRunnerConfigInterface ? $config->getParallelConfig() - : ParallelConfig::detect(); + : ParallelConfig::sequential(); } public function getConfigFile(): ?string From 9fb07666df8dd989e4747db857ec48fde274aff3 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Sun, 28 Jan 2024 00:06:33 +0100 Subject: [PATCH 06/77] Display information about parallel runner being experimental feature --- src/Console/Command/FixCommand.php | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/Console/Command/FixCommand.php b/src/Console/Command/FixCommand.php index d91e078eceb..a1a9850fa1d 100644 --- a/src/Console/Command/FixCommand.php +++ b/src/Console/Command/FixCommand.php @@ -258,6 +258,22 @@ protected function execute(InputInterface $input, OutputInterface $output): int if (null !== $stdErr) { $stdErr->writeln(Application::getAboutWithRuntime(true)); + // @TODO remove when parallel runner is mature enough and works as expected + if ($resolver->getParallelConfig()->getMaxProcesses() > 1) { + $stdErr->writeln( + sprintf( + $stdErr->isDecorated() ? '%s' : '%s', + 'Parallel runner is an experimental feature and may be unstable, use it at your own risk. Feedback highly appreciated!' + ) + ); + + $stdErr->writeln(sprintf( + 'Running analysis on %d cores with %d files per process.', + $resolver->getParallelConfig()->getMaxProcesses(), + $resolver->getParallelConfig()->getFilesPerProcess() + )); + } + $configFile = $resolver->getConfigFile(); $stdErr->writeln(sprintf('Loaded config %s%s.', $resolver->getConfig()->getName(), null === $configFile ? '' : ' from "'.$configFile.'"')); From 22d6bf3b1a287223625d3dbfb78f70de390ebeb6 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Sun, 28 Jan 2024 00:35:49 +0100 Subject: [PATCH 07/77] Auto detection for parallel config Using `ParallelConfig::detect()` can be used for utilising all available resources. Tested by running analysis natively on the host and inside Docker (with core limit set on the OrbStack level). In both scenarios it detects available cores correctly. --- composer.json | 1 + src/Runner/Parallel/ParallelConfig.php | 30 +++++++++++++++++++------- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/composer.json b/composer.json index 4e98ee57333..3df5f61e558 100644 --- a/composer.json +++ b/composer.json @@ -27,6 +27,7 @@ "clue/ndjson-react": "^1.0", "composer/semver": "^3.4", "composer/xdebug-handler": "^3.0.3", + "fidry/cpu-core-counter": "^1.0", "react/child-process": "^0.6.5", "react/event-loop": "^1.5", "react/promise": "^3.1", diff --git a/src/Runner/Parallel/ParallelConfig.php b/src/Runner/Parallel/ParallelConfig.php index 8e8c3e91124..99fa982dcd6 100644 --- a/src/Runner/Parallel/ParallelConfig.php +++ b/src/Runner/Parallel/ParallelConfig.php @@ -14,17 +14,27 @@ namespace PhpCsFixer\Runner\Parallel; +use Fidry\CpuCoreCounter\CpuCoreCounter; +use Fidry\CpuCoreCounter\Finder\DummyCpuCoreFinder; +use Fidry\CpuCoreCounter\Finder\FinderRegistry; + /** * @author Greg Korba */ final class ParallelConfig { + private const DEFAULT_FILES_PER_PROCESS = 10; + private const DEFAULT_PROCESS_TIMEOUT = 120; + private int $filesPerProcess; private int $maxProcesses; private int $processTimeout; - public function __construct(int $maxProcesses = 1, int $filesPerProcess = 10, int $processTimeout = 120) - { + public function __construct( + int $maxProcesses = 2, + int $filesPerProcess = self::DEFAULT_FILES_PER_PROCESS, + int $processTimeout = self::DEFAULT_PROCESS_TIMEOUT + ) { if ($maxProcesses <= 0 || $filesPerProcess <= 0 || $processTimeout <= 0) { throw new ParallelisationException('Invalid parallelisation configuration: only positive integers are allowed'); } @@ -54,11 +64,15 @@ public static function sequential(): self return new self(1); } - /** - * @TODO Automatic detection of available cores - */ - public static function detect(): self - { - return self::sequential(); + public static function detect( + int $filesPerProcess = self::DEFAULT_FILES_PER_PROCESS, + int $processTimeout = self::DEFAULT_PROCESS_TIMEOUT + ): self { + $counter = new CpuCoreCounter([ + ...FinderRegistry::getDefaultLogicalFinders(), + new DummyCpuCoreFinder(1), + ]); + + return new self($counter->getCount(), $filesPerProcess, $processTimeout); } } From 2afff3bfbd68aed9fef3bba506a5b7dbbf653bd9 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Sun, 28 Jan 2024 01:17:34 +0100 Subject: [PATCH 08/77] Use parallel runner with config auto-detection for the project itself --- .php-cs-fixer.dist.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.php-cs-fixer.dist.php b/.php-cs-fixer.dist.php index 0e786c4c8a5..c9585edc44e 100644 --- a/.php-cs-fixer.dist.php +++ b/.php-cs-fixer.dist.php @@ -14,8 +14,10 @@ use PhpCsFixer\Config; use PhpCsFixer\Finder; +use PhpCsFixer\Runner\Parallel\ParallelConfig; return (new Config()) + ->setParallelConfig(ParallelConfig::detect()) ->setRiskyAllowed(true) ->setRules([ '@PHP74Migration' => true, From 845b4abe748300112903bbdd913a67fef1be8e3e Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Sun, 28 Jan 2024 01:19:30 +0100 Subject: [PATCH 09/77] Pass analysis errors from worker to the main runner and report them to the error manager --- src/Console/Command/WorkerCommand.php | 10 +------ src/Error/Error.php | 30 ++++++++++++++++++- .../Parallel/ParallelisationException.php | 12 ++++++++ src/Runner/Runner.php | 14 ++++++++- 4 files changed, 55 insertions(+), 11 deletions(-) diff --git a/src/Console/Command/WorkerCommand.php b/src/Console/Command/WorkerCommand.php index 2a3abaa3774..5b159b7c356 100644 --- a/src/Console/Command/WorkerCommand.php +++ b/src/Console/Command/WorkerCommand.php @@ -18,7 +18,6 @@ use Clue\React\NDJson\Encoder; use PhpCsFixer\Config; use PhpCsFixer\Console\ConfigurationResolver; -use PhpCsFixer\Error\Error; use PhpCsFixer\Error\ErrorsManager; use PhpCsFixer\FixerFileProcessedEvent; use PhpCsFixer\Runner\Parallel\ParallelConfig; @@ -180,14 +179,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int ? $this->events[$i]->getStatus() : null; $result[$relativePath]['fixInfo'] = $analysisResult[$relativePath] ?? null; - // @TODO consider serialising whole Error, so it can be deserialised on server side and passed to error manager - $result[$relativePath]['errors'] = array_map( - static fn (Error $error): array => [ - 'type' => $error->getType(), - 'error_message' => null !== $error->getSource() ? $error->getSource()->getMessage() : null, - ], - $this->errorsManager->forPath($absolutePath) - ); + $result[$relativePath]['errors'] = $this->errorsManager->forPath($absolutePath); } $out->write(['action' => 'result', 'result' => $result]); diff --git a/src/Error/Error.php b/src/Error/Error.php index 1f14ed9f0f1..06e95a5a324 100644 --- a/src/Error/Error.php +++ b/src/Error/Error.php @@ -21,7 +21,7 @@ * * @internal */ -final class Error +final class Error implements \JsonSerializable { /** * Error which has occurred in linting phase, before applying any fixers. @@ -38,6 +38,7 @@ final class Error */ public const TYPE_LINT = 3; + /** @var self::TYPE_* */ private int $type; private string $filePath; @@ -90,4 +91,31 @@ public function getDiff(): ?string { return $this->diff; } + + /** + * @return array{ + * type: self::TYPE_*, + * filePath: string, + * source: null|array{message: string, code: int, file: string, line: int}, + * appliedFixers: list, + * diff: null|string + * } + */ + public function jsonSerialize(): array + { + return [ + 'type' => $this->type, + 'filePath' => $this->filePath, + 'source' => null !== $this->source + ? [ + 'message' => $this->source->getMessage(), + 'code' => $this->source->getCode(), + 'file' => $this->source->getFile(), + 'line' => $this->source->getLine(), + ] + : null, + 'appliedFixers' => $this->appliedFixers, + 'diff' => $this->diff, + ]; + } } diff --git a/src/Runner/Parallel/ParallelisationException.php b/src/Runner/Parallel/ParallelisationException.php index 5460519dd62..28a4b8ab9b7 100644 --- a/src/Runner/Parallel/ParallelisationException.php +++ b/src/Runner/Parallel/ParallelisationException.php @@ -27,4 +27,16 @@ public static function forUnknownIdentifier(ProcessIdentifier $identifier): self { return new self('Unknown process identifier: '.(string) $identifier); } + + /** + * @param array{message: string, code: int, file: string, line: int} $error + */ + public static function forWorkerError(array $error): self + { + $exception = new self($error['message'], $error['code']); + $exception->file = $error['file']; + $exception->line = $error['line']; + + return $exception; + } } diff --git a/src/Runner/Runner.php b/src/Runner/Runner.php index 53ef51f86d7..86dff99fdcc 100644 --- a/src/Runner/Runner.php +++ b/src/Runner/Runner.php @@ -214,7 +214,19 @@ function (array $analysisResult) use ($processPool, $process, $identifier, $file // Dispatch an event for each file processed and dispatch its status (required for progress output) $this->dispatchEvent(FixerFileProcessedEvent::NAME, new FixerFileProcessedEvent($result['status'])); - // @TODO Pass reported errors to the error manager + foreach ($result['errors'] ?? [] as $workerError) { + $error = new Error( + $workerError['type'], + $workerError['filePath'], + null !== $workerError['source'] + ? ParallelisationException::forWorkerError($workerError['source']) + : null, + $workerError['appliedFixers'], + $workerError['diff'] + ); + + $this->errorsManager->report($error); + } } // Request another chunk of files, if still available From 32355dfc8383a63f3a1b96b7993a53eff4d1ddd9 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Tue, 30 Jan 2024 00:09:29 +0100 Subject: [PATCH 10/77] Use exact same PHP binary for worker process --- src/Runner/Parallel/Process.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Runner/Parallel/Process.php b/src/Runner/Parallel/Process.php index b551b3d49f5..f4959dcfe0f 100644 --- a/src/Runner/Parallel/Process.php +++ b/src/Runner/Parallel/Process.php @@ -24,6 +24,7 @@ use React\Stream\WritableStreamInterface; use Symfony\Component\Console\Application; use Symfony\Component\Console\Input\ArgvInput; +use Symfony\Component\Process\PhpExecutableFinder; /** * Represents single process that is handled within parallel run. @@ -74,9 +75,14 @@ public static function create( int $serverPort ): self { $input = self::getArgvInput(); + $phpBinary = (new PhpExecutableFinder())->find(false); + + if (false === $phpBinary) { + throw new ParallelisationException('Cannot find PHP executable.'); + } $commandArgs = [ - 'php', + $phpBinary, escapeshellarg($_SERVER['argv'][0]), 'worker', '--port', From d5c4283051aee8a985949a65baae9d5385c633a1 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Tue, 30 Jan 2024 00:29:41 +0100 Subject: [PATCH 11/77] Remove quasi-parallel Composer script --- composer.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/composer.json b/composer.json index 3df5f61e558..028e024693f 100644 --- a/composer.json +++ b/composer.json @@ -93,7 +93,6 @@ ], "cs:check": "@php php-cs-fixer check --verbose --diff", "cs:fix": "@php php-cs-fixer fix", - "cs:fix:parallel": "echo '🔍 Will run in batches of 50 files.'; if [[ -f .php-cs-fixer.php ]]; then FIXER_CONFIG=.php-cs-fixer.php; else FIXER_CONFIG=.php-cs-fixer.dist.php; fi; php php-cs-fixer list-files --config=$FIXER_CONFIG | xargs -n 50 -P 8 php php-cs-fixer fix --config=$FIXER_CONFIG --path-mode intersection 2> /dev/null", "docs": "@php dev-tools/doc.php", "infection": "@test:mutation", "install-tools": "@composer --working-dir=dev-tools install", @@ -162,7 +161,6 @@ "auto-review": "Execute Auto-review", "cs:check": "Check coding standards", "cs:fix": "Fix coding standards", - "cs:fix:parallel": "Fix coding standards in naive parallel mode (using xargs)", "docs": "Regenerate docs", "infection": "Alias for 'test:mutation'", "install-tools": "Install DEV tools", From ea9368521ad6c0319344cdd92e598bb73adb5c6e Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Tue, 30 Jan 2024 00:30:36 +0100 Subject: [PATCH 12/77] Document usage of parallel runner --- doc/usage.rst | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/doc/usage.rst b/doc/usage.rst index d3d6942e2ca..3fc075103ff 100644 --- a/doc/usage.rst +++ b/doc/usage.rst @@ -21,11 +21,25 @@ If you do not have config file, you can run following command to fix non-hidden, php php-cs-fixer.phar fix . -With some magic of tools provided by your OS, you can also fix files in parallel: +You can also fix files in parallel, utilising more CPU cores. You can do this by using config class that implements ``PhpCsFixer\Runner\Parallel\ParallelConfig\ParallelRunnerConfigInterface``, and use ``setParallelConfig()`` method. Recommended way is to utilise auto-detecting parallel configuration: -.. code-block:: console +.. code-block:: php - php php-cs-fixer.phar list-files --config=.php-cs-fixer.dist.php | xargs -n 50 -P 8 php php-cs-fixer.phar fix --config=.php-cs-fixer.dist.php --path-mode intersection -v + setParallelConfig(ParallelConfig::detect()) + ; + +However, in some case you may want to fine-tune parallelisation with explicit values (e.g. in environments where auto-detection does not work properly and suggests more cores than it should): + +.. code-block:: php + + setParallelConfig(new ParallelConfig(4, 20)) + ; You can limit process to given file or files in a given directory and its subdirectories: From 6f11b20b2873e3e4a4f8421f2a91db15d67bbf6c Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Tue, 30 Jan 2024 01:21:14 +0100 Subject: [PATCH 13/77] Allow only static factory for creating `Process` --- src/Runner/Parallel/Process.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Runner/Parallel/Process.php b/src/Runner/Parallel/Process.php index f4959dcfe0f..58bb4805174 100644 --- a/src/Runner/Parallel/Process.php +++ b/src/Runner/Parallel/Process.php @@ -61,7 +61,7 @@ final class Process private ?TimerInterface $timer = null; - public function __construct(string $command, LoopInterface $loop, int $timeoutSeconds) + private function __construct(string $command, LoopInterface $loop, int $timeoutSeconds) { $this->command = $command; $this->loop = $loop; From 6bdfa0e2132883a9605a78623aa58e5c4c726a53 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Tue, 30 Jan 2024 02:05:41 +0100 Subject: [PATCH 14/77] Determine files count early from simple iterator --- src/Runner/Runner.php | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Runner/Runner.php b/src/Runner/Runner.php index 86dff99fdcc..4c609535bbf 100644 --- a/src/Runner/Runner.php +++ b/src/Runner/Runner.php @@ -93,6 +93,7 @@ public function __construct( ?DirectoryInterface $directory = null ) { $this->runnerConfig = $runnerConfig; + $this->fileCount = \count($fileIterator ?? []); // Required only for main process (calculating workers count) $this->fileIterator = $fileIterator; $this->fixers = $fixers; $this->differ = $differ; @@ -474,10 +475,6 @@ private function getFileIterator(): LintingResultAwareFileIteratorInterface : $this->fileIterator ); - // In order to determine the amount of required workers, we need to know how many files we need to analyse - $this->fileCount = \count(iterator_to_array($fileIterator)); - $fileIterator->rewind(); // Important! Without this 0 files would be analysed - $fileFilterIterator = new FileFilterIterator( $fileIterator, $this->eventDispatcher, From 039a1f7d34fcb7609b08fc967e2bdcbbcfe97036 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Thu, 1 Feb 2024 01:59:32 +0100 Subject: [PATCH 15/77] Revert BC break in `Runner` signature --- src/Console/Command/FixCommand.php | 13 ++--- src/Console/Command/WorkerCommand.php | 13 +++-- src/Runner/Runner.php | 60 +++++++++++++--------- tests/Runner/RunnerTest.php | 17 +++--- tests/Test/AbstractIntegrationTestCase.php | 6 +-- 5 files changed, 58 insertions(+), 51 deletions(-) diff --git a/src/Console/Command/FixCommand.php b/src/Console/Command/FixCommand.php index a1a9850fa1d..062577f5ec0 100644 --- a/src/Console/Command/FixCommand.php +++ b/src/Console/Command/FixCommand.php @@ -27,7 +27,6 @@ use PhpCsFixer\Error\ErrorsManager; use PhpCsFixer\FixerFileProcessedEvent; use PhpCsFixer\Runner\Runner; -use PhpCsFixer\Runner\RunnerConfig; use PhpCsFixer\ToolInfoInterface; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; @@ -305,20 +304,18 @@ protected function execute(InputInterface $input, OutputInterface $output): int ); $runner = new Runner( - new RunnerConfig( - $resolver->isDryRun(), - $resolver->shouldStopOnViolation(), - $resolver->getParallelConfig(), - $resolver->getConfigFile() - ), $finder, $resolver->getFixers(), $resolver->getDiffer(), ProgressOutputType::NONE !== $progressType ? $this->eventDispatcher : null, $this->errorsManager, $resolver->getLinter(), + $resolver->isDryRun(), $resolver->getCacheManager(), - $resolver->getDirectory() + $resolver->getDirectory(), + $resolver->shouldStopOnViolation(), + $resolver->getParallelConfig(), + $resolver->getConfigFile() ); $this->eventDispatcher->addListener(FixerFileProcessedEvent::NAME, [$progressOutput, 'onFixerFileProcessed']); diff --git a/src/Console/Command/WorkerCommand.php b/src/Console/Command/WorkerCommand.php index 5b159b7c356..0874b9383b9 100644 --- a/src/Console/Command/WorkerCommand.php +++ b/src/Console/Command/WorkerCommand.php @@ -22,7 +22,6 @@ use PhpCsFixer\FixerFileProcessedEvent; use PhpCsFixer\Runner\Parallel\ParallelConfig; use PhpCsFixer\Runner\Runner; -use PhpCsFixer\Runner\RunnerConfig; use PhpCsFixer\ToolInfoInterface; use React\EventLoop\StreamSelectLoop; use React\Socket\ConnectionInterface; @@ -218,25 +217,25 @@ private function createRunner(InputInterface $input): Runner 'using-cache' => $input->getOption('using-cache'), 'cache-file' => $input->getOption('cache-file'), 'diff' => $input->getOption('diff'), + 'stop-on-violation' => false, // @TODO Pass this option to the runner ], getcwd(), $this->toolInfo ); return new Runner( - new RunnerConfig( - $this->configurationResolver->isDryRun(), - false, // @TODO Pass this option to the runner - ParallelConfig::sequential() // IMPORTANT! Worker must run in sequential mode - ), null, // Paths are known when parallelisation server requests new chunk, not now $this->configurationResolver->getFixers(), $this->configurationResolver->getDiffer(), $this->eventDispatcher, $this->errorsManager, $this->configurationResolver->getLinter(), + $this->configurationResolver->isDryRun(), $this->configurationResolver->getCacheManager(), - $this->configurationResolver->getDirectory() + $this->configurationResolver->getDirectory(), + $this->configurationResolver->shouldStopOnViolation(), + ParallelConfig::sequential(), // IMPORTANT! Worker must run in sequential mode + $this->configurationResolver->getConfigFile() ); } } diff --git a/src/Runner/Runner.php b/src/Runner/Runner.php index 4c609535bbf..c8c4cdf3159 100644 --- a/src/Runner/Runner.php +++ b/src/Runner/Runner.php @@ -29,6 +29,7 @@ use PhpCsFixer\Linter\LinterInterface; use PhpCsFixer\Linter\LintingException; use PhpCsFixer\Linter\LintingResultInterface; +use PhpCsFixer\Runner\Parallel\ParallelConfig; use PhpCsFixer\Runner\Parallel\ParallelisationException; use PhpCsFixer\Runner\Parallel\Process; use PhpCsFixer\Runner\Parallel\ProcessIdentifier; @@ -46,13 +47,9 @@ * @author Greg Korba * * @phpstan-type _RunResult array, diff: string}> - * - * @internal */ final class Runner { - private RunnerConfig $runnerConfig; - private DifferInterface $differ; private ?DirectoryInterface $directory; @@ -63,10 +60,12 @@ final class Runner private CacheManagerInterface $cacheManager; + private bool $isDryRun; + private LinterInterface $linter; /** - * @var ?iterable<\SplFileInfo> + * @var ?\Traversable<\SplFileInfo> */ private $fileIterator; @@ -77,22 +76,30 @@ final class Runner */ private array $fixers; + private bool $stopOnViolation; + + private ParallelConfig $parallelConfig; + + private ?string $configFile; + /** - * @param null|iterable<\SplFileInfo> $fileIterator - * @param list $fixers + * @param null|\Traversable<\SplFileInfo> $fileIterator + * @param list $fixers */ public function __construct( - RunnerConfig $runnerConfig, - ?iterable $fileIterator, + ?\Traversable $fileIterator, array $fixers, DifferInterface $differ, ?EventDispatcherInterface $eventDispatcher, ErrorsManager $errorsManager, LinterInterface $linter, + bool $isDryRun, CacheManagerInterface $cacheManager, - ?DirectoryInterface $directory = null + ?DirectoryInterface $directory = null, + bool $stopOnViolation = false, + ?ParallelConfig $parallelConfig = null, + ?string $configFile = null ) { - $this->runnerConfig = $runnerConfig; $this->fileCount = \count($fileIterator ?? []); // Required only for main process (calculating workers count) $this->fileIterator = $fileIterator; $this->fixers = $fixers; @@ -100,8 +107,12 @@ public function __construct( $this->eventDispatcher = $eventDispatcher; $this->errorsManager = $errorsManager; $this->linter = $linter; + $this->isDryRun = $isDryRun; $this->cacheManager = $cacheManager; $this->directory = $directory ?? new Directory(''); + $this->stopOnViolation = $stopOnViolation; + $this->parallelConfig = $parallelConfig ?? ParallelConfig::sequential(); + $this->configFile = $configFile; } /** @@ -117,7 +128,7 @@ public function setFileIterator(iterable $fileIterator): void */ public function fix(): array { - return $this->runnerConfig->getParallelConfig()->getMaxProcesses() > 1 + return $this->parallelConfig->getMaxProcesses() > 1 ? $this->fixParallel() : $this->fixSequential(); } @@ -148,7 +159,7 @@ private function fixParallel(): array $fileChunk = function () use ($fileIterator): array { $files = []; - while (\count($files) < $this->runnerConfig->getParallelConfig()->getFilesPerProcess()) { + while (\count($files) < $this->parallelConfig->getFilesPerProcess()) { $current = $fileIterator->current(); if (null === $current) { @@ -191,15 +202,20 @@ private function fixParallel(): array }); $processesToSpawn = min( - $this->runnerConfig->getParallelConfig()->getMaxProcesses(), - (int) ceil($this->fileCount / $this->runnerConfig->getParallelConfig()->getFilesPerProcess()) + $this->parallelConfig->getMaxProcesses(), + (int) ceil($this->fileCount / $this->parallelConfig->getFilesPerProcess()) ); for ($i = 0; $i < $processesToSpawn; ++$i) { $identifier = ProcessIdentifier::create(); $process = Process::create( $streamSelectLoop, - $this->runnerConfig, + new RunnerConfig( + $this->isDryRun, + $this->stopOnViolation, + $this->parallelConfig, + $this->configFile + ), $identifier, $serverPort, ); @@ -285,7 +301,7 @@ private function fixSequential(): array $name = $this->directory->getRelativePathTo($file->__toString()); $changed[$name] = $fixInfo; - if ($this->runnerConfig->shouldStopOnViolation()) { + if ($this->stopOnViolation) { break; } } @@ -388,7 +404,7 @@ private function fixFile(\SplFileInfo $file, LintingResultInterface $lintingResu return null; } - if (!$this->runnerConfig->isDryRun()) { + if (!$this->isDryRun) { $fileName = $file->getRealPath(); if (!file_exists($fileName)) { @@ -469,14 +485,10 @@ private function getFileIterator(): LintingResultAwareFileIteratorInterface throw new \RuntimeException('File iterator is not configured. Pass paths during Runner initialisation or set them after with `setFileIterator()`.'); } - $fileIterator = new \ArrayIterator( + $fileFilterIterator = new FileFilterIterator( $this->fileIterator instanceof \IteratorAggregate ? $this->fileIterator->getIterator() - : $this->fileIterator - ); - - $fileFilterIterator = new FileFilterIterator( - $fileIterator, + : $this->fileIterator, $this->eventDispatcher, $this->cacheManager ); diff --git a/tests/Runner/RunnerTest.php b/tests/Runner/RunnerTest.php index b766a818197..4181237bb98 100644 --- a/tests/Runner/RunnerTest.php +++ b/tests/Runner/RunnerTest.php @@ -25,9 +25,7 @@ use PhpCsFixer\Linter\Linter; use PhpCsFixer\Linter\LinterInterface; use PhpCsFixer\Linter\LintingResultInterface; -use PhpCsFixer\Runner\Parallel\ParallelConfig; use PhpCsFixer\Runner\Runner; -use PhpCsFixer\Runner\RunnerConfig; use PhpCsFixer\Tests\TestCase; use Symfony\Component\Finder\Finder; @@ -58,15 +56,16 @@ public function testThatFixSuccessfully(): void $path = __DIR__.\DIRECTORY_SEPARATOR.'..'.\DIRECTORY_SEPARATOR.'Fixtures'.\DIRECTORY_SEPARATOR.'FixerTest'.\DIRECTORY_SEPARATOR.'fix'; $runner = new Runner( - new RunnerConfig(true, false, new ParallelConfig()), Finder::create()->in($path), $fixers, new NullDiffer(), null, new ErrorsManager(), $linter, + true, new NullCacheManager(), - new Directory($path) + new Directory($path), + false ); $changed = $runner->fix(); @@ -77,15 +76,16 @@ public function testThatFixSuccessfully(): void $path = __DIR__.\DIRECTORY_SEPARATOR.'..'.\DIRECTORY_SEPARATOR.'Fixtures'.\DIRECTORY_SEPARATOR.'FixerTest'.\DIRECTORY_SEPARATOR.'fix'; $runner = new Runner( - new RunnerConfig(true, true, new ParallelConfig()), Finder::create()->in($path), $fixers, new NullDiffer(), null, new ErrorsManager(), $linter, + true, new NullCacheManager(), new Directory($path), + true ); $changed = $runner->fix(); @@ -104,7 +104,6 @@ public function testThatFixInvalidFileReportsToErrorManager(): void $path = realpath(__DIR__.\DIRECTORY_SEPARATOR.'..').\DIRECTORY_SEPARATOR.'Fixtures'.\DIRECTORY_SEPARATOR.'FixerTest'.\DIRECTORY_SEPARATOR.'invalid'; $runner = new Runner( - new RunnerConfig(true, false, new ParallelConfig()), Finder::create()->in($path), [ new Fixer\ClassNotation\VisibilityRequiredFixer(), @@ -114,6 +113,7 @@ public function testThatFixInvalidFileReportsToErrorManager(): void null, $errorsManager, new Linter(), + true, new NullCacheManager() ); $changed = $runner->fix(); @@ -144,15 +144,16 @@ public function testThatDiffedFileIsPassedToDiffer(): void ]; $runner = new Runner( - new RunnerConfig(true, true, new ParallelConfig()), Finder::create()->in($path), $fixers, $differ, null, new ErrorsManager(), new Linter(), + true, new NullCacheManager(), - new Directory($path) + new Directory($path), + true ); $runner->fix(); diff --git a/tests/Test/AbstractIntegrationTestCase.php b/tests/Test/AbstractIntegrationTestCase.php index 3c786440888..1c984703342 100644 --- a/tests/Test/AbstractIntegrationTestCase.php +++ b/tests/Test/AbstractIntegrationTestCase.php @@ -26,9 +26,7 @@ use PhpCsFixer\Linter\LinterInterface; use PhpCsFixer\Linter\ProcessLinter; use PhpCsFixer\PhpunitConstraintIsIdenticalString\Constraint\IsIdenticalString; -use PhpCsFixer\Runner\Parallel\ParallelConfig; use PhpCsFixer\Runner\Runner; -use PhpCsFixer\Runner\RunnerConfig; use PhpCsFixer\Tests\TestCase; use PhpCsFixer\Tokenizer\Tokens; use PhpCsFixer\WhitespacesFixerConfig; @@ -257,13 +255,13 @@ protected function doTest(IntegrationCase $case): void $errorsManager = new ErrorsManager(); $fixers = self::createFixers($case); $runner = new Runner( - new RunnerConfig(false, false, new ParallelConfig()), new \ArrayIterator([new \SplFileInfo($tmpFile)]), $fixers, new UnifiedDiffer(), null, $errorsManager, $this->linter, + false, new NullCacheManager() ); @@ -317,13 +315,13 @@ protected function doTest(IntegrationCase $case): void } $runner = new Runner( - new RunnerConfig(false, false, new ParallelConfig()), new \ArrayIterator([new \SplFileInfo($tmpFile)]), array_reverse($fixers), new UnifiedDiffer(), null, $errorsManager, $this->linter, + false, new NullCacheManager() ); From e265e5104a3a1aa4b60e58a8f4fe7bf37182f782 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Thu, 1 Feb 2024 02:00:33 +0100 Subject: [PATCH 16/77] Improve type in `Error` constructor --- src/Error/Error.php | 1 + tests/Error/ErrorTest.php | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Error/Error.php b/src/Error/Error.php index 06e95a5a324..cc209aca238 100644 --- a/src/Error/Error.php +++ b/src/Error/Error.php @@ -53,6 +53,7 @@ final class Error implements \JsonSerializable private ?string $diff; /** + * @param self::TYPE_* $type * @param list $appliedFixers */ public function __construct(int $type, string $filePath, ?\Throwable $source = null, array $appliedFixers = [], ?string $diff = null) diff --git a/tests/Error/ErrorTest.php b/tests/Error/ErrorTest.php index 902bdc5d738..351c8c2af82 100644 --- a/tests/Error/ErrorTest.php +++ b/tests/Error/ErrorTest.php @@ -26,7 +26,7 @@ final class ErrorTest extends TestCase { public function testConstructorSetsValues(): void { - $type = 123; + $type = 1; $filePath = 'foo.php'; $error = new Error( @@ -43,7 +43,7 @@ public function testConstructorSetsValues(): void public function testConstructorSetsValues2(): void { - $type = 456; + $type = 2; $filePath = __FILE__; $source = new \Exception(); $appliedFixers = ['some_rule']; From 93935d9ccd3a81dade9c501fe624299dc5aecc3b Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Thu, 1 Feb 2024 08:26:56 +0100 Subject: [PATCH 17/77] Fix test for fallback ParallelConfig --- tests/Console/ConfigurationResolverTest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Console/ConfigurationResolverTest.php b/tests/Console/ConfigurationResolverTest.php index d5f8d8eed7e..d60bde3d688 100644 --- a/tests/Console/ConfigurationResolverTest.php +++ b/tests/Console/ConfigurationResolverTest.php @@ -57,10 +57,10 @@ public function testResolveParallelConfig(): void self::assertSame($parallelConfig, $resolver->getParallelConfig()); } - public function testResolveDefaultParallelConfig(): void + public function testDefaultParallelConfigFallbacksToSequential(): void { $parallelConfig = $this->createConfigurationResolver([])->getParallelConfig(); - $defaultParallelConfig = ParallelConfig::detect(); + $defaultParallelConfig = ParallelConfig::sequential(); self::assertSame($parallelConfig->getMaxProcesses(), $defaultParallelConfig->getMaxProcesses()); self::assertSame($parallelConfig->getFilesPerProcess(), $defaultParallelConfig->getFilesPerProcess()); From dd5380165148d9a15a35914bff58c66382f7c6c7 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Wed, 7 Feb 2024 01:24:27 +0100 Subject: [PATCH 18/77] Explicitly require `react/stream` Detected by Composer Require Checker --- composer.json | 1 + 1 file changed, 1 insertion(+) diff --git a/composer.json b/composer.json index 028e024693f..92de39b5574 100644 --- a/composer.json +++ b/composer.json @@ -32,6 +32,7 @@ "react/event-loop": "^1.5", "react/promise": "^3.1", "react/socket": "^1.15", + "react/stream": "^1.0", "sebastian/diff": "^4.0 || ^5.0 || ^6.0", "symfony/console": "^5.4 || ^6.0 || ^7.0", "symfony/event-dispatcher": "^5.4 || ^6.0 || ^7.0", From 1f892a65d1c3bf4b35c7048beed7ec71ee08fa56 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Wed, 7 Feb 2024 01:28:58 +0100 Subject: [PATCH 19/77] Introduce `DirectoryInterface::getAbsolutePath()` --- src/Cache/Directory.php | 5 +++++ src/Cache/DirectoryInterface.php | 2 ++ tests/Cache/FileCacheManagerTest.php | 23 +++++++++++++++-------- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/Cache/Directory.php b/src/Cache/Directory.php index 26573896512..0f319f458e3 100644 --- a/src/Cache/Directory.php +++ b/src/Cache/Directory.php @@ -28,6 +28,11 @@ public function __construct(string $directoryName) $this->directoryName = $directoryName; } + public function getAbsolutePath(): string + { + return $this->directoryName; + } + public function getRelativePathTo(string $file): string { $file = $this->normalizePath($file); diff --git a/src/Cache/DirectoryInterface.php b/src/Cache/DirectoryInterface.php index 2fdce86a8c2..c52ee6db0f3 100644 --- a/src/Cache/DirectoryInterface.php +++ b/src/Cache/DirectoryInterface.php @@ -20,4 +20,6 @@ interface DirectoryInterface { public function getRelativePathTo(string $file): string; + + public function getAbsolutePath(): string; } diff --git a/tests/Cache/FileCacheManagerTest.php b/tests/Cache/FileCacheManagerTest.php index 7dc7e6ea40e..1bc35ed2a30 100644 --- a/tests/Cache/FileCacheManagerTest.php +++ b/tests/Cache/FileCacheManagerTest.php @@ -130,7 +130,7 @@ public function testNeedFixingUsesRelativePathToFile(): void $file = '/foo/bar/baz/src/hello.php'; $relativePathToFile = 'src/hello.php'; - $directory = $this->createDirectoryDouble($relativePathToFile); + $directory = $this->createDirectoryDouble($file, $relativePathToFile); $cachedSignature = $this->createSignatureDouble(true); $signature = $this->createSignatureDouble(true); @@ -217,7 +217,7 @@ public function testSetFileUsesRelativePathToFile(): void $relativePathToFile = 'src/hello.php'; $fileContent = 'createDirectoryDouble($relativePathToFile); + $directory = $this->createDirectoryDouble($file, $relativePathToFile); $cachedSignature = $this->createSignatureDouble(true); $signature = $this->createSignatureDouble(true); @@ -237,19 +237,26 @@ private function getFile(): string return __DIR__.'/../Fixtures/.php_cs.empty-cache'; } - private function createDirectoryDouble(string $relativePathToFile): DirectoryInterface + private function createDirectoryDouble(string $absolutePath, string $relativePath): DirectoryInterface { - return new class($relativePathToFile) implements DirectoryInterface { - private string $relativePathToFile; + return new class($absolutePath, $relativePath) implements DirectoryInterface { + private string $relativePath; + private string $absolutePath; - public function __construct(string $relativePathToFile) + public function __construct(string $absolutePath, string $relativePath) { - $this->relativePathToFile = $relativePathToFile; + $this->absolutePath = $absolutePath; + $this->relativePath = $relativePath; + } + + public function getAbsolutePath(): string + { + return $this->absolutePath; } public function getRelativePathTo(string $file): string { - return $this->relativePathToFile; + return $this->relativePath; } }; } From d0364b160448161b23ad0101454bfd6290324314 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Wed, 7 Feb 2024 01:34:33 +0100 Subject: [PATCH 20/77] Support cache in parallel analysis Cache check is done during file iteration - if file is in cache and wasn't changed, then it's skip by iterator and doesn't even go to the worker. That's why `ReadonlyCacheManager` used on worker's side is a bit superfluous, but I thought it's safer to use it instead of `NullCacheManager`. Anyway, each file is stored to cache when worker reports analysis result to the parallelisation operator, so it's basically similar to sequential analysis, because file write is done only in one process. --- src/Console/Command/WorkerCommand.php | 6 +++- src/Runner/Parallel/ReadonlyCacheManager.php | 34 ++++++++++++++++++++ src/Runner/Runner.php | 1 + 3 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 src/Runner/Parallel/ReadonlyCacheManager.php diff --git a/src/Console/Command/WorkerCommand.php b/src/Console/Command/WorkerCommand.php index 0874b9383b9..fc88e933c0f 100644 --- a/src/Console/Command/WorkerCommand.php +++ b/src/Console/Command/WorkerCommand.php @@ -21,6 +21,7 @@ use PhpCsFixer\Error\ErrorsManager; use PhpCsFixer\FixerFileProcessedEvent; use PhpCsFixer\Runner\Parallel\ParallelConfig; +use PhpCsFixer\Runner\Parallel\ReadonlyCacheManager; use PhpCsFixer\Runner\Runner; use PhpCsFixer\ToolInfoInterface; use React\EventLoop\StreamSelectLoop; @@ -53,6 +54,7 @@ final class WorkerCommand extends Command private ConfigurationResolver $configurationResolver; private ErrorsManager $errorsManager; private EventDispatcherInterface $eventDispatcher; + private ReadonlyCacheManager $readonlyCacheManager; /** @var array */ private array $events; @@ -223,6 +225,8 @@ private function createRunner(InputInterface $input): Runner $this->toolInfo ); + $this->readonlyCacheManager = new ReadonlyCacheManager($this->configurationResolver->getCacheManager()); + return new Runner( null, // Paths are known when parallelisation server requests new chunk, not now $this->configurationResolver->getFixers(), @@ -231,7 +235,7 @@ private function createRunner(InputInterface $input): Runner $this->errorsManager, $this->configurationResolver->getLinter(), $this->configurationResolver->isDryRun(), - $this->configurationResolver->getCacheManager(), + $this->readonlyCacheManager, $this->configurationResolver->getDirectory(), $this->configurationResolver->shouldStopOnViolation(), ParallelConfig::sequential(), // IMPORTANT! Worker must run in sequential mode diff --git a/src/Runner/Parallel/ReadonlyCacheManager.php b/src/Runner/Parallel/ReadonlyCacheManager.php new file mode 100644 index 00000000000..d59cc445b1e --- /dev/null +++ b/src/Runner/Parallel/ReadonlyCacheManager.php @@ -0,0 +1,34 @@ + + * Dariusz Rumiński + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace PhpCsFixer\Runner\Parallel; + +use PhpCsFixer\Cache\CacheManagerInterface; + +final class ReadonlyCacheManager implements CacheManagerInterface +{ + private CacheManagerInterface $cacheManager; + + public function __construct(CacheManagerInterface $cacheManager) + { + $this->cacheManager = $cacheManager; + } + + public function needFixing(string $file, string $fileContent): bool + { + return $this->cacheManager->needFixing($file, $fileContent); + } + + public function setFile(string $file, string $fileContent): void {} +} diff --git a/src/Runner/Runner.php b/src/Runner/Runner.php index c8c4cdf3159..f8a95e9f53f 100644 --- a/src/Runner/Runner.php +++ b/src/Runner/Runner.php @@ -230,6 +230,7 @@ function (array $analysisResult) use ($processPool, $process, $identifier, $file } // Dispatch an event for each file processed and dispatch its status (required for progress output) $this->dispatchEvent(FixerFileProcessedEvent::NAME, new FixerFileProcessedEvent($result['status'])); + $this->cacheManager->setFile($file, file_get_contents($this->directory->getAbsolutePath().\DIRECTORY_SEPARATOR.$file)); foreach ($result['errors'] ?? [] as $workerError) { $error = new Error( From 63529c258a291ce9b8284b74c61c9fbcfe829fbd Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Wed, 7 Feb 2024 02:25:18 +0100 Subject: [PATCH 21/77] Save cache info in parallel analysis only when it makes sense --- src/Runner/Runner.php | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/Runner/Runner.php b/src/Runner/Runner.php index f8a95e9f53f..520998d3bd9 100644 --- a/src/Runner/Runner.php +++ b/src/Runner/Runner.php @@ -230,7 +230,16 @@ function (array $analysisResult) use ($processPool, $process, $identifier, $file } // Dispatch an event for each file processed and dispatch its status (required for progress output) $this->dispatchEvent(FixerFileProcessedEvent::NAME, new FixerFileProcessedEvent($result['status'])); - $this->cacheManager->setFile($file, file_get_contents($this->directory->getAbsolutePath().\DIRECTORY_SEPARATOR.$file)); + + if ( + FixerFileProcessedEvent::STATUS_NO_CHANGES === (int) $result['status'] + || ( + FixerFileProcessedEvent::STATUS_FIXED === (int) $result['status'] + && !$this->isDryRun + ) + ) { + $this->cacheManager->setFile($file, file_get_contents($this->directory->getAbsolutePath().\DIRECTORY_SEPARATOR.$file)); + } foreach ($result['errors'] ?? [] as $workerError) { $error = new Error( From 39a8c0ef064391610a40951343b7f5fe9587f2eb Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Wed, 7 Feb 2024 09:32:40 +0100 Subject: [PATCH 22/77] Apply Julien's suggestions from code review Co-authored-by: Julien Falque --- src/Console/Command/WorkerCommand.php | 2 +- src/Runner/Parallel/Process.php | 3 +-- src/Runner/Parallel/ProcessPool.php | 4 ++-- src/Runner/Runner.php | 2 +- src/Runner/RunnerConfig.php | 6 +++--- tests/Console/ConfigurationResolverTest.php | 6 +++--- 6 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/Console/Command/WorkerCommand.php b/src/Console/Command/WorkerCommand.php index fc88e933c0f..a3ba53bdc92 100644 --- a/src/Console/Command/WorkerCommand.php +++ b/src/Console/Command/WorkerCommand.php @@ -108,7 +108,7 @@ protected function configure(): void 'using-cache', '', InputOption::VALUE_REQUIRED, - 'Does cache should be used (can be `yes` or `no`).' + 'Should cache be used (can be `yes` or `no`).' ), new InputOption('cache-file', '', InputOption::VALUE_REQUIRED, 'The path to the cache file.'), new InputOption('diff', '', InputOption::VALUE_NONE, 'Prints diff for each file.'), diff --git a/src/Runner/Parallel/Process.php b/src/Runner/Parallel/Process.php index 58bb4805174..1eb989e1775 100644 --- a/src/Runner/Parallel/Process.php +++ b/src/Runner/Parallel/Process.php @@ -180,8 +180,7 @@ public function request(array $data): void $this->in->write($data); $this->timer = $this->loop->addTimer($this->timeoutSeconds, function (): void { - $onError = $this->onError; - $onError( + ($this->onError)( new \Exception( sprintf( 'Child process timed out after %d seconds. Try making it longer using `ParallelConfig`.', diff --git a/src/Runner/Parallel/ProcessPool.php b/src/Runner/Parallel/ProcessPool.php index f0caa10823b..59b52aa29e9 100644 --- a/src/Runner/Parallel/ProcessPool.php +++ b/src/Runner/Parallel/ProcessPool.php @@ -42,7 +42,7 @@ public function __construct(TcpServer $server, ?callable $onServerClose = null) public function getProcess(ProcessIdentifier $identifier): Process { - if (!\array_key_exists((string) $identifier, $this->processes)) { + if (!isset($this->processes[(string) $identifier])) { throw ParallelisationException::forUnknownIdentifier($identifier); } @@ -56,7 +56,7 @@ public function addProcess(ProcessIdentifier $identifier, Process $process): voi public function endProcessIfKnown(ProcessIdentifier $identifier): void { - if (!\array_key_exists((string) $identifier, $this->processes)) { + if (!isset($this->processes[(string) $identifier])) { return; } diff --git a/src/Runner/Runner.php b/src/Runner/Runner.php index 520998d3bd9..b52926a8aac 100644 --- a/src/Runner/Runner.php +++ b/src/Runner/Runner.php @@ -65,7 +65,7 @@ final class Runner private LinterInterface $linter; /** - * @var ?\Traversable<\SplFileInfo> + * @var null|\Traversable<\SplFileInfo> */ private $fileIterator; diff --git a/src/Runner/RunnerConfig.php b/src/Runner/RunnerConfig.php index 18128dec0d9..eb3dfc83abf 100644 --- a/src/Runner/RunnerConfig.php +++ b/src/Runner/RunnerConfig.php @@ -21,10 +21,10 @@ */ final class RunnerConfig { - private bool $isDryRun = false; - private bool $stopOnViolation = false; + private bool $isDryRun; + private bool $stopOnViolation; private ParallelConfig $parallelConfig; - private ?string $configFile = null; + private ?string $configFile; public function __construct( bool $isDryRun, diff --git a/tests/Console/ConfigurationResolverTest.php b/tests/Console/ConfigurationResolverTest.php index d60bde3d688..47c49587e08 100644 --- a/tests/Console/ConfigurationResolverTest.php +++ b/tests/Console/ConfigurationResolverTest.php @@ -62,9 +62,9 @@ public function testDefaultParallelConfigFallbacksToSequential(): void $parallelConfig = $this->createConfigurationResolver([])->getParallelConfig(); $defaultParallelConfig = ParallelConfig::sequential(); - self::assertSame($parallelConfig->getMaxProcesses(), $defaultParallelConfig->getMaxProcesses()); - self::assertSame($parallelConfig->getFilesPerProcess(), $defaultParallelConfig->getFilesPerProcess()); - self::assertSame($parallelConfig->getProcessTimeout(), $defaultParallelConfig->getProcessTimeout()); + self::assertSame($defaultParallelConfig->getMaxProcesses(), $parallelConfig->getMaxProcesses()); + self::assertSame($defaultParallelConfig->getFilesPerProcess(), $parallelConfig->getFilesPerProcess()); + self::assertSame($defaultParallelConfig->getProcessTimeout(), $parallelConfig->getProcessTimeout()); } public function testSetOptionWithUndefinedOption(): void From 41ed875469714a218ca1bc0f863d29487e6b8a06 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Wed, 7 Feb 2024 09:35:41 +0100 Subject: [PATCH 23/77] Sync `--using-cache` description --- src/Console/Command/FixCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Console/Command/FixCommand.php b/src/Console/Command/FixCommand.php index 062577f5ec0..dc7367bd756 100644 --- a/src/Console/Command/FixCommand.php +++ b/src/Console/Command/FixCommand.php @@ -206,7 +206,7 @@ protected function configure(): void new InputOption('config', '', InputOption::VALUE_REQUIRED, 'The path to a config file.'), new InputOption('dry-run', '', InputOption::VALUE_NONE, 'Only shows which files would have been modified.'), new InputOption('rules', '', InputOption::VALUE_REQUIRED, 'List of rules that should be run against configured paths.'), - new InputOption('using-cache', '', InputOption::VALUE_REQUIRED, 'Does cache should be used (can be `yes` or `no`).'), + new InputOption('using-cache', '', InputOption::VALUE_REQUIRED, 'Should cache be used (can be `yes` or `no`).'), new InputOption('cache-file', '', InputOption::VALUE_REQUIRED, 'The path to the cache file.'), new InputOption('diff', '', InputOption::VALUE_NONE, 'Prints diff for each file.'), new InputOption('format', '', InputOption::VALUE_REQUIRED, 'To output results in other formats.'), From 34f3de09af48f34b63bffe2fe73ed13a38ac96e7 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Wed, 7 Feb 2024 09:40:07 +0100 Subject: [PATCH 24/77] Use `InvalidArgumentException` for parallelisation misconfiguration + PHPStan narrowed types --- src/Runner/Parallel/ParallelConfig.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Runner/Parallel/ParallelConfig.php b/src/Runner/Parallel/ParallelConfig.php index 99fa982dcd6..2268cf83048 100644 --- a/src/Runner/Parallel/ParallelConfig.php +++ b/src/Runner/Parallel/ParallelConfig.php @@ -30,13 +30,18 @@ final class ParallelConfig private int $maxProcesses; private int $processTimeout; + /** + * @param positive-int $maxProcesses + * @param positive-int $filesPerProcess + * @param positive-int $processTimeout + */ public function __construct( int $maxProcesses = 2, int $filesPerProcess = self::DEFAULT_FILES_PER_PROCESS, int $processTimeout = self::DEFAULT_PROCESS_TIMEOUT ) { if ($maxProcesses <= 0 || $filesPerProcess <= 0 || $processTimeout <= 0) { - throw new ParallelisationException('Invalid parallelisation configuration: only positive integers are allowed'); + throw new \InvalidArgumentException('Invalid parallelisation configuration: only positive integers are allowed'); } $this->maxProcesses = $maxProcesses; From 4bee6dd827ff0d9f2b6a6d999c801628cc9ac348 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Thu, 8 Feb 2024 23:08:39 +0100 Subject: [PATCH 25/77] Fix BC-break in `DirectoryInterface` Use absolute file path for reporting analysis result and convert it to relative path when processing on parallelisation operator's side. It gives the same effect in terms of caching, but does not require `DirectoryInterface::getAbsolutePath()`. --- src/Cache/Directory.php | 5 ----- src/Cache/DirectoryInterface.php | 2 -- src/Console/Command/WorkerCommand.php | 6 +++--- src/Runner/Runner.php | 8 +++++--- tests/Cache/FileCacheManagerTest.php | 17 +++++------------ 5 files changed, 13 insertions(+), 25 deletions(-) diff --git a/src/Cache/Directory.php b/src/Cache/Directory.php index 0f319f458e3..26573896512 100644 --- a/src/Cache/Directory.php +++ b/src/Cache/Directory.php @@ -28,11 +28,6 @@ public function __construct(string $directoryName) $this->directoryName = $directoryName; } - public function getAbsolutePath(): string - { - return $this->directoryName; - } - public function getRelativePathTo(string $file): string { $file = $this->normalizePath($file); diff --git a/src/Cache/DirectoryInterface.php b/src/Cache/DirectoryInterface.php index c52ee6db0f3..2fdce86a8c2 100644 --- a/src/Cache/DirectoryInterface.php +++ b/src/Cache/DirectoryInterface.php @@ -20,6 +20,4 @@ interface DirectoryInterface { public function getRelativePathTo(string $file): string; - - public function getAbsolutePath(): string; } diff --git a/src/Console/Command/WorkerCommand.php b/src/Console/Command/WorkerCommand.php index a3ba53bdc92..61881c1829e 100644 --- a/src/Console/Command/WorkerCommand.php +++ b/src/Console/Command/WorkerCommand.php @@ -176,11 +176,11 @@ protected function execute(InputInterface $input, OutputInterface $output): int $relativePath = $this->configurationResolver->getDirectory()->getRelativePathTo($absolutePath); // @phpstan-ignore-next-line False-positive caused by assigning empty array to $events property - $result[$relativePath]['status'] = isset($this->events[$i]) + $result[$absolutePath]['status'] = isset($this->events[$i]) ? $this->events[$i]->getStatus() : null; - $result[$relativePath]['fixInfo'] = $analysisResult[$relativePath] ?? null; - $result[$relativePath]['errors'] = $this->errorsManager->forPath($absolutePath); + $result[$absolutePath]['fixInfo'] = $analysisResult[$relativePath] ?? null; + $result[$absolutePath]['errors'] = $this->errorsManager->forPath($absolutePath); } $out->write(['action' => 'result', 'result' => $result]); diff --git a/src/Runner/Runner.php b/src/Runner/Runner.php index b52926a8aac..1546f41293d 100644 --- a/src/Runner/Runner.php +++ b/src/Runner/Runner.php @@ -223,10 +223,12 @@ private function fixParallel(): array $process->start( // [REACT] Handle worker's "result" action (analysis report) function (array $analysisResult) use ($processPool, $process, $identifier, $fileChunk, &$changed): void { - foreach ($analysisResult as $file => $result) { + foreach ($analysisResult as $fileAbsolutePath => $result) { + $fileRelativePath = $this->directory->getRelativePathTo($fileAbsolutePath); + // Pass-back information about applied changes (only if there are any) if (isset($result['fixInfo'])) { - $changed[$file] = $result['fixInfo']; + $changed[$fileRelativePath] = $result['fixInfo']; } // Dispatch an event for each file processed and dispatch its status (required for progress output) $this->dispatchEvent(FixerFileProcessedEvent::NAME, new FixerFileProcessedEvent($result['status'])); @@ -238,7 +240,7 @@ function (array $analysisResult) use ($processPool, $process, $identifier, $file && !$this->isDryRun ) ) { - $this->cacheManager->setFile($file, file_get_contents($this->directory->getAbsolutePath().\DIRECTORY_SEPARATOR.$file)); + $this->cacheManager->setFile($fileRelativePath, file_get_contents($fileAbsolutePath)); } foreach ($result['errors'] ?? [] as $workerError) { diff --git a/tests/Cache/FileCacheManagerTest.php b/tests/Cache/FileCacheManagerTest.php index 1bc35ed2a30..943ba72104e 100644 --- a/tests/Cache/FileCacheManagerTest.php +++ b/tests/Cache/FileCacheManagerTest.php @@ -130,7 +130,7 @@ public function testNeedFixingUsesRelativePathToFile(): void $file = '/foo/bar/baz/src/hello.php'; $relativePathToFile = 'src/hello.php'; - $directory = $this->createDirectoryDouble($file, $relativePathToFile); + $directory = $this->createDirectoryDouble($relativePathToFile); $cachedSignature = $this->createSignatureDouble(true); $signature = $this->createSignatureDouble(true); @@ -217,7 +217,7 @@ public function testSetFileUsesRelativePathToFile(): void $relativePathToFile = 'src/hello.php'; $fileContent = 'createDirectoryDouble($file, $relativePathToFile); + $directory = $this->createDirectoryDouble($relativePathToFile); $cachedSignature = $this->createSignatureDouble(true); $signature = $this->createSignatureDouble(true); @@ -237,23 +237,16 @@ private function getFile(): string return __DIR__.'/../Fixtures/.php_cs.empty-cache'; } - private function createDirectoryDouble(string $absolutePath, string $relativePath): DirectoryInterface + private function createDirectoryDouble(string $relativePath): DirectoryInterface { - return new class($absolutePath, $relativePath) implements DirectoryInterface { + return new class($relativePath) implements DirectoryInterface { private string $relativePath; - private string $absolutePath; - public function __construct(string $absolutePath, string $relativePath) + public function __construct(string $relativePath) { - $this->absolutePath = $absolutePath; $this->relativePath = $relativePath; } - public function getAbsolutePath(): string - { - return $this->absolutePath; - } - public function getRelativePathTo(string $file): string { return $this->relativePath; From e3726fb51dc3aecdd2f7e4b2cfbf55ae29e6b43b Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Thu, 8 Feb 2024 23:51:03 +0100 Subject: [PATCH 26/77] Report file analysis result per file instead of per chunk It improves caching, because cache save can be triggered more frequently and errors or interrupting during processing large chunk won't cause analysis info being lost (which would lead to re-analysing file that was already processed). --- src/Console/Command/WorkerCommand.php | 33 ++++++++++----------- src/Runner/Parallel/Process.php | 11 ++++--- src/Runner/Runner.php | 41 +++++++++++++++++---------- 3 files changed, 50 insertions(+), 35 deletions(-) diff --git a/src/Console/Command/WorkerCommand.php b/src/Console/Command/WorkerCommand.php index 61881c1829e..46ce61cc562 100644 --- a/src/Console/Command/WorkerCommand.php +++ b/src/Console/Command/WorkerCommand.php @@ -21,6 +21,7 @@ use PhpCsFixer\Error\ErrorsManager; use PhpCsFixer\FixerFileProcessedEvent; use PhpCsFixer\Runner\Parallel\ParallelConfig; +use PhpCsFixer\Runner\Parallel\Process; use PhpCsFixer\Runner\Parallel\ReadonlyCacheManager; use PhpCsFixer\Runner\Runner; use PhpCsFixer\ToolInfoInterface; @@ -157,33 +158,33 @@ protected function execute(InputInterface $input, OutputInterface $output): int // [REACT] Listen for messages from the parallelisation operator (analysis requests) $in->on('data', function (array $json) use ($runner, $out): void { - if ('run' !== $json['action']) { + if (Process::WORKER_ACTION_RUN !== $json['action']) { return; } /** @var iterable $files */ $files = $json['files']; - // Reset events because we want to collect only those coming from analysed files chunk - $this->events = []; - $runner->setFileIterator(new \ArrayIterator( - array_map(static fn (string $path) => new \SplFileInfo($path), $files) - )); - $analysisResult = $runner->fix(); - - $result = []; foreach ($files as $i => $absolutePath) { $relativePath = $this->configurationResolver->getDirectory()->getRelativePathTo($absolutePath); - // @phpstan-ignore-next-line False-positive caused by assigning empty array to $events property - $result[$absolutePath]['status'] = isset($this->events[$i]) - ? $this->events[$i]->getStatus() - : null; - $result[$absolutePath]['fixInfo'] = $analysisResult[$relativePath] ?? null; - $result[$absolutePath]['errors'] = $this->errorsManager->forPath($absolutePath); + // Reset events because we want to collect only those coming from analysed files chunk + $this->events = []; + $runner->setFileIterator(new \ArrayIterator([new \SplFileInfo($absolutePath)])); + $analysisResult = $runner->fix(); + + $out->write([ + 'action' => 'result', + 'file' => $absolutePath, + // @phpstan-ignore-next-line False-positive caused by assigning empty array to $events property + 'status' => isset($this->events[0]) ? $this->events[0]->getStatus() : null, + 'fixInfo' => $analysisResult[$relativePath] ?? null, + 'errors' => $this->errorsManager->forPath($absolutePath), + ]); } - $out->write(['action' => 'result', 'result' => $result]); + // Request another file chunk (if available, the parallelisation operator will request new "run" action) + $out->write(['action' => 'getFileChunk']); }); }) ; diff --git a/src/Runner/Parallel/Process.php b/src/Runner/Parallel/Process.php index 1eb989e1775..4a9cb336426 100644 --- a/src/Runner/Parallel/Process.php +++ b/src/Runner/Parallel/Process.php @@ -38,6 +38,11 @@ */ final class Process { + public const RUNNER_ACTION_HELLO = 'hello'; + public const RUNNER_ACTION_RESULT = 'result'; + public const RUNNER_ACTION_GET_FILE_CHUNK = 'getFileChunk'; + public const WORKER_ACTION_RUN = 'run'; + // Properties required for process instantiation private string $command; private LoopInterface $loop; @@ -219,11 +224,9 @@ public function bindConnection(ReadableStreamInterface $out, WritableStreamInter $out->on('data', function (array $json): void { $this->cancelTimer(); - if ('result' !== $json['action']) { - return; - } - ($this->onData)($json['result']); + // Pass everything to the parallelisation operator, it should decide how to handle the data + ($this->onData)($json); }); $out->on('error', function (\Throwable $error): void { ($this->onError)($error); diff --git a/src/Runner/Runner.php b/src/Runner/Runner.php index 1546f41293d..90f268b56b5 100644 --- a/src/Runner/Runner.php +++ b/src/Runner/Runner.php @@ -182,7 +182,7 @@ private function fixParallel(): array // [REACT] Bind connection when worker's process requests "hello" action (enables 2-way communication) $decoder->on('data', static function (array $data) use ($processPool, $fileChunk, $decoder, $encoder): void { - if ('hello' !== $data['action']) { + if (Process::RUNNER_ACTION_HELLO !== $data['action']) { return; } @@ -221,29 +221,32 @@ private function fixParallel(): array ); $processPool->addProcess($identifier, $process); $process->start( - // [REACT] Handle worker's "result" action (analysis report) - function (array $analysisResult) use ($processPool, $process, $identifier, $fileChunk, &$changed): void { - foreach ($analysisResult as $fileAbsolutePath => $result) { + // [REACT] Handle workers' responses (multiple actions possible) + function (array $workerResponse) use ($processPool, $process, $identifier, $fileChunk, &$changed): void { + // File analysis result (we want close-to-realtime progress with frequent cache savings) + if (Process::RUNNER_ACTION_RESULT === $workerResponse['action']) { + $fileAbsolutePath = $workerResponse['file']; $fileRelativePath = $this->directory->getRelativePathTo($fileAbsolutePath); // Pass-back information about applied changes (only if there are any) - if (isset($result['fixInfo'])) { - $changed[$fileRelativePath] = $result['fixInfo']; + if (isset($workerResponse['fixInfo'])) { + $changed[$fileRelativePath] = $workerResponse['fixInfo']; } // Dispatch an event for each file processed and dispatch its status (required for progress output) - $this->dispatchEvent(FixerFileProcessedEvent::NAME, new FixerFileProcessedEvent($result['status'])); + $this->dispatchEvent(FixerFileProcessedEvent::NAME, new FixerFileProcessedEvent($workerResponse['status'])); if ( - FixerFileProcessedEvent::STATUS_NO_CHANGES === (int) $result['status'] + FixerFileProcessedEvent::STATUS_NO_CHANGES === (int) $workerResponse['status'] || ( - FixerFileProcessedEvent::STATUS_FIXED === (int) $result['status'] + FixerFileProcessedEvent::STATUS_FIXED === (int) $workerResponse['status'] && !$this->isDryRun ) ) { $this->cacheManager->setFile($fileRelativePath, file_get_contents($fileAbsolutePath)); } - foreach ($result['errors'] ?? [] as $workerError) { + // Worker requests for another file chunk when all files were processed + foreach ($workerResponse['errors'] ?? [] as $workerError) { $error = new Error( $workerError['type'], $workerError['filePath'], @@ -256,18 +259,26 @@ function (array $analysisResult) use ($processPool, $process, $identifier, $file $this->errorsManager->report($error); } + + return; } - // Request another chunk of files, if still available - $job = $fileChunk(); + if (Process::RUNNER_ACTION_GET_FILE_CHUNK === $workerResponse['action']) { + // Request another chunk of files, if still available + $job = $fileChunk(); + + if (0 === \count($job)) { + $processPool->endProcessIfKnown($identifier); + + return; + } - if (0 === \count($job)) { - $processPool->endProcessIfKnown($identifier); + $process->request(['action' => 'run', 'files' => $job]); return; } - $process->request(['action' => 'run', 'files' => $job]); + throw new ParallelisationException('Unsupported action: '.($workerResponse['action'] ?? 'n/a')); }, // [REACT] Handle errors encountered during worker's execution From c9b75bfa78ace262a635c38706f4a5029769b6e1 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Fri, 9 Feb 2024 00:30:01 +0100 Subject: [PATCH 27/77] Clean up parallel actions handling - extract constants to dedicated class with more fitting name - use constants in every place where communication between runner and worker occurs --- src/Console/Command/WorkerCommand.php | 10 ++++----- src/Runner/Parallel/ParallelAction.php | 31 ++++++++++++++++++++++++++ src/Runner/Parallel/Process.php | 5 ----- src/Runner/Runner.php | 11 ++++----- 4 files changed, 42 insertions(+), 15 deletions(-) create mode 100644 src/Runner/Parallel/ParallelAction.php diff --git a/src/Console/Command/WorkerCommand.php b/src/Console/Command/WorkerCommand.php index 46ce61cc562..5ff743b66a5 100644 --- a/src/Console/Command/WorkerCommand.php +++ b/src/Console/Command/WorkerCommand.php @@ -20,8 +20,8 @@ use PhpCsFixer\Console\ConfigurationResolver; use PhpCsFixer\Error\ErrorsManager; use PhpCsFixer\FixerFileProcessedEvent; +use PhpCsFixer\Runner\Parallel\ParallelAction; use PhpCsFixer\Runner\Parallel\ParallelConfig; -use PhpCsFixer\Runner\Parallel\Process; use PhpCsFixer\Runner\Parallel\ReadonlyCacheManager; use PhpCsFixer\Runner\Runner; use PhpCsFixer\ToolInfoInterface; @@ -148,7 +148,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $in = new Decoder($connection, true, 512, $jsonInvalidUtf8Ignore); // [REACT] Initialise connection with the parallelisation operator - $out->write(['action' => 'hello', 'identifier' => $identifier]); + $out->write(['action' => ParallelAction::RUNNER_HELLO, 'identifier' => $identifier]); $handleError = static function (\Throwable $error): void { // @TODO Handle communication errors @@ -158,7 +158,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int // [REACT] Listen for messages from the parallelisation operator (analysis requests) $in->on('data', function (array $json) use ($runner, $out): void { - if (Process::WORKER_ACTION_RUN !== $json['action']) { + if (ParallelAction::WORKER_RUN !== $json['action']) { return; } @@ -174,7 +174,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $analysisResult = $runner->fix(); $out->write([ - 'action' => 'result', + 'action' => ParallelAction::RUNNER_RESULT, 'file' => $absolutePath, // @phpstan-ignore-next-line False-positive caused by assigning empty array to $events property 'status' => isset($this->events[0]) ? $this->events[0]->getStatus() : null, @@ -184,7 +184,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int } // Request another file chunk (if available, the parallelisation operator will request new "run" action) - $out->write(['action' => 'getFileChunk']); + $out->write(['action' => ParallelAction::RUNNER_GET_FILE_CHUNK]); }); }) ; diff --git a/src/Runner/Parallel/ParallelAction.php b/src/Runner/Parallel/ParallelAction.php new file mode 100644 index 00000000000..cdc2e5c843a --- /dev/null +++ b/src/Runner/Parallel/ParallelAction.php @@ -0,0 +1,31 @@ + + * Dariusz Rumiński + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace PhpCsFixer\Runner\Parallel; + +/** + * @author Greg Korba + * + * @internal + */ +final class ParallelAction +{ + // Actions handled by the runner + public const RUNNER_HELLO = 'hello'; + public const RUNNER_RESULT = 'result'; + public const RUNNER_GET_FILE_CHUNK = 'getFileChunk'; + + // Actions handled by the worker + public const WORKER_RUN = 'run'; +} diff --git a/src/Runner/Parallel/Process.php b/src/Runner/Parallel/Process.php index 4a9cb336426..ffb31775d32 100644 --- a/src/Runner/Parallel/Process.php +++ b/src/Runner/Parallel/Process.php @@ -38,11 +38,6 @@ */ final class Process { - public const RUNNER_ACTION_HELLO = 'hello'; - public const RUNNER_ACTION_RESULT = 'result'; - public const RUNNER_ACTION_GET_FILE_CHUNK = 'getFileChunk'; - public const WORKER_ACTION_RUN = 'run'; - // Properties required for process instantiation private string $command; private LoopInterface $loop; diff --git a/src/Runner/Runner.php b/src/Runner/Runner.php index 90f268b56b5..537a42dcd52 100644 --- a/src/Runner/Runner.php +++ b/src/Runner/Runner.php @@ -29,6 +29,7 @@ use PhpCsFixer\Linter\LinterInterface; use PhpCsFixer\Linter\LintingException; use PhpCsFixer\Linter\LintingResultInterface; +use PhpCsFixer\Runner\Parallel\ParallelAction; use PhpCsFixer\Runner\Parallel\ParallelConfig; use PhpCsFixer\Runner\Parallel\ParallelisationException; use PhpCsFixer\Runner\Parallel\Process; @@ -182,7 +183,7 @@ private function fixParallel(): array // [REACT] Bind connection when worker's process requests "hello" action (enables 2-way communication) $decoder->on('data', static function (array $data) use ($processPool, $fileChunk, $decoder, $encoder): void { - if (Process::RUNNER_ACTION_HELLO !== $data['action']) { + if (ParallelAction::RUNNER_HELLO !== $data['action']) { return; } @@ -197,7 +198,7 @@ private function fixParallel(): array return; } - $process->request(['action' => 'run', 'files' => $job]); + $process->request(['action' => ParallelAction::WORKER_RUN, 'files' => $job]); }); }); @@ -224,7 +225,7 @@ private function fixParallel(): array // [REACT] Handle workers' responses (multiple actions possible) function (array $workerResponse) use ($processPool, $process, $identifier, $fileChunk, &$changed): void { // File analysis result (we want close-to-realtime progress with frequent cache savings) - if (Process::RUNNER_ACTION_RESULT === $workerResponse['action']) { + if (ParallelAction::RUNNER_RESULT === $workerResponse['action']) { $fileAbsolutePath = $workerResponse['file']; $fileRelativePath = $this->directory->getRelativePathTo($fileAbsolutePath); @@ -263,7 +264,7 @@ function (array $workerResponse) use ($processPool, $process, $identifier, $file return; } - if (Process::RUNNER_ACTION_GET_FILE_CHUNK === $workerResponse['action']) { + if (ParallelAction::RUNNER_GET_FILE_CHUNK === $workerResponse['action']) { // Request another chunk of files, if still available $job = $fileChunk(); @@ -273,7 +274,7 @@ function (array $workerResponse) use ($processPool, $process, $identifier, $file return; } - $process->request(['action' => 'run', 'files' => $job]); + $process->request(['action' => ParallelAction::WORKER_RUN, 'files' => $job]); return; } From 4d1e7aa68bb50904c686b80d962e885bf3d2e694 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Mon, 12 Feb 2024 23:35:42 +0100 Subject: [PATCH 28/77] Dirty workaround for running sequential runner in tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There were 2 problems: - when child processes were spawned, `ArgvInput` was created to pass the arguments/options to the worker, but when fix/check command was executed via command tester, `ArgvInput` was using PHPUnit's argv from top-level command (and was failing with "The "--configuration" option does not exist.") - Passing `InputInterface` instance from command to process (did an experiment with static property set in command to skip passing input through the Runner instance) was not enough because output was invalid and tests were failing. I hope we can figure out something better after discussion on code review 😉. --- phpunit.xml.dist | 1 + src/Runner/Parallel/ParallelConfig.php | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/phpunit.xml.dist b/phpunit.xml.dist index fd2e6580250..1d3fd20cd4d 100644 --- a/phpunit.xml.dist +++ b/phpunit.xml.dist @@ -56,5 +56,6 @@ + diff --git a/src/Runner/Parallel/ParallelConfig.php b/src/Runner/Parallel/ParallelConfig.php index 2268cf83048..2baffcbb1cc 100644 --- a/src/Runner/Parallel/ParallelConfig.php +++ b/src/Runner/Parallel/ParallelConfig.php @@ -73,6 +73,10 @@ public static function detect( int $filesPerProcess = self::DEFAULT_FILES_PER_PROCESS, int $processTimeout = self::DEFAULT_PROCESS_TIMEOUT ): self { + if (filter_var(getenv('PHP_CS_FIXER_TEST_SUITE'), FILTER_VALIDATE_BOOLEAN)) { + return self::sequential(); + } + $counter = new CpuCoreCounter([ ...FinderRegistry::getDefaultLogicalFinders(), new DummyCpuCoreFinder(1), From 9d965eee745e06131a313422aa0232eba634df2f Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Wed, 14 Feb 2024 00:20:45 +0100 Subject: [PATCH 29/77] Add most of the tests Some of the tests are little hacky at this point, maybe it's possible to refactor a bit to achieve better testability. Good enough for now, though. --- src/Runner/Parallel/Process.php | 12 +- src/Runner/Parallel/ProcessPool.php | 6 +- tests/Runner/Parallel/ParallelConfigTest.php | 69 ++++++++ .../Parallel/ParallelisationExceptionTest.php | 51 ++++++ .../Runner/Parallel/ProcessIdentifierTest.php | 50 ++++++ tests/Runner/Parallel/ProcessPoolTest.php | 160 ++++++++++++++++++ .../Parallel/ReadonlyCacheManagerTest.php | 84 +++++++++ tests/Runner/RunnerConfigTest.php | 64 +++++++ 8 files changed, 489 insertions(+), 7 deletions(-) create mode 100644 tests/Runner/Parallel/ParallelConfigTest.php create mode 100644 tests/Runner/Parallel/ParallelisationExceptionTest.php create mode 100644 tests/Runner/Parallel/ProcessIdentifierTest.php create mode 100644 tests/Runner/Parallel/ProcessPoolTest.php create mode 100644 tests/Runner/Parallel/ReadonlyCacheManagerTest.php create mode 100644 tests/Runner/RunnerConfigTest.php diff --git a/src/Runner/Parallel/Process.php b/src/Runner/Parallel/Process.php index ffb31775d32..da12bc97181 100644 --- a/src/Runner/Parallel/Process.php +++ b/src/Runner/Parallel/Process.php @@ -44,7 +44,7 @@ final class Process private int $timeoutSeconds; // Properties required for process execution - private ReactProcess $process; + private ?ReactProcess $process = null; private ?WritableStreamInterface $in = null; /** @var false|resource */ @@ -83,7 +83,7 @@ public static function create( $commandArgs = [ $phpBinary, - escapeshellarg($_SERVER['argv'][0]), + escapeshellarg(realpath(__DIR__.'/../../../php-cs-fixer')), 'worker', '--port', (string) $serverPort, @@ -194,7 +194,7 @@ public function request(array $data): void public function quit(): void { $this->cancelTimer(); - if (!$this->process->isRunning()) { + if (null === $this->process || !$this->process->isRunning()) { return; } @@ -242,7 +242,11 @@ private static function getArgvInput(): ArgvInput // In order to have full list of options supported by the command (e.g. `--verbose`) $fixCommand->mergeApplicationDefinition(false); - return new ArgvInput(null, $fixCommand->getDefinition()); + // Workaround for tests, where top-level PHPUnit command has args/options that don't exist in the Fixer's command + $argv = $_SERVER['argv'] ?? []; + $isFixerApplication = str_ends_with($argv[0] ?? '', 'php-cs-fixer'); + + return new ArgvInput($isFixerApplication ? $argv : [], $fixCommand->getDefinition()); } private function cancelTimer(): void diff --git a/src/Runner/Parallel/ProcessPool.php b/src/Runner/Parallel/ProcessPool.php index 59b52aa29e9..ec0734e0d30 100644 --- a/src/Runner/Parallel/ProcessPool.php +++ b/src/Runner/Parallel/ProcessPool.php @@ -14,7 +14,7 @@ namespace PhpCsFixer\Runner\Parallel; -use React\Socket\TcpServer; +use React\Socket\ServerInterface; /** * Represents collection of active processes that are being run in parallel. @@ -26,7 +26,7 @@ */ final class ProcessPool { - private TcpServer $server; + private ServerInterface $server; /** @var null|(callable(): void) */ private $onServerClose; @@ -34,7 +34,7 @@ final class ProcessPool /** @var array */ private array $processes = []; - public function __construct(TcpServer $server, ?callable $onServerClose = null) + public function __construct(ServerInterface $server, ?callable $onServerClose = null) { $this->server = $server; $this->onServerClose = $onServerClose; diff --git a/tests/Runner/Parallel/ParallelConfigTest.php b/tests/Runner/Parallel/ParallelConfigTest.php new file mode 100644 index 00000000000..34c121be669 --- /dev/null +++ b/tests/Runner/Parallel/ParallelConfigTest.php @@ -0,0 +1,69 @@ + + * Dariusz Rumiński + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace PhpCsFixer\Tests\Runner\Parallel; + +use PhpCsFixer\Runner\Parallel\ParallelConfig; +use PhpCsFixer\Tests\TestCase; + +/** + * @internal + * + * @covers \PhpCsFixer\Runner\Parallel\ParallelConfig + * + * @TODO Test `detect()` method, but first discuss the best way to do it. + */ +final class ParallelConfigTest extends TestCase +{ + /** + * @dataProvider provideExceptionIsThrownOnNegativeValuesCases + */ + public function testExceptionIsThrownOnNegativeValues( + int $maxProcesses, + int $filesPerProcess, + int $processTimeout + ): void { + $this->expectException(\InvalidArgumentException::class); + + new ParallelConfig($maxProcesses, $filesPerProcess, $processTimeout); + } + + /** + * @return iterable + */ + public static function provideExceptionIsThrownOnNegativeValuesCases(): iterable + { + yield [-1, 1, 1]; + + yield [1, -1, 1]; + + yield [1, 1, -1]; + } + + public function testGettersAreReturningProperValues(): void + { + $config = new ParallelConfig(2, 10, 120); + + self::assertSame(2, $config->getMaxProcesses()); + self::assertSame(10, $config->getFilesPerProcess()); + self::assertSame(120, $config->getProcessTimeout()); + } + + public function testSequentialConfigHasExactlyOneProcess(): void + { + $config = ParallelConfig::sequential(); + + self::assertSame(1, $config->getMaxProcesses()); + } +} diff --git a/tests/Runner/Parallel/ParallelisationExceptionTest.php b/tests/Runner/Parallel/ParallelisationExceptionTest.php new file mode 100644 index 00000000000..b3e8f83601f --- /dev/null +++ b/tests/Runner/Parallel/ParallelisationExceptionTest.php @@ -0,0 +1,51 @@ + + * Dariusz Rumiński + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace PhpCsFixer\Tests\Runner\Parallel; + +use PhpCsFixer\Runner\Parallel\ParallelisationException; +use PhpCsFixer\Runner\Parallel\ProcessIdentifier; +use PhpCsFixer\Tests\TestCase; + +/** + * @internal + * + * @covers \PhpCsFixer\Runner\Parallel\ParallelisationException + */ +final class ParallelisationExceptionTest extends TestCase +{ + public function testCreateForUnknownIdentifier(): void + { + $identifier = ProcessIdentifier::fromRaw('php-cs-fixer_parallel_foo'); + $exception = ParallelisationException::forUnknownIdentifier($identifier); + + self::assertSame('Unknown process identifier: php-cs-fixer_parallel_foo', $exception->getMessage()); + self::assertSame(0, $exception->getCode()); + } + + public function testCreateForWorkerError(): void + { + $exception = ParallelisationException::forWorkerError([ + 'message' => 'foo', + 'code' => 1, + 'file' => 'foo.php', + 'line' => 1, + ]); + + self::assertSame('foo', $exception->getMessage()); + self::assertSame(1, $exception->getCode()); + self::assertSame('foo.php', $exception->getFile()); + self::assertSame(1, $exception->getLine()); + } +} diff --git a/tests/Runner/Parallel/ProcessIdentifierTest.php b/tests/Runner/Parallel/ProcessIdentifierTest.php new file mode 100644 index 00000000000..5e74a97f60b --- /dev/null +++ b/tests/Runner/Parallel/ProcessIdentifierTest.php @@ -0,0 +1,50 @@ + + * Dariusz Rumiński + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace PhpCsFixer\Tests\Runner\Parallel; + +use PhpCsFixer\Runner\Parallel\ParallelisationException; +use PhpCsFixer\Runner\Parallel\ProcessIdentifier; +use PhpCsFixer\Tests\TestCase; + +/** + * @internal + * + * @covers \PhpCsFixer\Runner\Parallel\ProcessIdentifier + */ +final class ProcessIdentifierTest extends TestCase +{ + /** + * @dataProvider provideFromRawCases + */ + public function testFromRaw(string $rawIdentifier, bool $valid): void + { + if (!$valid) { + self::expectException(ParallelisationException::class); + } + + $identifier = ProcessIdentifier::fromRaw($rawIdentifier); + self::assertSame($rawIdentifier, (string) $identifier); + } + + /** + * @return iterable + */ + public static function provideFromRawCases(): iterable + { + yield ['php-cs-fixer_parallel_foo', true]; + + yield ['invalid', false]; + } +} diff --git a/tests/Runner/Parallel/ProcessPoolTest.php b/tests/Runner/Parallel/ProcessPoolTest.php new file mode 100644 index 00000000000..567a8d934df --- /dev/null +++ b/tests/Runner/Parallel/ProcessPoolTest.php @@ -0,0 +1,160 @@ + + * Dariusz Rumiński + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace PhpCsFixer\Tests\Runner\Parallel; + +use PhpCsFixer\Runner\Parallel\ParallelConfig; +use PhpCsFixer\Runner\Parallel\ParallelisationException; +use PhpCsFixer\Runner\Parallel\Process; +use PhpCsFixer\Runner\Parallel\ProcessIdentifier; +use PhpCsFixer\Runner\Parallel\ProcessPool; +use PhpCsFixer\Runner\RunnerConfig; +use PhpCsFixer\Tests\TestCase; +use React\EventLoop\StreamSelectLoop; +use React\Socket\ServerInterface; + +/** + * @internal + * + * @covers \PhpCsFixer\Runner\Parallel\ProcessPool + */ +final class ProcessPoolTest extends TestCase +{ + public bool $serverClosed = false; + + public function testGetProcessWithInvalidIdentifier(): void + { + self::expectException(ParallelisationException::class); + + $this->getProcessPool()->getProcess(ProcessIdentifier::create()); + } + + public function testGetProcessWithValidIdentifier(): void + { + $identifier = ProcessIdentifier::create(); + $processPool = $this->getProcessPool(); + $process = $this->createProcess($identifier); + + $processPool->addProcess($identifier, $process); + + self::assertSame($process, $processPool->getProcess($identifier)); + } + + public function testEndProcessIfKnownWithUnknownIdentifier(): void + { + $identifier1 = ProcessIdentifier::create(); + $identifier2 = ProcessIdentifier::create(); + $processPool = $this->getProcessPool(); + $process = $this->createProcess($identifier1); + + $processPool->addProcess($identifier1, $process); + + // This is unregistered process, so it does nothing + $processPool->endProcessIfKnown($identifier2); + + self::assertFalse($this->serverClosed); + } + + public function testEndProcessIfKnownWithKnownIdentifier(): void + { + $identifier = ProcessIdentifier::create(); + $processPool = $this->getProcessPool(); + $process = $this->createProcess($identifier); + $processPool->addProcess($identifier, $process); + $processPool->endProcessIfKnown($identifier); + + self::assertTrue($this->serverClosed); + } + + public function testEndAll(): void + { + $processPool = $this->getProcessPool(); + + $identifier1 = ProcessIdentifier::create(); + $process1 = $this->createProcess($identifier1); + $processPool->addProcess($identifier1, $process1); + + $identifier2 = ProcessIdentifier::create(); + $process2 = $this->createProcess($identifier2); + $processPool->addProcess($identifier2, $process2); + + $processPool->endAll(); + + self::assertTrue($this->serverClosed); + } + + private function createProcess(ProcessIdentifier $identifier): Process + { + return Process::create( + new StreamSelectLoop(), + new RunnerConfig( + true, + false, + ParallelConfig::sequential() + ), + $identifier, + 10_000 + ); + } + + private function getProcessPool(?callable $onServerClose = null): ProcessPool + { + $this->serverClosed = false; + $test = $this; + + return new ProcessPool( + new class($test) implements ServerInterface { + private ProcessPoolTest $test; + + public function __construct(ProcessPoolTest $test) + { + $this->test = $test; + } + + public function close(): void + { + $this->test->serverClosed = true; + } + + public function getAddress(): ?string + { + return null; + } + + public function pause(): void {} + + public function resume(): void {} + + /** @phpstan-ignore-next-line */ + public function on($event, callable $listener): void {} + + /** @phpstan-ignore-next-line */ + public function once($event, callable $listener): void {} + + /** @phpstan-ignore-next-line */ + public function removeListener($event, callable $listener): void {} + + /** @phpstan-ignore-next-line */ + public function removeAllListeners($event = null): void {} + + /** @phpstan-ignore-next-line */ + public function listeners($event = null): void {} + + /** @phpstan-ignore-next-line */ + public function emit($event, array $arguments = []): void {} + }, + $onServerClose + ); + } +} diff --git a/tests/Runner/Parallel/ReadonlyCacheManagerTest.php b/tests/Runner/Parallel/ReadonlyCacheManagerTest.php new file mode 100644 index 00000000000..1e21d833fdb --- /dev/null +++ b/tests/Runner/Parallel/ReadonlyCacheManagerTest.php @@ -0,0 +1,84 @@ + + * Dariusz Rumiński + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace PhpCsFixer\Tests\Runner\Parallel; + +use PhpCsFixer\Cache\CacheManagerInterface; +use PhpCsFixer\Runner\Parallel\ReadonlyCacheManager; +use PhpCsFixer\Tests\TestCase; + +/** + * @internal + * + * @covers \PhpCsFixer\Runner\Parallel\ReadonlyCacheManager + */ +final class ReadonlyCacheManagerTest extends TestCase +{ + /** + * @dataProvider provideNeedFixingCases + */ + public function testNeedFixing(bool $needsFixing): void + { + $cacheManager = new ReadonlyCacheManager($this->getInnerCacheManager($needsFixing)); + + self::assertSame($needsFixing, $cacheManager->needFixing('foo.php', 'getInnerCacheManager(false)); + $cacheManager->setFile('foo.php', ' + */ + public static function provideNeedFixingCases(): iterable + { + yield [true]; + + yield [false]; + } + + private function getInnerCacheManager(bool $needsFixing): CacheManagerInterface + { + return new class($needsFixing) implements CacheManagerInterface { + private bool $needsFixing; + + public function __construct(bool $needsFixing) + { + $this->needsFixing = $needsFixing; + } + + public function needFixing(string $file, string $fileContent): bool + { + return $this->needsFixing; + } + + public function setFile(string $file, string $fileContent): void + { + throw new \LogicException('Should not be called.'); + } + }; + } +} diff --git a/tests/Runner/RunnerConfigTest.php b/tests/Runner/RunnerConfigTest.php new file mode 100644 index 00000000000..2446b1ba13f --- /dev/null +++ b/tests/Runner/RunnerConfigTest.php @@ -0,0 +1,64 @@ + + * Dariusz Rumiński + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace PhpCsFixer\Tests\Runner; + +use PhpCsFixer\Runner\Parallel\ParallelConfig; +use PhpCsFixer\Runner\RunnerConfig; +use PhpCsFixer\Tests\TestCase; + +/** + * @internal + * + * @covers \PhpCsFixer\Runner\RunnerConfig + */ +final class RunnerConfigTest extends TestCase +{ + /** + * @dataProvider provideGettersReturnCorrectDataCases + */ + public function testGettersReturnCorrectData( + bool $isDryRun, + bool $stopOnViolation, + ParallelConfig $parallelConfig, + ?string $configFile = null + ): void { + $config = new RunnerConfig($isDryRun, $stopOnViolation, $parallelConfig, $configFile); + + self::assertSame($isDryRun, $config->isDryRun()); + self::assertSame($stopOnViolation, $config->shouldStopOnViolation()); + self::assertSame($parallelConfig, $config->getParallelConfig()); + self::assertSame($configFile, $config->getConfigFile()); + } + + /** + * @return iterable + */ + public static function provideGettersReturnCorrectDataCases(): iterable + { + yield 'null config file' => [ + false, + false, + new ParallelConfig(1, 2, 3), + null, + ]; + + yield 'config file provided' => [ + false, + false, + new ParallelConfig(1, 2, 3), + __DIR__.'/../../../.php-cs-fixer.dist.php', + ]; + } +} From 4d477ff7ecfca4166785980f1a580d8155abf332 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Wed, 14 Feb 2024 01:48:34 +0100 Subject: [PATCH 30/77] Extract process creation to separate factory Passing root process' input to the runner and then to `ProcessFactory` allows switching `InputInterface` implementation and test process creation. --- src/Console/Command/FixCommand.php | 1 + src/Console/Command/WorkerCommand.php | 1 + src/Runner/Parallel/Process.php | 76 +---------- src/Runner/Parallel/ProcessFactory.php | 81 +++++++++++ src/Runner/Runner.php | 12 +- tests/Runner/Parallel/ProcessFactoryTest.php | 133 +++++++++++++++++++ tests/Runner/Parallel/ProcessPoolTest.php | 21 ++- 7 files changed, 246 insertions(+), 79 deletions(-) create mode 100644 src/Runner/Parallel/ProcessFactory.php create mode 100644 tests/Runner/Parallel/ProcessFactoryTest.php diff --git a/src/Console/Command/FixCommand.php b/src/Console/Command/FixCommand.php index dc7367bd756..091b53e138c 100644 --- a/src/Console/Command/FixCommand.php +++ b/src/Console/Command/FixCommand.php @@ -315,6 +315,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $resolver->getDirectory(), $resolver->shouldStopOnViolation(), $resolver->getParallelConfig(), + $input, $resolver->getConfigFile() ); diff --git a/src/Console/Command/WorkerCommand.php b/src/Console/Command/WorkerCommand.php index 5ff743b66a5..df59287188e 100644 --- a/src/Console/Command/WorkerCommand.php +++ b/src/Console/Command/WorkerCommand.php @@ -240,6 +240,7 @@ private function createRunner(InputInterface $input): Runner $this->configurationResolver->getDirectory(), $this->configurationResolver->shouldStopOnViolation(), ParallelConfig::sequential(), // IMPORTANT! Worker must run in sequential mode + null, $this->configurationResolver->getConfigFile() ); } diff --git a/src/Runner/Parallel/Process.php b/src/Runner/Parallel/Process.php index da12bc97181..e2514c8bb58 100644 --- a/src/Runner/Parallel/Process.php +++ b/src/Runner/Parallel/Process.php @@ -14,17 +14,11 @@ namespace PhpCsFixer\Runner\Parallel; -use PhpCsFixer\Console\Command\FixCommand; -use PhpCsFixer\Runner\RunnerConfig; -use PhpCsFixer\ToolInfo; use React\ChildProcess\Process as ReactProcess; use React\EventLoop\LoopInterface; use React\EventLoop\TimerInterface; use React\Stream\ReadableStreamInterface; use React\Stream\WritableStreamInterface; -use Symfony\Component\Console\Application; -use Symfony\Component\Console\Input\ArgvInput; -use Symfony\Component\Process\PhpExecutableFinder; /** * Represents single process that is handled within parallel run. @@ -61,60 +55,13 @@ final class Process private ?TimerInterface $timer = null; - private function __construct(string $command, LoopInterface $loop, int $timeoutSeconds) + public function __construct(string $command, LoopInterface $loop, int $timeoutSeconds) { $this->command = $command; $this->loop = $loop; $this->timeoutSeconds = $timeoutSeconds; } - public static function create( - LoopInterface $loop, - RunnerConfig $runnerConfig, - ProcessIdentifier $identifier, - int $serverPort - ): self { - $input = self::getArgvInput(); - $phpBinary = (new PhpExecutableFinder())->find(false); - - if (false === $phpBinary) { - throw new ParallelisationException('Cannot find PHP executable.'); - } - - $commandArgs = [ - $phpBinary, - escapeshellarg(realpath(__DIR__.'/../../../php-cs-fixer')), - 'worker', - '--port', - (string) $serverPort, - '--identifier', - escapeshellarg((string) $identifier), - ]; - - if ($runnerConfig->isDryRun()) { - $commandArgs[] = '--dry-run'; - } - - if (filter_var($input->getOption('diff'), FILTER_VALIDATE_BOOLEAN)) { - $commandArgs[] = '--diff'; - } - - foreach (['allow-risky', 'config', 'rules', 'using-cache', 'cache-file'] as $option) { - $optionValue = $input->getOption($option); - - if (null !== $optionValue) { - $commandArgs[] = "--{$option}"; - $commandArgs[] = escapeshellarg($optionValue); - } - } - - return new self( - implode(' ', $commandArgs), - $loop, - $runnerConfig->getParallelConfig()->getProcessTimeout() - ); - } - /** * @param callable(mixed[] $json): void $onData callback to be called when data is received from the parallelisation operator * @param callable(\Throwable $exception): void $onError callback to be called when an exception occurs @@ -228,27 +175,6 @@ public function bindConnection(ReadableStreamInterface $out, WritableStreamInter }); } - /** - * Probably we should pass the input from the fix/check command explicitly, so it does not have to be re-created, - * but for now it's good enough to simulate it here. It works as expected and we don't need to refactor the full - * path from the CLI command, through Runner, up to this class. - */ - private static function getArgvInput(): ArgvInput - { - $fixCommand = new FixCommand(new ToolInfo()); - $application = new Application(); - $application->add($fixCommand); - - // In order to have full list of options supported by the command (e.g. `--verbose`) - $fixCommand->mergeApplicationDefinition(false); - - // Workaround for tests, where top-level PHPUnit command has args/options that don't exist in the Fixer's command - $argv = $_SERVER['argv'] ?? []; - $isFixerApplication = str_ends_with($argv[0] ?? '', 'php-cs-fixer'); - - return new ArgvInput($isFixerApplication ? $argv : [], $fixCommand->getDefinition()); - } - private function cancelTimer(): void { if (null === $this->timer) { diff --git a/src/Runner/Parallel/ProcessFactory.php b/src/Runner/Parallel/ProcessFactory.php new file mode 100644 index 00000000000..daecf691c3d --- /dev/null +++ b/src/Runner/Parallel/ProcessFactory.php @@ -0,0 +1,81 @@ + + * Dariusz Rumiński + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace PhpCsFixer\Runner\Parallel; + +use PhpCsFixer\Runner\RunnerConfig; +use React\EventLoop\LoopInterface; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Process\PhpExecutableFinder; + +/** + * @author Greg Korba + * + * @internal + */ +final class ProcessFactory +{ + private InputInterface $input; + + public function __construct(InputInterface $input) + { + $this->input = $input; + } + + public function create( + LoopInterface $loop, + RunnerConfig $runnerConfig, + ProcessIdentifier $identifier, + int $serverPort + ): Process { + $phpBinary = (new PhpExecutableFinder())->find(false); + + if (false === $phpBinary) { + throw new ParallelisationException('Cannot find PHP executable.'); + } + + $commandArgs = [ + $phpBinary, + escapeshellarg(realpath(__DIR__.'/../../../php-cs-fixer')), + 'worker', + '--port', + (string) $serverPort, + '--identifier', + escapeshellarg((string) $identifier), + ]; + + if ($runnerConfig->isDryRun()) { + $commandArgs[] = '--dry-run'; + } + + if (filter_var($this->input->getOption('diff'), FILTER_VALIDATE_BOOLEAN)) { + $commandArgs[] = '--diff'; + } + + foreach (['allow-risky', 'config', 'rules', 'using-cache', 'cache-file'] as $option) { + $optionValue = $this->input->getOption($option); + + if (null !== $optionValue) { + $commandArgs[] = "--{$option}"; + $commandArgs[] = escapeshellarg($optionValue); + } + } + + return new Process( + implode(' ', $commandArgs), + $loop, + $runnerConfig->getParallelConfig()->getProcessTimeout() + ); + } +} diff --git a/src/Runner/Runner.php b/src/Runner/Runner.php index 537a42dcd52..b9bc713bc1a 100644 --- a/src/Runner/Runner.php +++ b/src/Runner/Runner.php @@ -32,13 +32,14 @@ use PhpCsFixer\Runner\Parallel\ParallelAction; use PhpCsFixer\Runner\Parallel\ParallelConfig; use PhpCsFixer\Runner\Parallel\ParallelisationException; -use PhpCsFixer\Runner\Parallel\Process; +use PhpCsFixer\Runner\Parallel\ProcessFactory; use PhpCsFixer\Runner\Parallel\ProcessIdentifier; use PhpCsFixer\Runner\Parallel\ProcessPool; use PhpCsFixer\Tokenizer\Tokens; use React\EventLoop\StreamSelectLoop; use React\Socket\ConnectionInterface; use React\Socket\TcpServer; +use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\EventDispatcher\EventDispatcherInterface; use Symfony\Component\Filesystem\Exception\IOException; use Symfony\Contracts\EventDispatcher\Event; @@ -81,6 +82,8 @@ final class Runner private ParallelConfig $parallelConfig; + private ?InputInterface $input; + private ?string $configFile; /** @@ -99,6 +102,7 @@ public function __construct( ?DirectoryInterface $directory = null, bool $stopOnViolation = false, ?ParallelConfig $parallelConfig = null, + ?InputInterface $input = null, ?string $configFile = null ) { $this->fileCount = \count($fileIterator ?? []); // Required only for main process (calculating workers count) @@ -113,6 +117,7 @@ public function __construct( $this->directory = $directory ?? new Directory(''); $this->stopOnViolation = $stopOnViolation; $this->parallelConfig = $parallelConfig ?? ParallelConfig::sequential(); + $this->input = $input; $this->configFile = $configFile; } @@ -129,7 +134,7 @@ public function setFileIterator(iterable $fileIterator): void */ public function fix(): array { - return $this->parallelConfig->getMaxProcesses() > 1 + return $this->parallelConfig->getMaxProcesses() > 1 && null !== $this->input ? $this->fixParallel() : $this->fixSequential(); } @@ -206,10 +211,11 @@ private function fixParallel(): array $this->parallelConfig->getMaxProcesses(), (int) ceil($this->fileCount / $this->parallelConfig->getFilesPerProcess()) ); + $processFactory = new ProcessFactory($this->input); for ($i = 0; $i < $processesToSpawn; ++$i) { $identifier = ProcessIdentifier::create(); - $process = Process::create( + $process = $processFactory->create( $streamSelectLoop, new RunnerConfig( $this->isDryRun, diff --git a/tests/Runner/Parallel/ProcessFactoryTest.php b/tests/Runner/Parallel/ProcessFactoryTest.php new file mode 100644 index 00000000000..0e1a41fed47 --- /dev/null +++ b/tests/Runner/Parallel/ProcessFactoryTest.php @@ -0,0 +1,133 @@ + + * Dariusz Rumiński + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace PhpCsFixer\Tests\Runner\Parallel; + +use PhpCsFixer\Console\Command\FixCommand; +use PhpCsFixer\Preg; +use PhpCsFixer\Runner\Parallel\ParallelConfig; +use PhpCsFixer\Runner\Parallel\ProcessFactory; +use PhpCsFixer\Runner\Parallel\ProcessIdentifier; +use PhpCsFixer\Runner\RunnerConfig; +use PhpCsFixer\Tests\TestCase; +use PhpCsFixer\ToolInfo; +use React\EventLoop\StreamSelectLoop; +use Symfony\Component\Console\Application; +use Symfony\Component\Console\Input\ArrayInput; +use Symfony\Component\Console\Input\InputDefinition; + +/** + * @author Greg Korba + * + * @internal + * + * @covers \PhpCsFixer\Runner\Parallel\ProcessFactory + */ +final class ProcessFactoryTest extends TestCase +{ + private InputDefinition $inputDefinition; + + protected function setUp(): void + { + $fixCommand = new FixCommand(new ToolInfo()); + $application = new Application(); + $application->add($fixCommand); + + // In order to have full list of options supported by the command (e.g. `--verbose`) + $fixCommand->mergeApplicationDefinition(false); + + $this->inputDefinition = $fixCommand->getDefinition(); + } + + /** + * @requires OS Linux|Darwin + * + * @param array $input + * + * @dataProvider provideCreateCases + */ + public function testCreate(array $input, RunnerConfig $config, string $expectedAdditionalArgs): void + { + $factory = new ProcessFactory(new ArrayInput($input, $this->inputDefinition)); + $identifier = ProcessIdentifier::create(); + + $process = $factory->create(new StreamSelectLoop(), $config, $identifier, 1_234); + + $processReflection = new \ReflectionClass($process); + $commandReflection = $processReflection->getProperty('command'); + $commandReflection->setAccessible(true); + $command = $commandReflection->getValue($process); + + // PHP binary and Fixer executable are not fixed, so we need to remove them from the command + $command = Preg::replace('/^(.*php-cs-fixer[\'"]? )+(.+)/', '$2', $command); + + self::assertSame( + trim( + sprintf( + 'worker --port 1234 --identifier \'%s\' %s', + (string) $identifier, + trim($expectedAdditionalArgs) + ) + ), + $command + ); + + $timeoutReflection = $processReflection->getProperty('timeoutSeconds'); + $timeoutReflection->setAccessible(true); + $timeoutSeconds = $timeoutReflection->getValue($process); + + self::assertSame($config->getParallelConfig()->getProcessTimeout(), $timeoutSeconds); + } + + /** + * @return iterable, 1: RunnerConfig, 2: string}> + */ + public static function provideCreateCases(): iterable + { + yield 'no additional params' => [[], self::createRunnerConfig(false), '']; + + yield 'dry run' => [[], self::createRunnerConfig(true), '--dry-run']; + + yield 'diff enabled' => [['--diff' => true], self::createRunnerConfig(false), '--diff']; + + yield 'allow risky' => [['--allow-risky' => 'yes'], self::createRunnerConfig(false), '--allow-risky \'yes\'']; + + yield 'config' => [['--config' => 'foo.php'], self::createRunnerConfig(false), '--config \'foo.php\'']; + + yield 'rules' => [['--rules' => '@PhpCsFixer'], self::createRunnerConfig(false), '--rules \'@PhpCsFixer\'']; + + yield 'using-cache' => [['--using-cache' => 'no'], self::createRunnerConfig(false), '--using-cache \'no\'']; + + yield 'cache-file' => [ + ['--cache-file' => 'cache.json'], + self::createRunnerConfig(false), + '--cache-file \'cache.json\'', + ]; + + yield 'dry run with other options' => [ + [ + '--config' => 'conf.php', + '--diff' => true, + '--using-cache' => 'yes', + ], + self::createRunnerConfig(true), + '--dry-run --diff --config \'conf.php\' --using-cache \'yes\'', + ]; + } + + private static function createRunnerConfig(bool $dryRun): RunnerConfig + { + return new RunnerConfig($dryRun, false, ParallelConfig::sequential()); + } +} diff --git a/tests/Runner/Parallel/ProcessPoolTest.php b/tests/Runner/Parallel/ProcessPoolTest.php index 567a8d934df..6d124c46932 100644 --- a/tests/Runner/Parallel/ProcessPoolTest.php +++ b/tests/Runner/Parallel/ProcessPoolTest.php @@ -14,15 +14,20 @@ namespace PhpCsFixer\Tests\Runner\Parallel; +use PhpCsFixer\Console\Command\FixCommand; use PhpCsFixer\Runner\Parallel\ParallelConfig; use PhpCsFixer\Runner\Parallel\ParallelisationException; use PhpCsFixer\Runner\Parallel\Process; +use PhpCsFixer\Runner\Parallel\ProcessFactory; use PhpCsFixer\Runner\Parallel\ProcessIdentifier; use PhpCsFixer\Runner\Parallel\ProcessPool; use PhpCsFixer\Runner\RunnerConfig; use PhpCsFixer\Tests\TestCase; +use PhpCsFixer\ToolInfo; use React\EventLoop\StreamSelectLoop; use React\Socket\ServerInterface; +use Symfony\Component\Console\Application; +use Symfony\Component\Console\Input\ArrayInput; /** * @internal @@ -33,6 +38,20 @@ final class ProcessPoolTest extends TestCase { public bool $serverClosed = false; + private ProcessFactory $processFactory; + + protected function setUp(): void + { + $fixCommand = new FixCommand(new ToolInfo()); + $application = new Application(); + $application->add($fixCommand); + + // In order to have full list of options supported by the command (e.g. `--verbose`) + $fixCommand->mergeApplicationDefinition(false); + + $this->processFactory = new ProcessFactory(new ArrayInput([], $fixCommand->getDefinition())); + } + public function testGetProcessWithInvalidIdentifier(): void { self::expectException(ParallelisationException::class); @@ -96,7 +115,7 @@ public function testEndAll(): void private function createProcess(ProcessIdentifier $identifier): Process { - return Process::create( + return $this->processFactory->create( new StreamSelectLoop(), new RunnerConfig( true, From 2ec2ed8971d3a74920994c5edb0a533f6a197376 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Wed, 14 Feb 2024 02:14:41 +0100 Subject: [PATCH 31/77] Fix "$nextResult must not be accessed before initialization" --- src/Runner/FileCachingLintingFileIterator.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Runner/FileCachingLintingFileIterator.php b/src/Runner/FileCachingLintingFileIterator.php index 368dd8d7c49..c46cf3d33d9 100644 --- a/src/Runner/FileCachingLintingFileIterator.php +++ b/src/Runner/FileCachingLintingFileIterator.php @@ -27,8 +27,8 @@ final class FileCachingLintingFileIterator extends \CachingIterator implements LintingResultAwareFileIteratorInterface { private LinterInterface $linter; - private ?LintingResultInterface $currentResult; - private ?LintingResultInterface $nextResult; + private ?LintingResultInterface $currentResult = null; + private ?LintingResultInterface $nextResult = null; /** * @param \Iterator $iterator From 19d5bf23ab7ad6849398fcec861c1c2044f2907f Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Wed, 14 Feb 2024 02:25:48 +0100 Subject: [PATCH 32/77] Fix tests on 7.4 with lowest deps --- tests/Runner/Parallel/ProcessPoolTest.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/Runner/Parallel/ProcessPoolTest.php b/tests/Runner/Parallel/ProcessPoolTest.php index 6d124c46932..c3f80b687c1 100644 --- a/tests/Runner/Parallel/ProcessPoolTest.php +++ b/tests/Runner/Parallel/ProcessPoolTest.php @@ -156,13 +156,13 @@ public function pause(): void {} public function resume(): void {} /** @phpstan-ignore-next-line */ - public function on($event, callable $listener): void {} + public function on($event, $listener): void {} /** @phpstan-ignore-next-line */ - public function once($event, callable $listener): void {} + public function once($event, $listener): void {} /** @phpstan-ignore-next-line */ - public function removeListener($event, callable $listener): void {} + public function removeListener($event, $listener): void {} /** @phpstan-ignore-next-line */ public function removeAllListeners($event = null): void {} From cffeb9794c94d821a73d1b2d0fa43a8e9744851e Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Wed, 14 Feb 2024 19:19:47 +0100 Subject: [PATCH 33/77] Test that `FixCommand` can be run with parallel runner --- src/Console/Command/FixCommand.php | 5 ++-- tests/Console/Command/FixCommandTest.php | 37 +++++++++++++++++++++++- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/src/Console/Command/FixCommand.php b/src/Console/Command/FixCommand.php index 091b53e138c..d6ad9c15230 100644 --- a/src/Console/Command/FixCommand.php +++ b/src/Console/Command/FixCommand.php @@ -267,9 +267,10 @@ protected function execute(InputInterface $input, OutputInterface $output): int ); $stdErr->writeln(sprintf( - 'Running analysis on %d cores with %d files per process.', + 'Running analysis on %d cores with %d file%s per process.', $resolver->getParallelConfig()->getMaxProcesses(), - $resolver->getParallelConfig()->getFilesPerProcess() + $resolver->getParallelConfig()->getFilesPerProcess(), + $resolver->getParallelConfig()->getFilesPerProcess() > 1 ? 's' : '' )); } diff --git a/tests/Console/Command/FixCommandTest.php b/tests/Console/Command/FixCommandTest.php index 3b65dfadb5f..179c2631fb3 100644 --- a/tests/Console/Command/FixCommandTest.php +++ b/tests/Console/Command/FixCommandTest.php @@ -73,7 +73,42 @@ public function testEmptyFormatValue(): void } /** - * @param array $arguments + * There's no simple way to cover parallelisation with tests, because it involves a lot of hardcoded logic under the hood, + * like opening server, communicating through sockets, etc. That's why we only test `fix` command with proper + * parallel config, so runner utilises multi-processing internally. Expected outcome is information about utilising multiple CPUs. + * + * @covers \PhpCsFixer\Console\Command\WorkerCommand + * @covers \PhpCsFixer\Runner\Runner::fixParallel + */ + public function testParallelRun(): void + { + $pathToDistConfig = __DIR__.'/../../../.php-cs-fixer.dist.php'; + $configWithFixedParallelConfig = <<setRules(['header_comment' => ['header' => 'PARALLEL!']]); + \$config->setParallelConfig(new \\PhpCsFixer\\Runner\\Parallel\\ParallelConfig(2, 1, 300)); + + return \$config; + PHP; + $tmpFile = tempnam(sys_get_temp_dir(), 'php-cs-fixer-parallel-config-').'.php'; + file_put_contents($tmpFile, $configWithFixedParallelConfig); + + $cmdTester = $this->doTestExecute( + [ + '--config' => $tmpFile, + 'path' => [__DIR__], + ] + ); + + self::assertStringContainsString('Running analysis on 2 cores with 1 file per process.', $cmdTester->getDisplay()); + self::assertStringContainsString('(header_comment)', $cmdTester->getDisplay()); + self::assertSame(8, $cmdTester->getStatusCode()); + } + + /** + * @param array $arguments */ private function doTestExecute(array $arguments): CommandTester { From b92bb5259f0e546df39c813ff39b53a940da0358 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Wed, 14 Feb 2024 23:51:24 +0100 Subject: [PATCH 34/77] Allow wider range of React packages Tested on: - PHP 8.3 - PHP 8.3 with --prefer-lowest - PHP 7.4 with --prefer-lowest --- composer.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/composer.json b/composer.json index 92de39b5574..877ade73174 100644 --- a/composer.json +++ b/composer.json @@ -29,9 +29,9 @@ "composer/xdebug-handler": "^3.0.3", "fidry/cpu-core-counter": "^1.0", "react/child-process": "^0.6.5", - "react/event-loop": "^1.5", - "react/promise": "^3.1", - "react/socket": "^1.15", + "react/event-loop": "^1.0", + "react/promise": "^2.0 || ^3.0", + "react/socket": "^1.0", "react/stream": "^1.0", "sebastian/diff": "^4.0 || ^5.0 || ^6.0", "symfony/console": "^5.4 || ^6.0 || ^7.0", From df9184c92e9af5b0886775afa956b4125531c1dd Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Thu, 15 Feb 2024 02:14:32 +0100 Subject: [PATCH 35/77] Introduce `--sequential` option to `fix` command - it can be helpful when using parallel config but there's need to disable it temporarily (it's easier to add option to the command than editing config file) - it solves the internal problem with parallel runner being used for running tests that operate on command's output (parallel run produces undeterministic output because files are not processed in the strict order) --- doc/usage.rst | 2 ++ phpunit.xml.dist | 1 - src/Console/Command/FixCommand.php | 4 ++++ src/Console/ConfigurationResolver.php | 3 ++- src/Runner/Parallel/ParallelConfig.php | 4 ---- tests/Console/ConfigurationResolverTest.php | 10 +++++++++- tests/Smoke/StdinTest.php | 2 +- 7 files changed, 18 insertions(+), 8 deletions(-) diff --git a/doc/usage.rst b/doc/usage.rst index 3fc075103ff..62871b66b0f 100644 --- a/doc/usage.rst +++ b/doc/usage.rst @@ -112,6 +112,8 @@ Complete configuration for rules can be supplied using a ``json`` formatted stri The ``--dry-run`` flag will run the fixer without making changes to your files (implicitly set when you use ``check`` command). +The ``--sequential`` flag will enforce sequential analysis even if parallel config is provided. + The ``--diff`` flag can be used to let the fixer output all the changes it makes in ``udiff`` format. The ``--allow-risky`` option (pass ``yes`` or ``no``) allows you to set whether risky rules may run. Default value is taken from config file. diff --git a/phpunit.xml.dist b/phpunit.xml.dist index 1d3fd20cd4d..fd2e6580250 100644 --- a/phpunit.xml.dist +++ b/phpunit.xml.dist @@ -56,6 +56,5 @@ - diff --git a/src/Console/Command/FixCommand.php b/src/Console/Command/FixCommand.php index d6ad9c15230..bf428fb2fff 100644 --- a/src/Console/Command/FixCommand.php +++ b/src/Console/Command/FixCommand.php @@ -150,6 +150,8 @@ public function getHelp(): string The --dry-run flag will run the fixer without making changes to your files. + The --sequential flag will enforce sequential analysis even if parallel config is provided. + The --diff flag can be used to let the fixer output all the changes it makes. The --allow-risky option (pass `yes` or `no`) allows you to set whether risky rules may run. Default value is taken from config file. @@ -212,6 +214,7 @@ protected function configure(): void new InputOption('format', '', InputOption::VALUE_REQUIRED, 'To output results in other formats.'), new InputOption('stop-on-violation', '', InputOption::VALUE_NONE, 'Stop execution on first violation.'), new InputOption('show-progress', '', InputOption::VALUE_REQUIRED, 'Type of progress indicator (none, dots).'), + new InputOption('sequential', 's', InputOption::VALUE_NONE, 'Enforce sequential analysis.'), ] ); } @@ -243,6 +246,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int 'stop-on-violation' => $input->getOption('stop-on-violation'), 'verbosity' => $verbosity, 'show-progress' => $input->getOption('show-progress'), + 'sequential' => $input->getOption('sequential'), ], getcwd(), $this->toolInfo diff --git a/src/Console/ConfigurationResolver.php b/src/Console/ConfigurationResolver.php index 91cc7bb8388..211f19990e3 100644 --- a/src/Console/ConfigurationResolver.php +++ b/src/Console/ConfigurationResolver.php @@ -120,6 +120,7 @@ final class ConfigurationResolver 'path' => [], 'path-mode' => self::PATH_MODE_OVERRIDE, 'rules' => null, + 'sequential' => null, 'show-progress' => null, 'stop-on-violation' => null, 'using-cache' => null, @@ -280,7 +281,7 @@ public function getParallelConfig(): ParallelConfig { $config = $this->getConfig(); - return $config instanceof ParallelRunnerConfigInterface + return true !== $this->options['sequential'] && $config instanceof ParallelRunnerConfigInterface ? $config->getParallelConfig() : ParallelConfig::sequential(); } diff --git a/src/Runner/Parallel/ParallelConfig.php b/src/Runner/Parallel/ParallelConfig.php index 2baffcbb1cc..2268cf83048 100644 --- a/src/Runner/Parallel/ParallelConfig.php +++ b/src/Runner/Parallel/ParallelConfig.php @@ -73,10 +73,6 @@ public static function detect( int $filesPerProcess = self::DEFAULT_FILES_PER_PROCESS, int $processTimeout = self::DEFAULT_PROCESS_TIMEOUT ): self { - if (filter_var(getenv('PHP_CS_FIXER_TEST_SUITE'), FILTER_VALIDATE_BOOLEAN)) { - return self::sequential(); - } - $counter = new CpuCoreCounter([ ...FinderRegistry::getDefaultLogicalFinders(), new DummyCpuCoreFinder(1), diff --git a/tests/Console/ConfigurationResolverTest.php b/tests/Console/ConfigurationResolverTest.php index 47c49587e08..06d1c3a4d91 100644 --- a/tests/Console/ConfigurationResolverTest.php +++ b/tests/Console/ConfigurationResolverTest.php @@ -67,6 +67,14 @@ public function testDefaultParallelConfigFallbacksToSequential(): void self::assertSame($defaultParallelConfig->getProcessTimeout(), $parallelConfig->getProcessTimeout()); } + public function testCliSequentialOptionOverridesParallelConfig(): void + { + $config = (new Config())->setParallelConfig(new ParallelConfig(10)); + $resolver = $this->createConfigurationResolver(['sequential' => true], $config); + + self::assertSame(1, $resolver->getParallelConfig()->getMaxProcesses()); + } + public function testSetOptionWithUndefinedOption(): void { $this->expectException(InvalidConfigurationException::class); @@ -1163,7 +1171,7 @@ public function testResolveCommandLineInputOverridesDefault(): void $options = $definition->getOptions(); self::assertSame( - ['path-mode', 'allow-risky', 'config', 'dry-run', 'rules', 'using-cache', 'cache-file', 'diff', 'format', 'stop-on-violation', 'show-progress'], + ['path-mode', 'allow-risky', 'config', 'dry-run', 'rules', 'using-cache', 'cache-file', 'diff', 'format', 'stop-on-violation', 'show-progress', 'sequential'], array_keys($options), 'Expected options mismatch, possibly test needs updating.' ); diff --git a/tests/Smoke/StdinTest.php b/tests/Smoke/StdinTest.php index acee72616ab..400d55b3c09 100644 --- a/tests/Smoke/StdinTest.php +++ b/tests/Smoke/StdinTest.php @@ -36,7 +36,7 @@ public function testFixingStdin(): void { $cwd = __DIR__.'/../..'; - $command = 'php php-cs-fixer fix --rules=@PSR2 --dry-run --diff --using-cache=no'; + $command = 'php php-cs-fixer fix --sequential --rules=@PSR2 --dry-run --diff --using-cache=no'; $inputFile = 'tests/Fixtures/Integration/set/@PSR2.test-in.php'; $fileResult = CommandExecutor::create("{$command} {$inputFile}", $cwd)->getResult(false); From 24ac18b2a348930341fcc487d4ffeb27c431ea00 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Thu, 15 Feb 2024 03:31:40 +0100 Subject: [PATCH 36/77] More tests, more coverage --- src/Console/Command/WorkerCommand.php | 95 ++++++++++--------- tests/Console/Command/WorkerCommandTest.php | 90 ++++++++++++++++++ tests/Runner/Parallel/ParallelConfigTest.php | 8 ++ .../Runner/Parallel/ProcessIdentifierTest.php | 7 ++ 4 files changed, 155 insertions(+), 45 deletions(-) create mode 100644 tests/Console/Command/WorkerCommandTest.php diff --git a/src/Console/Command/WorkerCommand.php b/src/Console/Command/WorkerCommand.php index df59287188e..359b4cdbfbe 100644 --- a/src/Console/Command/WorkerCommand.php +++ b/src/Console/Command/WorkerCommand.php @@ -142,51 +142,56 @@ protected function execute(InputInterface $input, OutputInterface $output): int $tcpConnector = new TcpConnector($loop); $tcpConnector ->connect(sprintf('127.0.0.1:%d', $port)) - ->then(function (ConnectionInterface $connection) use ($runner, $identifier): void { - $jsonInvalidUtf8Ignore = \defined('JSON_INVALID_UTF8_IGNORE') ? JSON_INVALID_UTF8_IGNORE : 0; - $out = new Encoder($connection, $jsonInvalidUtf8Ignore); - $in = new Decoder($connection, true, 512, $jsonInvalidUtf8Ignore); - - // [REACT] Initialise connection with the parallelisation operator - $out->write(['action' => ParallelAction::RUNNER_HELLO, 'identifier' => $identifier]); - - $handleError = static function (\Throwable $error): void { - // @TODO Handle communication errors - }; - $out->on('error', $handleError); - $in->on('error', $handleError); - - // [REACT] Listen for messages from the parallelisation operator (analysis requests) - $in->on('data', function (array $json) use ($runner, $out): void { - if (ParallelAction::WORKER_RUN !== $json['action']) { - return; - } - - /** @var iterable $files */ - $files = $json['files']; - - foreach ($files as $i => $absolutePath) { - $relativePath = $this->configurationResolver->getDirectory()->getRelativePathTo($absolutePath); - - // Reset events because we want to collect only those coming from analysed files chunk - $this->events = []; - $runner->setFileIterator(new \ArrayIterator([new \SplFileInfo($absolutePath)])); - $analysisResult = $runner->fix(); - - $out->write([ - 'action' => ParallelAction::RUNNER_RESULT, - 'file' => $absolutePath, - // @phpstan-ignore-next-line False-positive caused by assigning empty array to $events property - 'status' => isset($this->events[0]) ? $this->events[0]->getStatus() : null, - 'fixInfo' => $analysisResult[$relativePath] ?? null, - 'errors' => $this->errorsManager->forPath($absolutePath), - ]); - } - - // Request another file chunk (if available, the parallelisation operator will request new "run" action) - $out->write(['action' => ParallelAction::RUNNER_GET_FILE_CHUNK]); - }); - }) + ->then( + function (ConnectionInterface $connection) use ($runner, $identifier): void { + $jsonInvalidUtf8Ignore = \defined('JSON_INVALID_UTF8_IGNORE') ? JSON_INVALID_UTF8_IGNORE : 0; + $out = new Encoder($connection, $jsonInvalidUtf8Ignore); + $in = new Decoder($connection, true, 512, $jsonInvalidUtf8Ignore); + + // [REACT] Initialise connection with the parallelisation operator + $out->write(['action' => ParallelAction::RUNNER_HELLO, 'identifier' => $identifier]); + + $handleError = static function (\Throwable $error): void { + // @TODO Handle communication errors + }; + $out->on('error', $handleError); + $in->on('error', $handleError); + + // [REACT] Listen for messages from the parallelisation operator (analysis requests) + $in->on('data', function (array $json) use ($runner, $out): void { + if (ParallelAction::WORKER_RUN !== $json['action']) { + return; + } + + /** @var iterable $files */ + $files = $json['files']; + + foreach ($files as $i => $absolutePath) { + $relativePath = $this->configurationResolver->getDirectory()->getRelativePathTo($absolutePath); + + // Reset events because we want to collect only those coming from analysed files chunk + $this->events = []; + $runner->setFileIterator(new \ArrayIterator([new \SplFileInfo($absolutePath)])); + $analysisResult = $runner->fix(); + + $out->write([ + 'action' => ParallelAction::RUNNER_RESULT, + 'file' => $absolutePath, + // @phpstan-ignore-next-line False-positive caused by assigning empty array to $events property + 'status' => isset($this->events[0]) ? $this->events[0]->getStatus() : null, + 'fixInfo' => $analysisResult[$relativePath] ?? null, + 'errors' => $this->errorsManager->forPath($absolutePath), + ]); + } + + // Request another file chunk (if available, the parallelisation operator will request new "run" action) + $out->write(['action' => ParallelAction::RUNNER_GET_FILE_CHUNK]); + }); + }, + static function (\Throwable $error) use ($errorOutput): void { + $errorOutput->writeln($error->getMessage()); + } + ) ; $loop->run(); diff --git a/tests/Console/Command/WorkerCommandTest.php b/tests/Console/Command/WorkerCommandTest.php new file mode 100644 index 00000000000..579f5dc2f96 --- /dev/null +++ b/tests/Console/Command/WorkerCommandTest.php @@ -0,0 +1,90 @@ + + * Dariusz Rumiński + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace PhpCsFixer\Tests\Console\Command; + +use PhpCsFixer\Console\Application; +use PhpCsFixer\Console\Command\WorkerCommand; +use PhpCsFixer\Runner\Parallel\ProcessIdentifier; +use PhpCsFixer\Tests\TestCase; +use PhpCsFixer\ToolInfo; +use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Output\OutputInterface; +use Symfony\Component\Console\Tester\CommandTester; + +/** + * @author Greg Korba + * + * @internal + * + * @covers \PhpCsFixer\Console\Command\WorkerCommand + */ +final class WorkerCommandTest extends TestCase +{ + public function testMissingIdentifierCausesFailure(): void + { + $commandTester = $this->doTestExecute(['--port' => 12_345]); + + self::assertSame(Command::FAILURE, $commandTester->getStatusCode()); + self::assertStringContainsString('Missing parallelisation options', $commandTester->getErrorOutput()); + } + + public function testMissingCausesFailure(): void + { + $commandTester = $this->doTestExecute(['--identifier' => (string) ProcessIdentifier::create()]); + + self::assertSame(Command::FAILURE, $commandTester->getStatusCode()); + self::assertStringContainsString('Missing parallelisation options', $commandTester->getErrorOutput()); + } + + public function testWorkerCantConnectToServerWhenExecutedDirectly(): void + { + $commandTester = $this->doTestExecute([ + '--identifier' => (string) ProcessIdentifier::create(), + '--port' => 12_345, + ]); + + self::assertStringContainsString( + 'Connection refused', + $commandTester->getErrorOutput() + ); + } + + /** + * @param array $arguments + */ + private function doTestExecute(array $arguments): CommandTester + { + $application = new Application(); + $application->add(new WorkerCommand(new ToolInfo())); + + $command = $application->find('worker'); + $commandTester = new CommandTester($command); + + $commandTester->execute( + array_merge( + ['command' => $command->getName()], + $arguments + ), + [ + 'capture_stderr_separately' => true, + 'interactive' => false, + 'decorated' => false, + 'verbosity' => OutputInterface::VERBOSITY_DEBUG, + ] + ); + + return $commandTester; + } +} diff --git a/tests/Runner/Parallel/ParallelConfigTest.php b/tests/Runner/Parallel/ParallelConfigTest.php index 34c121be669..1112ca34973 100644 --- a/tests/Runner/Parallel/ParallelConfigTest.php +++ b/tests/Runner/Parallel/ParallelConfigTest.php @@ -66,4 +66,12 @@ public function testSequentialConfigHasExactlyOneProcess(): void self::assertSame(1, $config->getMaxProcesses()); } + + public function testDetectConfiguration(): void + { + $config = ParallelConfig::detect(1, 100); + + self::assertSame(1, $config->getFilesPerProcess()); + self::assertSame(100, $config->getProcessTimeout()); + } } diff --git a/tests/Runner/Parallel/ProcessIdentifierTest.php b/tests/Runner/Parallel/ProcessIdentifierTest.php index 5e74a97f60b..c3d6450dbad 100644 --- a/tests/Runner/Parallel/ProcessIdentifierTest.php +++ b/tests/Runner/Parallel/ProcessIdentifierTest.php @@ -25,6 +25,13 @@ */ final class ProcessIdentifierTest extends TestCase { + public function testCreateIdentifier(): void + { + $identifier = ProcessIdentifier::create(); + + self::assertStringStartsWith('php-cs-fixer_parallel_', (string) $identifier); + } + /** * @dataProvider provideFromRawCases */ From 41cb3e029a420e9a561c31e47e73aa1dbc535829 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Thu, 15 Feb 2024 09:22:37 +0100 Subject: [PATCH 37/77] Fix smoke tests --- tests/Smoke/PharTest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Smoke/PharTest.php b/tests/Smoke/PharTest.php index 3fa43c07693..ec5b947e535 100644 --- a/tests/Smoke/PharTest.php +++ b/tests/Smoke/PharTest.php @@ -94,7 +94,7 @@ public function testFix(): void { self::assertSame( 0, - self::executePharCommand('fix src/Config.php -vvv --dry-run --diff --using-cache=no 2>&1')->getCode() + self::executePharCommand('fix src/Config.php -vvv --dry-run --sequential --diff --using-cache=no 2>&1')->getCode() ); } @@ -120,7 +120,7 @@ public function testReport(string $usingCache): void { try { $json = self::executePharCommand(sprintf( - 'fix %s --dry-run --format=json --rules=\'%s\' --using-cache=%s', + 'fix %s --dry-run --sequential --format=json --rules=\'%s\' --using-cache=%s', __FILE__, json_encode(['concat_space' => ['spacing' => 'one']], JSON_THROW_ON_ERROR), $usingCache, From 3c9af5752ac059951a396f28e6cf59c0adbfbf7f Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Thu, 15 Feb 2024 09:58:21 +0100 Subject: [PATCH 38/77] Fix parallelisation for PHAR usage --- src/Runner/Parallel/ProcessFactory.php | 14 +++++++++++++- tests/Smoke/PharTest.php | 13 ++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/Runner/Parallel/ProcessFactory.php b/src/Runner/Parallel/ProcessFactory.php index daecf691c3d..2b02ac69d06 100644 --- a/src/Runner/Parallel/ProcessFactory.php +++ b/src/Runner/Parallel/ProcessFactory.php @@ -45,9 +45,21 @@ public function create( throw new ParallelisationException('Cannot find PHP executable.'); } + $mainScript = realpath(__DIR__.'/../../../php-cs-fixer'); + if (false === $mainScript + && isset($_SERVER['argv'][0]) + && str_contains($_SERVER['argv'][0], 'php-cs-fixer') + ) { + $mainScript = $_SERVER['argv'][0]; + } + + if (!is_file($mainScript)) { + throw new ParallelisationException('Cannot determine Fixer executable.'); + } + $commandArgs = [ $phpBinary, - escapeshellarg(realpath(__DIR__.'/../../../php-cs-fixer')), + escapeshellarg($mainScript), 'worker', '--port', (string) $serverPort, diff --git a/tests/Smoke/PharTest.php b/tests/Smoke/PharTest.php index ec5b947e535..df414fe8b84 100644 --- a/tests/Smoke/PharTest.php +++ b/tests/Smoke/PharTest.php @@ -90,7 +90,7 @@ public function testDescribe(): void ); } - public function testFix(): void + public function testFixSequential(): void { self::assertSame( 0, @@ -98,6 +98,17 @@ public function testFix(): void ); } + public function testFixParallel(): void + { + $command = self::executePharCommand('fix src/Config.php -vvv --dry-run --diff --using-cache=no 2>&1'); + + self::assertSame(0, $command->getCode()); + self::assertMatchesRegularExpression( + '/Running analysis on [0-9]+ cores with [0-9]+ files per process/', + $command->getOutput() + ); + } + public function testFixHelp(): void { self::assertSame( From b06ad0c1fba6f1491c93ca924489bbd8a31d6cef Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Thu, 15 Feb 2024 17:09:27 +0100 Subject: [PATCH 39/77] Add missing phpDoc for callable supported in `ProcessPool` constructor --- src/Runner/Parallel/ProcessPool.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Runner/Parallel/ProcessPool.php b/src/Runner/Parallel/ProcessPool.php index ec0734e0d30..d48f9edcd18 100644 --- a/src/Runner/Parallel/ProcessPool.php +++ b/src/Runner/Parallel/ProcessPool.php @@ -34,6 +34,9 @@ final class ProcessPool /** @var array */ private array $processes = []; + /** + * @param null|(callable(): void) $onServerClose + */ public function __construct(ServerInterface $server, ?callable $onServerClose = null) { $this->server = $server; From a30503853ba1a82abca8ad282bf418bd24157f37 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Thu, 15 Feb 2024 17:10:18 +0100 Subject: [PATCH 40/77] Add missing tests --- tests/ConfigTest.php | 18 +++++++++++ tests/Error/ErrorTest.php | 27 ++++++++++++++++ tests/Error/ErrorsManagerTest.php | 20 ++++++++++++ tests/Runner/RunnerTest.php | 53 ++++++++++++++++++++++++++++++- 4 files changed, 117 insertions(+), 1 deletion(-) diff --git a/tests/ConfigTest.php b/tests/ConfigTest.php index 82b6d1f6588..54ebf0de4b6 100644 --- a/tests/ConfigTest.php +++ b/tests/ConfigTest.php @@ -23,6 +23,7 @@ use PhpCsFixer\Fixer\ArrayNotation\NoWhitespaceBeforeCommaInArrayFixer; use PhpCsFixer\Fixer\ControlStructure\IncludeFixer; use PhpCsFixer\Fixer\FixerInterface; +use PhpCsFixer\Runner\Parallel\ParallelConfig; use PhpCsFixer\ToolInfo; use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Console\Tester\CommandTester; @@ -268,4 +269,21 @@ public function testConfigConstructorWithName(): void self::assertSame($anonymousConfig->getName(), 'default'); self::assertSame($namedConfig->getName(), 'foo'); } + + public function testConfigWithDefaultParallelConfig(): void + { + $config = new Config(); + + self::assertSame(1, $config->getParallelConfig()->getMaxProcesses()); + } + + public function testConfigWithExplicitParallelConfig(): void + { + $config = new Config(); + $config->setParallelConfig(new ParallelConfig(5, 10, 15)); + + self::assertSame(5, $config->getParallelConfig()->getMaxProcesses()); + self::assertSame(10, $config->getParallelConfig()->getFilesPerProcess()); + self::assertSame(15, $config->getParallelConfig()->getProcessTimeout()); + } } diff --git a/tests/Error/ErrorTest.php b/tests/Error/ErrorTest.php index 351c8c2af82..1bddb55c953 100644 --- a/tests/Error/ErrorTest.php +++ b/tests/Error/ErrorTest.php @@ -63,4 +63,31 @@ public function testConstructorSetsValues2(): void self::assertSame($appliedFixers, $error->getAppliedFixers()); self::assertSame($diff, $error->getDiff()); } + + public function testErrorCanBeSerialised(): void + { + $type = 2; + $filePath = __FILE__; + $source = new \Exception(); + $appliedFixers = ['some_rule']; + $diff = '__diff__'; + + $error = new Error( + $type, + $filePath, + $source, + $appliedFixers, + $diff + ); + $serialisedError = $error->jsonSerialize(); + + self::assertSame($type, $serialisedError['type']); + self::assertSame($filePath, $serialisedError['filePath']); + self::assertSame($source->getMessage(), $serialisedError['source']['message']); + self::assertSame($source->getLine(), $serialisedError['source']['line']); + self::assertSame($source->getFile(), $serialisedError['source']['file']); + self::assertSame($source->getCode(), $serialisedError['source']['code']); + self::assertSame($appliedFixers, $serialisedError['appliedFixers']); + self::assertSame($diff, $serialisedError['diff']); + } } diff --git a/tests/Error/ErrorsManagerTest.php b/tests/Error/ErrorsManagerTest.php index 872b023cc4f..70ee5657154 100644 --- a/tests/Error/ErrorsManagerTest.php +++ b/tests/Error/ErrorsManagerTest.php @@ -100,4 +100,24 @@ public function testThatCanReportAndRetrieveInvalidFileErrors(): void self::assertCount(0, $errorsManager->getInvalidErrors()); self::assertCount(0, $errorsManager->getExceptionErrors()); } + + public function testThatCanReportAndRetrieveErrorsForSpecificPath(): void + { + $errorsManager = new ErrorsManager(); + + // All kind of errors for the same path + $errorsManager->report(new Error(Error::TYPE_LINT, 'foo.php')); + $errorsManager->report(new Error(Error::TYPE_EXCEPTION, 'foo.php')); + $errorsManager->report(new Error(Error::TYPE_INVALID, 'foo.php')); + + // Additional errors for a different path + $errorsManager->report(new Error(Error::TYPE_INVALID, 'bar.php')); + $errorsManager->report(new Error(Error::TYPE_LINT, 'baz.php')); + + self::assertFalse($errorsManager->isEmpty()); + + $errors = $errorsManager->forPath('foo.php'); + + self::assertCount(3, $errors); + } } diff --git a/tests/Runner/RunnerTest.php b/tests/Runner/RunnerTest.php index 4181237bb98..e994ddb7023 100644 --- a/tests/Runner/RunnerTest.php +++ b/tests/Runner/RunnerTest.php @@ -17,6 +17,7 @@ use PhpCsFixer\AccessibleObject\AccessibleObject; use PhpCsFixer\Cache\Directory; use PhpCsFixer\Cache\NullCacheManager; +use PhpCsFixer\Console\Command\FixCommand; use PhpCsFixer\Differ\DifferInterface; use PhpCsFixer\Differ\NullDiffer; use PhpCsFixer\Error\Error; @@ -25,8 +26,12 @@ use PhpCsFixer\Linter\Linter; use PhpCsFixer\Linter\LinterInterface; use PhpCsFixer\Linter\LintingResultInterface; +use PhpCsFixer\Runner\Parallel\ParallelConfig; +use PhpCsFixer\Runner\Parallel\ParallelisationException; use PhpCsFixer\Runner\Runner; use PhpCsFixer\Tests\TestCase; +use PhpCsFixer\ToolInfo; +use Symfony\Component\Console\Input\ArrayInput; use Symfony\Component\Finder\Finder; /** @@ -97,8 +102,9 @@ public function testThatFixSuccessfully(): void /** * @covers \PhpCsFixer\Runner\Runner::fix * @covers \PhpCsFixer\Runner\Runner::fixFile + * @covers \PhpCsFixer\Runner\Runner::fixSequential */ - public function testThatFixInvalidFileReportsToErrorManager(): void + public function testThatSequentialFixOfInvalidFileReportsToErrorManager(): void { $errorsManager = new ErrorsManager(); @@ -131,6 +137,51 @@ public function testThatFixInvalidFileReportsToErrorManager(): void self::assertSame($pathToInvalidFile, $error->getFilePath()); } + /** + * @covers \PhpCsFixer\Runner\Runner::fix + * @covers \PhpCsFixer\Runner\Runner::fixFile + * @covers \PhpCsFixer\Runner\Runner::fixParallel + */ + public function testThatParallelFixOfInvalidFileReportsToErrorManager(): void + { + $errorsManager = new ErrorsManager(); + + $path = realpath(__DIR__.\DIRECTORY_SEPARATOR.'..').\DIRECTORY_SEPARATOR.'Fixtures'.\DIRECTORY_SEPARATOR.'FixerTest'.\DIRECTORY_SEPARATOR.'invalid'; + $runner = new Runner( + Finder::create()->in($path), + [ + new Fixer\ClassNotation\VisibilityRequiredFixer(), + new Fixer\Import\NoUnusedImportsFixer(), // will be ignored cause of test keyword in namespace + ], + new NullDiffer(), + null, + $errorsManager, + new Linter(), + true, + new NullCacheManager(), + null, + false, + new ParallelConfig(2, 1, 50), + new ArrayInput([], (new FixCommand(new ToolInfo()))->getDefinition()) + ); + $changed = $runner->fix(); + $pathToInvalidFile = $path.\DIRECTORY_SEPARATOR.'somefile.php'; + + self::assertCount(0, $changed); + + $errors = $errorsManager->getInvalidErrors(); + + self::assertCount(1, $errors); + + $error = $errors[0]; + + self::assertInstanceOf(Error::class, $error); + self::assertInstanceOf(ParallelisationException::class, $error->getSource()); + + self::assertSame(Error::TYPE_INVALID, $error->getType()); + self::assertSame($pathToInvalidFile, $error->getFilePath()); + } + /** * @covers \PhpCsFixer\Runner\Runner::fix * @covers \PhpCsFixer\Runner\Runner::fixFile From 53bd0057460b3c296bb2966702bdf45569d4707b Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Fri, 16 Feb 2024 09:58:19 +0100 Subject: [PATCH 41/77] Backward compatibility for `cs:fix:parallel` script --- composer.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/composer.json b/composer.json index 877ade73174..e0c8b6d6e86 100644 --- a/composer.json +++ b/composer.json @@ -94,6 +94,10 @@ ], "cs:check": "@php php-cs-fixer check --verbose --diff", "cs:fix": "@php php-cs-fixer fix", + "cs:fix:parallel": [ + "echo '⚠️ This script is deprecated! Utilise built-in parallelisation instead.';", + "@cs:fix" + ], "docs": "@php dev-tools/doc.php", "infection": "@test:mutation", "install-tools": "@composer --working-dir=dev-tools install", @@ -162,6 +166,7 @@ "auto-review": "Execute Auto-review", "cs:check": "Check coding standards", "cs:fix": "Fix coding standards", + "cs:fix:parallel": "⚠️DEPRECATED! Use cs:fix with proper parallel config", "docs": "Regenerate docs", "infection": "Alias for 'test:mutation'", "install-tools": "Install DEV tools", From f0918dddddec8fbe2341170f9573dcb13f7f910e Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Fri, 16 Feb 2024 16:49:36 +0100 Subject: [PATCH 42/77] Test that callback is executed on server close --- tests/Runner/Parallel/ProcessPoolTest.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/Runner/Parallel/ProcessPoolTest.php b/tests/Runner/Parallel/ProcessPoolTest.php index c3f80b687c1..cab6bccd13d 100644 --- a/tests/Runner/Parallel/ProcessPoolTest.php +++ b/tests/Runner/Parallel/ProcessPoolTest.php @@ -98,7 +98,10 @@ public function testEndProcessIfKnownWithKnownIdentifier(): void public function testEndAll(): void { - $processPool = $this->getProcessPool(); + $callbackExecuted = false; + $processPool = $this->getProcessPool(static function () use (&$callbackExecuted): void { + $callbackExecuted = true; + }); $identifier1 = ProcessIdentifier::create(); $process1 = $this->createProcess($identifier1); @@ -111,6 +114,7 @@ public function testEndAll(): void $processPool->endAll(); self::assertTrue($this->serverClosed); + self::assertTrue($callbackExecuted, 'Callback was not executed on server close.'); } private function createProcess(ProcessIdentifier $identifier): Process From f8798a59f33eafef707a115c98bb681e80f91b5f Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Fri, 16 Feb 2024 16:59:15 +0100 Subject: [PATCH 43/77] Test server-worker communication --- src/Console/Command/WorkerCommand.php | 18 +++- src/Runner/Parallel/ParallelAction.php | 1 + src/Runner/Parallel/ProcessFactory.php | 20 +++-- src/Runner/Runner.php | 2 + tests/Console/Command/WorkerCommandTest.php | 91 +++++++++++++++++++++ 5 files changed, 123 insertions(+), 9 deletions(-) diff --git a/src/Console/Command/WorkerCommand.php b/src/Console/Command/WorkerCommand.php index 359b4cdbfbe..2043ff78622 100644 --- a/src/Console/Command/WorkerCommand.php +++ b/src/Console/Command/WorkerCommand.php @@ -119,7 +119,6 @@ protected function configure(): void protected function execute(InputInterface $input, OutputInterface $output): int { - $verbosity = $output->getVerbosity(); $errorOutput = $output instanceof ConsoleOutputInterface ? $output->getErrorOutput() : $output; $identifier = $input->getOption('identifier'); $port = $input->getOption('port'); @@ -143,7 +142,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int $tcpConnector ->connect(sprintf('127.0.0.1:%d', $port)) ->then( - function (ConnectionInterface $connection) use ($runner, $identifier): void { + /** @codeCoverageIgnore */ + function (ConnectionInterface $connection) use ($loop, $runner, $identifier): void { $jsonInvalidUtf8Ignore = \defined('JSON_INVALID_UTF8_IGNORE') ? JSON_INVALID_UTF8_IGNORE : 0; $out = new Encoder($connection, $jsonInvalidUtf8Ignore); $in = new Decoder($connection, true, 512, $jsonInvalidUtf8Ignore); @@ -158,8 +158,18 @@ function (ConnectionInterface $connection) use ($runner, $identifier): void { $in->on('error', $handleError); // [REACT] Listen for messages from the parallelisation operator (analysis requests) - $in->on('data', function (array $json) use ($runner, $out): void { - if (ParallelAction::WORKER_RUN !== $json['action']) { + $in->on('data', function (array $json) use ($loop, $runner, $out): void { + $action = $json['action'] ?? null; + + // Parallelisation operator does not have more to do, let's close the connection + if (ParallelAction::WORKER_THANK_YOU === $action) { + $loop->stop(); + + return; + } + + // At this point we only expect analysis requests, so let's return early for any other message + if (ParallelAction::WORKER_RUN !== $action) { return; } diff --git a/src/Runner/Parallel/ParallelAction.php b/src/Runner/Parallel/ParallelAction.php index cdc2e5c843a..e516afe729e 100644 --- a/src/Runner/Parallel/ParallelAction.php +++ b/src/Runner/Parallel/ParallelAction.php @@ -28,4 +28,5 @@ final class ParallelAction // Actions handled by the worker public const WORKER_RUN = 'run'; + public const WORKER_THANK_YOU = 'thankYou'; } diff --git a/src/Runner/Parallel/ProcessFactory.php b/src/Runner/Parallel/ProcessFactory.php index 2b02ac69d06..a50f9680bcd 100644 --- a/src/Runner/Parallel/ProcessFactory.php +++ b/src/Runner/Parallel/ProcessFactory.php @@ -39,6 +39,20 @@ public function create( ProcessIdentifier $identifier, int $serverPort ): Process { + $commandArgs = $this->getCommandArgs($serverPort, $identifier, $runnerConfig); + + return new Process( + implode(' ', $commandArgs), + $loop, + $runnerConfig->getParallelConfig()->getProcessTimeout() + ); + } + + /** + * @return list + */ + public function getCommandArgs(int $serverPort, ProcessIdentifier $identifier, RunnerConfig $runnerConfig): array + { $phpBinary = (new PhpExecutableFinder())->find(false); if (false === $phpBinary) { @@ -84,10 +98,6 @@ public function create( } } - return new Process( - implode(' ', $commandArgs), - $loop, - $runnerConfig->getParallelConfig()->getProcessTimeout() - ); + return $commandArgs; } } diff --git a/src/Runner/Runner.php b/src/Runner/Runner.php index b9bc713bc1a..927869af748 100644 --- a/src/Runner/Runner.php +++ b/src/Runner/Runner.php @@ -198,6 +198,7 @@ private function fixParallel(): array $job = $fileChunk(); if (0 === \count($job)) { + $process->request(['action' => ParallelAction::WORKER_THANK_YOU]); $processPool->endProcessIfKnown($identifier); return; @@ -275,6 +276,7 @@ function (array $workerResponse) use ($processPool, $process, $identifier, $file $job = $fileChunk(); if (0 === \count($job)) { + $process->request(['action' => ParallelAction::WORKER_THANK_YOU]); $processPool->endProcessIfKnown($identifier); return; diff --git a/tests/Console/Command/WorkerCommandTest.php b/tests/Console/Command/WorkerCommandTest.php index 579f5dc2f96..42bf33757e8 100644 --- a/tests/Console/Command/WorkerCommandTest.php +++ b/tests/Console/Command/WorkerCommandTest.php @@ -14,12 +14,25 @@ namespace PhpCsFixer\Tests\Console\Command; +use Clue\React\NDJson\Decoder; +use Clue\React\NDJson\Encoder; use PhpCsFixer\Console\Application; +use PhpCsFixer\Console\Command\FixCommand; use PhpCsFixer\Console\Command\WorkerCommand; +use PhpCsFixer\FixerFileProcessedEvent; +use PhpCsFixer\Runner\Parallel\ParallelAction; +use PhpCsFixer\Runner\Parallel\ParallelConfig; +use PhpCsFixer\Runner\Parallel\ProcessFactory; use PhpCsFixer\Runner\Parallel\ProcessIdentifier; +use PhpCsFixer\Runner\RunnerConfig; use PhpCsFixer\Tests\TestCase; use PhpCsFixer\ToolInfo; +use React\ChildProcess\Process; +use React\EventLoop\StreamSelectLoop; +use React\Socket\ConnectionInterface; +use React\Socket\TcpServer; use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Input\ArrayInput; use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Console\Tester\CommandTester; @@ -61,6 +74,84 @@ public function testWorkerCantConnectToServerWhenExecutedDirectly(): void ); } + public function testWorkerCommunicatesWithTheServer(): void + { + $streamSelectLoop = new StreamSelectLoop(); + $server = new TcpServer('127.0.0.1:0', $streamSelectLoop); + $serverPort = parse_url($server->getAddress() ?? '', PHP_URL_PORT); + $processIdentifier = ProcessIdentifier::create(); + $processFactory = new ProcessFactory( + new ArrayInput([], (new FixCommand(new ToolInfo()))->getDefinition()) + ); + $process = new Process(implode(' ', $processFactory->getCommandArgs( + $serverPort, + $processIdentifier, + new RunnerConfig(true, false, ParallelConfig::sequential()) + ))); + + /** + * @var array{ + * identifier: string, + * messages: list>, + * connected: bool, + * chunkRequested: bool, + * resultReported: bool + * } $workerScope + */ + $workerScope = [ + 'identifier' => (string) $processIdentifier, + 'messages' => [], + 'connected' => false, + 'chunkRequested' => false, + 'resultReported' => false, + ]; + + $server->on( + 'connection', + static function (ConnectionInterface $connection) use (&$workerScope): void { + $jsonInvalidUtf8Ignore = \defined('JSON_INVALID_UTF8_IGNORE') ? JSON_INVALID_UTF8_IGNORE : 0; + $decoder = new Decoder($connection, true, 512, $jsonInvalidUtf8Ignore); + $encoder = new Encoder($connection, $jsonInvalidUtf8Ignore); + + $decoder->on( + 'data', + static function (array $data) use ($encoder, &$workerScope): void { + $workerScope['messages'][] = $data; + $ds = \DIRECTORY_SEPARATOR; + + if (ParallelAction::RUNNER_HELLO === $data['action']) { + $encoder->write(['action' => ParallelAction::WORKER_RUN, 'files' => [ + realpath(__DIR__.$ds.'..'.$ds.'..').$ds.'Fixtures'.$ds.'FixerTest'.$ds.'fix'.$ds.'somefile.php', + ]]); + + return; + } + + if (3 === \count($workerScope['messages'])) { + $encoder->write(['action' => ParallelAction::WORKER_THANK_YOU]); + } + } + ); + } + ); + $process->on('exit', static function () use ($streamSelectLoop): void { + $streamSelectLoop->stop(); + }); + + // Start worker in the async process, handle communication with server and wait for it to exit + $process->start($streamSelectLoop); + $streamSelectLoop->run(); + + self::assertSame(Command::SUCCESS, $process->getExitCode()); + self::assertCount(3, $workerScope['messages']); + self::assertSame(ParallelAction::RUNNER_HELLO, $workerScope['messages'][0]['action']); + self::assertSame(ParallelAction::RUNNER_RESULT, $workerScope['messages'][1]['action']); + self::assertSame(FixerFileProcessedEvent::STATUS_FIXED, $workerScope['messages'][1]['status']); + self::assertSame(ParallelAction::RUNNER_GET_FILE_CHUNK, $workerScope['messages'][2]['action']); + + $server->close(); + } + /** * @param array $arguments */ From def627544cd0266d72fc51199f1dcf61741ade23 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Fri, 1 Mar 2024 02:23:00 +0100 Subject: [PATCH 44/77] Add error handling https://twitter.com/_Codito_/status/1763374111571472558 Needs more tests! --- src/Console/Application.php | 35 +++++++++++ src/Console/Command/FixCommand.php | 8 ++- .../FixCommandExitStatusCalculator.php | 12 +++- src/Console/Command/WorkerCommand.php | 14 ++++- src/Console/Output/ErrorOutput.php | 37 +++++++++++ src/Error/ErrorsManager.php | 28 ++++++++- src/Error/WorkerError.php | 63 +++++++++++++++++++ src/Runner/Parallel/ParallelAction.php | 1 + src/Runner/Runner.php | 44 +++++++++++-- .../FixCommandExitStatusCalculatorTest.php | 48 +++++++++----- tests/Error/WorkerErrorTest.php | 47 ++++++++++++++ 11 files changed, 309 insertions(+), 28 deletions(-) create mode 100644 src/Error/WorkerError.php create mode 100644 tests/Error/WorkerErrorTest.php diff --git a/src/Console/Application.php b/src/Console/Application.php index b9c53e14d33..c0044436207 100644 --- a/src/Console/Application.php +++ b/src/Console/Application.php @@ -28,6 +28,7 @@ use PhpCsFixer\ToolInfo; use PhpCsFixer\Utils; use Symfony\Component\Console\Application as BaseApplication; +use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Command\ListCommand; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\ConsoleOutputInterface; @@ -46,6 +47,7 @@ final class Application extends BaseApplication public const VERSION_CODENAME = '15 Keys Accelerate'; private ToolInfo $toolInfo; + private ?Command $executedCommand = null; public function __construct() { @@ -162,4 +164,37 @@ protected function getDefaultCommands(): array { return [new HelpCommand(), new ListCommand()]; } + + /** + * @throws \Throwable + */ + protected function doRunCommand(Command $command, InputInterface $input, OutputInterface $output): int + { + $this->executedCommand = $command; + + return parent::doRunCommand($command, $input, $output); + } + + protected function doRenderThrowable(\Throwable $e, OutputInterface $output): void + { + // Since parallel analysis utilises child processes, and they have their own output, + // we need to capture the output of the child process to determine it there was an exception. + // Default render format is not machine-friendly, so we need to override it for `worker` command, + // in order to be able to easily parse exception data for further displaying on main process' side. + if ($this->executedCommand instanceof WorkerCommand) { + $output->writeln(WorkerCommand::ERROR_PREFIX.json_encode( + [ + 'message' => $e->getMessage(), + 'file' => $e->getFile(), + 'line' => $e->getLine(), + 'code' => $e->getCode(), + 'trace' => $e->getTraceAsString(), + ] + )); + + return; + } + + parent::doRenderThrowable($e, $output); + } } diff --git a/src/Console/Command/FixCommand.php b/src/Console/Command/FixCommand.php index bf428fb2fff..587ee5bb441 100644 --- a/src/Console/Command/FixCommand.php +++ b/src/Console/Command/FixCommand.php @@ -348,6 +348,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int ? $output->write($reporter->generate($reportSummary)) : $output->write($reporter->generate($reportSummary), false, OutputInterface::OUTPUT_RAW); + $workerErrors = $this->errorsManager->getWorkerErrors(); $invalidErrors = $this->errorsManager->getInvalidErrors(); $exceptionErrors = $this->errorsManager->getExceptionErrors(); $lintErrors = $this->errorsManager->getLintErrors(); @@ -355,6 +356,10 @@ protected function execute(InputInterface $input, OutputInterface $output): int if (null !== $stdErr) { $errorOutput = new ErrorOutput($stdErr); + if (\count($workerErrors) > 0) { + $errorOutput->listWorkerErrors($workerErrors); + } + if (\count($invalidErrors) > 0) { $errorOutput->listErrors('linting before fixing', $invalidErrors); } @@ -375,7 +380,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int \count($changed) > 0, \count($invalidErrors) > 0, \count($exceptionErrors) > 0, - \count($lintErrors) > 0 + \count($lintErrors) > 0, + \count($workerErrors) > 0 ); } diff --git a/src/Console/Command/FixCommandExitStatusCalculator.php b/src/Console/Command/FixCommandExitStatusCalculator.php index 727dfff52ba..61831ac6e64 100644 --- a/src/Console/Command/FixCommandExitStatusCalculator.php +++ b/src/Console/Command/FixCommandExitStatusCalculator.php @@ -28,8 +28,14 @@ final class FixCommandExitStatusCalculator public const EXIT_STATUS_FLAG_HAS_INVALID_FIXER_CONFIG = 32; public const EXIT_STATUS_FLAG_EXCEPTION_IN_APP = 64; - public function calculate(bool $isDryRun, bool $hasChangedFiles, bool $hasInvalidErrors, bool $hasExceptionErrors, bool $hasLintErrorsAfterFixing): int - { + public function calculate( + bool $isDryRun, + bool $hasChangedFiles, + bool $hasInvalidErrors, + bool $hasExceptionErrors, + bool $hasLintErrorsAfterFixing, + bool $hasWorkerErrors + ): int { $exitStatus = 0; if ($isDryRun) { @@ -42,7 +48,7 @@ public function calculate(bool $isDryRun, bool $hasChangedFiles, bool $hasInvali } } - if ($hasExceptionErrors || $hasLintErrorsAfterFixing) { + if ($hasExceptionErrors || $hasLintErrorsAfterFixing || $hasWorkerErrors) { $exitStatus |= self::EXIT_STATUS_FLAG_EXCEPTION_IN_APP; } diff --git a/src/Console/Command/WorkerCommand.php b/src/Console/Command/WorkerCommand.php index 2043ff78622..28c794d1797 100644 --- a/src/Console/Command/WorkerCommand.php +++ b/src/Console/Command/WorkerCommand.php @@ -45,6 +45,9 @@ #[AsCommand(name: 'worker', description: 'Internal command for running fixers in parallel', hidden: true)] final class WorkerCommand extends Command { + /** @var string Prefix used before JSON-encoded error printed in the worker's process */ + public const ERROR_PREFIX = 'WORKER_ERROR::'; + /** @var string */ protected static $defaultName = 'worker'; @@ -151,8 +154,15 @@ function (ConnectionInterface $connection) use ($loop, $runner, $identifier): vo // [REACT] Initialise connection with the parallelisation operator $out->write(['action' => ParallelAction::RUNNER_HELLO, 'identifier' => $identifier]); - $handleError = static function (\Throwable $error): void { - // @TODO Handle communication errors + $handleError = static function (\Throwable $error) use ($out): void { + $out->write([ + 'action' => ParallelAction::RUNNER_ERROR_REPORT, + 'message' => $error->getMessage(), + 'file' => $error->getFile(), + 'line' => $error->getLine(), + 'code' => $error->getCode(), + 'trace' => $error->getTraceAsString(), + ]); }; $out->on('error', $handleError); $in->on('error', $handleError); diff --git a/src/Console/Output/ErrorOutput.php b/src/Console/Output/ErrorOutput.php index 79bb57b7320..168d6719705 100644 --- a/src/Console/Output/ErrorOutput.php +++ b/src/Console/Output/ErrorOutput.php @@ -16,6 +16,7 @@ use PhpCsFixer\Differ\DiffConsoleFormatter; use PhpCsFixer\Error\Error; +use PhpCsFixer\Error\WorkerError; use PhpCsFixer\Linter\LintingException; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Formatter\OutputFormatter; @@ -118,6 +119,42 @@ public function listErrors(string $process, array $errors): void } } + /** + * @param array $errors + */ + public function listWorkerErrors(array $errors): void + { + $this->output->writeln(['', 'Errors reported from workers (parallel analysis):']); + + $showDetails = $this->output->getVerbosity() >= OutputInterface::VERBOSITY_VERY_VERBOSE; + $showTrace = $this->output->getVerbosity() >= OutputInterface::VERBOSITY_DEBUG; + + foreach ($errors as $i => $error) { + $this->output->writeln(sprintf('%4d) %s', $i + 1, $error->getMessage())); + + if (!$showDetails) { + continue; + } + + $this->output->writeln(sprintf( + ' in %s on line %d', + $error->getFilePath(), + $error->getLine() + )); + + if ($showTrace) { + $this->output->writeln([ + ' Stack trace:', + ...array_map( + static fn (string $frame) => " {$frame}", + explode("\n", $error->getTrace()) + ), + '', + ]); + } + } + } + /** * @param array{ * function?: string, diff --git a/src/Error/ErrorsManager.php b/src/Error/ErrorsManager.php index e6333db80f1..2a42b3c7ff6 100644 --- a/src/Error/ErrorsManager.php +++ b/src/Error/ErrorsManager.php @@ -28,6 +28,21 @@ final class ErrorsManager */ private array $errors = []; + /** + * @var list + */ + private array $workerErrors = []; + + /** + * Returns worker errors reported during processing files in parallel. + * + * @return list + */ + public function getWorkerErrors(): array + { + return $this->workerErrors; + } + /** * Returns errors reported during linting before fixing. * @@ -73,11 +88,20 @@ public function forPath(string $path): array */ public function isEmpty(): bool { - return [] === $this->errors; + return [] === $this->errors && [] === $this->workerErrors; } - public function report(Error $error): void + /** + * @param Error|WorkerError $error + */ + public function report($error): void { + if ($error instanceof WorkerError) { + $this->workerErrors[] = $error; + + return; + } + $this->errors[] = $error; } } diff --git a/src/Error/WorkerError.php b/src/Error/WorkerError.php new file mode 100644 index 00000000000..e5d21c34add --- /dev/null +++ b/src/Error/WorkerError.php @@ -0,0 +1,63 @@ + + * Dariusz Rumiński + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace PhpCsFixer\Error; + +/** + * @author Greg Korba + * + * @internal + */ +final class WorkerError +{ + private string $message; + private string $filePath; + private int $line; + private int $code; + private string $trace; + + public function __construct(string $message, string $filePath, int $line, int $code, string $trace) + { + $this->message = $message; + $this->filePath = $filePath; + $this->line = $line; + $this->code = $code; + $this->trace = $trace; + } + + public function getMessage(): string + { + return $this->message; + } + + public function getFilePath(): string + { + return $this->filePath; + } + + public function getLine(): int + { + return $this->line; + } + + public function getCode(): int + { + return $this->code; + } + + public function getTrace(): string + { + return $this->trace; + } +} diff --git a/src/Runner/Parallel/ParallelAction.php b/src/Runner/Parallel/ParallelAction.php index e516afe729e..3b7703c185c 100644 --- a/src/Runner/Parallel/ParallelAction.php +++ b/src/Runner/Parallel/ParallelAction.php @@ -22,6 +22,7 @@ final class ParallelAction { // Actions handled by the runner + public const RUNNER_ERROR_REPORT = 'errorReport'; public const RUNNER_HELLO = 'hello'; public const RUNNER_RESULT = 'result'; public const RUNNER_GET_FILE_CHUNK = 'getFileChunk'; diff --git a/src/Runner/Runner.php b/src/Runner/Runner.php index 927869af748..da1d5358b9b 100644 --- a/src/Runner/Runner.php +++ b/src/Runner/Runner.php @@ -20,15 +20,18 @@ use PhpCsFixer\Cache\CacheManagerInterface; use PhpCsFixer\Cache\Directory; use PhpCsFixer\Cache\DirectoryInterface; +use PhpCsFixer\Console\Command\WorkerCommand; use PhpCsFixer\Differ\DifferInterface; use PhpCsFixer\Error\Error; use PhpCsFixer\Error\ErrorsManager; +use PhpCsFixer\Error\WorkerError; use PhpCsFixer\FileReader; use PhpCsFixer\Fixer\FixerInterface; use PhpCsFixer\FixerFileProcessedEvent; use PhpCsFixer\Linter\LinterInterface; use PhpCsFixer\Linter\LintingException; use PhpCsFixer\Linter\LintingResultInterface; +use PhpCsFixer\Preg; use PhpCsFixer\Runner\Parallel\ParallelAction; use PhpCsFixer\Runner\Parallel\ParallelConfig; use PhpCsFixer\Runner\Parallel\ParallelisationException; @@ -287,25 +290,58 @@ function (array $workerResponse) use ($processPool, $process, $identifier, $file return; } + if (ParallelAction::RUNNER_ERROR_REPORT === $workerResponse['action']) { + $this->errorsManager->report(new WorkerError( + $workerResponse['message'], + $workerResponse['file'], + (int) $workerResponse['line'], + (int) $workerResponse['code'], + $workerResponse['trace'] + )); + + return; + } + throw new ParallelisationException('Unsupported action: '.($workerResponse['action'] ?? 'n/a')); }, // [REACT] Handle errors encountered during worker's execution - static function (\Throwable $error) use ($processPool): void { - // @TODO Pass-back error to the main process so it can be displayed to the user + function (\Throwable $error) use ($processPool): void { + $this->errorsManager->report(new WorkerError( + $error->getMessage(), + $error->getFile(), + $error->getLine(), + $error->getCode(), + $error->getTraceAsString() + )); $processPool->endAll(); }, // [REACT] Handle worker's shutdown - static function ($exitCode, string $output) use ($processPool, $identifier): void { + function ($exitCode, string $output) use ($processPool, $identifier): void { $processPool->endProcessIfKnown($identifier); if (0 === $exitCode || null === $exitCode) { return; } - // @TODO Handle output string for non-zero exit codes + $errorsReported = Preg::matchAll( + sprintf('/^(?:%s)([^\n]+)+/m', WorkerCommand::ERROR_PREFIX), + $output, + $matches + ); + + if ($errorsReported > 0) { + $error = json_decode($matches[1][0], true); + $this->errorsManager->report(new WorkerError( + $error['message'], + $error['file'], + (int) $error['line'], + (int) $error['code'], + $error['trace'] + )); + } } ); } diff --git a/tests/Console/Command/FixCommandExitStatusCalculatorTest.php b/tests/Console/Command/FixCommandExitStatusCalculatorTest.php index 6b2e7aee873..60244b66f01 100644 --- a/tests/Console/Command/FixCommandExitStatusCalculatorTest.php +++ b/tests/Console/Command/FixCommandExitStatusCalculatorTest.php @@ -30,42 +30,58 @@ final class FixCommandExitStatusCalculatorTest extends TestCase /** * @dataProvider provideCalculateCases */ - public function testCalculate(int $expected, bool $isDryRun, bool $hasChangedFiles, bool $hasInvalidErrors, bool $hasExceptionErrors, bool $hasLintErrorsAfterFixing): void - { + public function testCalculate( + int $expected, + bool $isDryRun, + bool $hasChangedFiles, + bool $hasInvalidErrors, + bool $hasExceptionErrors, + bool $hasLintErrorsAfterFixing, + bool $hasWorkerErrors + ): void { $calculator = new FixCommandExitStatusCalculator(); self::assertSame( $expected, - $calculator->calculate($isDryRun, $hasChangedFiles, $hasInvalidErrors, $hasExceptionErrors, $hasLintErrorsAfterFixing) + $calculator->calculate( + $isDryRun, + $hasChangedFiles, + $hasInvalidErrors, + $hasExceptionErrors, + $hasLintErrorsAfterFixing, + $hasWorkerErrors + ) ); } public static function provideCalculateCases(): iterable { - yield [0, true, false, false, false, false]; + yield [0, true, false, false, false, false, false]; + + yield [0, false, false, false, false, false, false]; - yield [0, false, false, false, false, false]; + yield [8, true, true, false, false, false, false]; - yield [8, true, true, false, false, false]; + yield [0, false, true, false, false, false, false]; - yield [0, false, true, false, false, false]; + yield [4, true, false, true, false, false, false]; - yield [4, true, false, true, false, false]; + yield [0, false, false, true, false, false, false]; - yield [0, false, false, true, false, false]; + yield [12, true, true, true, false, false, false]; - yield [12, true, true, true, false, false]; + yield [0, false, true, true, false, false, false]; - yield [0, false, true, true, false, false]; + yield [76, true, true, true, true, false, false]; - yield [76, true, true, true, true, false]; + yield [64, false, false, false, false, true, false]; - yield [64, false, false, false, false, true]; + yield [64, false, false, false, true, false, false]; - yield [64, false, false, false, true, false]; + yield [64, false, false, false, true, true, false]; - yield [64, false, false, false, true, true]; + yield [64, false, false, false, false, false, true]; - yield [8 | 64, true, true, false, true, true]; + yield [8 | 64, true, true, false, true, true, false]; } } diff --git a/tests/Error/WorkerErrorTest.php b/tests/Error/WorkerErrorTest.php new file mode 100644 index 00000000000..bc54f01a7e3 --- /dev/null +++ b/tests/Error/WorkerErrorTest.php @@ -0,0 +1,47 @@ + + * Dariusz Rumiński + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace PhpCsFixer\Tests\Error; + +use PhpCsFixer\Error\WorkerError; +use PhpCsFixer\Tests\TestCase; + +/** + * @covers \PhpCsFixer\Error\WorkerError + * + * @internal + */ +final class WorkerErrorTest extends TestCase +{ + public function testConstructorDataCanBeAccessed(): void + { + $message = 'BOOM!'; + $filePath = '/path/to/file.php'; + $line = 10; + $code = 100; + $trace = <<<'TRACE' + #0 Foo + #1 Bar + #2 {main} + TRACE; + + $error = new WorkerError($message, $filePath, $line, $code, $trace); + + self::assertSame($message, $error->getMessage()); + self::assertSame($filePath, $error->getFilePath()); + self::assertSame($line, $error->getLine()); + self::assertSame($code, $error->getCode()); + self::assertSame($trace, $error->getTrace()); + } +} From cd8eb2f470ccc70586178777bf80c3739df47407 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Fri, 1 Mar 2024 02:55:40 +0100 Subject: [PATCH 45/77] Fix test file/class name Broken during rebase. --- ...gIteratorTest.php => FileCachingLintingFileIteratorTest.php} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename tests/Runner/{FileCachingLintingIteratorTest.php => FileCachingLintingFileIteratorTest.php} (98%) diff --git a/tests/Runner/FileCachingLintingIteratorTest.php b/tests/Runner/FileCachingLintingFileIteratorTest.php similarity index 98% rename from tests/Runner/FileCachingLintingIteratorTest.php rename to tests/Runner/FileCachingLintingFileIteratorTest.php index 15939b459d6..a29c43e117a 100644 --- a/tests/Runner/FileCachingLintingIteratorTest.php +++ b/tests/Runner/FileCachingLintingFileIteratorTest.php @@ -24,7 +24,7 @@ * * @covers \PhpCsFixer\Runner\FileCachingLintingFileIterator */ -final class FileCachingLintingIteratorTest extends TestCase +final class FileCachingLintingFileIteratorTest extends TestCase { public function testLintingEmpty(): void { From a18b4e0dbba1a12170d52dcf05fd5aba13def320 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Fri, 1 Mar 2024 02:56:16 +0100 Subject: [PATCH 46/77] Add simple test for `PhpCsFixer\Runner\Parallel\Process` class --- tests/Runner/Parallel/ProcessTest.php | 36 +++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 tests/Runner/Parallel/ProcessTest.php diff --git a/tests/Runner/Parallel/ProcessTest.php b/tests/Runner/Parallel/ProcessTest.php new file mode 100644 index 00000000000..a2a6bef8ec3 --- /dev/null +++ b/tests/Runner/Parallel/ProcessTest.php @@ -0,0 +1,36 @@ + + * Dariusz Rumiński + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace PhpCsFixer\Tests\Runner\Parallel; + +use PhpCsFixer\Runner\Parallel\ParallelisationException; +use PhpCsFixer\Runner\Parallel\Process; +use PhpCsFixer\Tests\TestCase; +use React\EventLoop\StreamSelectLoop; + +/** + * @internal + * + * @covers \PhpCsFixer\Runner\Parallel\Process + */ +final class ProcessTest extends TestCase +{ + public function testRequestCantBeInvokedBeforeStart(): void + { + self::expectException(ParallelisationException::class); + + $process = new Process('php -v', new StreamSelectLoop(), 123); + $process->request([]); + } +} From 2cec9a2f06cc8668ce4b4e8426edb8b8c32d1632 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Fri, 1 Mar 2024 03:00:42 +0100 Subject: [PATCH 47/77] Remove superfluous assertion --- tests/Runner/RunnerTest.php | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/Runner/RunnerTest.php b/tests/Runner/RunnerTest.php index e994ddb7023..ea6887bf5dd 100644 --- a/tests/Runner/RunnerTest.php +++ b/tests/Runner/RunnerTest.php @@ -175,7 +175,6 @@ public function testThatParallelFixOfInvalidFileReportsToErrorManager(): void $error = $errors[0]; - self::assertInstanceOf(Error::class, $error); self::assertInstanceOf(ParallelisationException::class, $error->getSource()); self::assertSame(Error::TYPE_INVALID, $error->getType()); From 21f4e2821e9510edc4ee668da4573c3f3f599304 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Fri, 1 Mar 2024 03:09:54 +0100 Subject: [PATCH 48/77] Run `WorkerCommandTest::testWorkerCommunicatesWithTheServer` only on unix-like OS --- tests/Console/Command/WorkerCommandTest.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/Console/Command/WorkerCommandTest.php b/tests/Console/Command/WorkerCommandTest.php index 42bf33757e8..d8eb5d88dc6 100644 --- a/tests/Console/Command/WorkerCommandTest.php +++ b/tests/Console/Command/WorkerCommandTest.php @@ -74,6 +74,9 @@ public function testWorkerCantConnectToServerWhenExecutedDirectly(): void ); } + /** + * @requires OS Linux|Darwin + */ public function testWorkerCommunicatesWithTheServer(): void { $streamSelectLoop = new StreamSelectLoop(); From 2053e7f646ceef7d157f7923437132f45924e215 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Fri, 1 Mar 2024 23:38:43 +0100 Subject: [PATCH 49/77] Add various tests related to error handling --- src/Console/Command/WorkerCommand.php | 9 +-- tests/Console/ApplicationTest.php | 18 ++++++ tests/Console/Command/WorkerCommandTest.php | 15 ++--- tests/Console/Output/ErrorOutputTest.php | 61 +++++++++++++++++++++ tests/Error/ErrorsManagerTest.php | 24 ++++++++ 5 files changed, 114 insertions(+), 13 deletions(-) diff --git a/src/Console/Command/WorkerCommand.php b/src/Console/Command/WorkerCommand.php index 28c794d1797..381e82f2c97 100644 --- a/src/Console/Command/WorkerCommand.php +++ b/src/Console/Command/WorkerCommand.php @@ -22,6 +22,7 @@ use PhpCsFixer\FixerFileProcessedEvent; use PhpCsFixer\Runner\Parallel\ParallelAction; use PhpCsFixer\Runner\Parallel\ParallelConfig; +use PhpCsFixer\Runner\Parallel\ParallelisationException; use PhpCsFixer\Runner\Parallel\ReadonlyCacheManager; use PhpCsFixer\Runner\Runner; use PhpCsFixer\ToolInfoInterface; @@ -127,17 +128,13 @@ protected function execute(InputInterface $input, OutputInterface $output): int $port = $input->getOption('port'); if (null === $identifier || !is_numeric($port)) { - $errorOutput->writeln('Missing parallelisation options'); - - return Command::FAILURE; + throw new ParallelisationException('Missing parallelisation options'); } try { $runner = $this->createRunner($input); } catch (\Throwable $e) { - $errorOutput->writeln($e->getMessage()); - - return Command::FAILURE; + throw new ParallelisationException('Unable to create runner: '.$e->getMessage(), 0, $e); } $loop = new StreamSelectLoop(); diff --git a/tests/Console/ApplicationTest.php b/tests/Console/ApplicationTest.php index 789a59df409..bf00a337cff 100644 --- a/tests/Console/ApplicationTest.php +++ b/tests/Console/ApplicationTest.php @@ -15,7 +15,10 @@ namespace PhpCsFixer\Tests\Console; use PhpCsFixer\Console\Application; +use PhpCsFixer\Console\Command\WorkerCommand; use PhpCsFixer\Tests\TestCase; +use PhpCsFixer\ToolInfo; +use Symfony\Component\Console\Tester\ApplicationTester; /** * @internal @@ -37,4 +40,19 @@ public function testGetMajorVersion(): void { self::assertSame(3, Application::getMajorVersion()); } + + public function testWorkerExceptionsAreRenderedInMachineFriendlyWay(): void + { + $app = new Application(); + $app->add(new WorkerCommand(new ToolInfo())); + $app->setAutoExit(false); // see: https://symfony.com/doc/current/console.html#testing-commands + + $appTester = new ApplicationTester($app); + $appTester->run(['worker']); + + self::assertStringContainsString( + WorkerCommand::ERROR_PREFIX.'{"message":"Missing parallelisation options"', + $appTester->getDisplay() + ); + } } diff --git a/tests/Console/Command/WorkerCommandTest.php b/tests/Console/Command/WorkerCommandTest.php index d8eb5d88dc6..288cae7c481 100644 --- a/tests/Console/Command/WorkerCommandTest.php +++ b/tests/Console/Command/WorkerCommandTest.php @@ -22,6 +22,7 @@ use PhpCsFixer\FixerFileProcessedEvent; use PhpCsFixer\Runner\Parallel\ParallelAction; use PhpCsFixer\Runner\Parallel\ParallelConfig; +use PhpCsFixer\Runner\Parallel\ParallelisationException; use PhpCsFixer\Runner\Parallel\ProcessFactory; use PhpCsFixer\Runner\Parallel\ProcessIdentifier; use PhpCsFixer\Runner\RunnerConfig; @@ -47,18 +48,18 @@ final class WorkerCommandTest extends TestCase { public function testMissingIdentifierCausesFailure(): void { - $commandTester = $this->doTestExecute(['--port' => 12_345]); + self::expectException(ParallelisationException::class); + self::expectExceptionMessage('Missing parallelisation options'); - self::assertSame(Command::FAILURE, $commandTester->getStatusCode()); - self::assertStringContainsString('Missing parallelisation options', $commandTester->getErrorOutput()); + $commandTester = $this->doTestExecute(['--port' => 12_345]); } - public function testMissingCausesFailure(): void + public function testMissingPortCausesFailure(): void { - $commandTester = $this->doTestExecute(['--identifier' => (string) ProcessIdentifier::create()]); + self::expectException(ParallelisationException::class); + self::expectExceptionMessage('Missing parallelisation options'); - self::assertSame(Command::FAILURE, $commandTester->getStatusCode()); - self::assertStringContainsString('Missing parallelisation options', $commandTester->getErrorOutput()); + $commandTester = $this->doTestExecute(['--identifier' => (string) ProcessIdentifier::create()]); } public function testWorkerCantConnectToServerWhenExecutedDirectly(): void diff --git a/tests/Console/Output/ErrorOutputTest.php b/tests/Console/Output/ErrorOutputTest.php index 82c71c4608d..bd9e396ab2a 100644 --- a/tests/Console/Output/ErrorOutputTest.php +++ b/tests/Console/Output/ErrorOutputTest.php @@ -16,6 +16,7 @@ use PhpCsFixer\Console\Output\ErrorOutput; use PhpCsFixer\Error\Error; +use PhpCsFixer\Error\WorkerError; use PhpCsFixer\Linter\LintingException; use PhpCsFixer\Tests\TestCase; use Symfony\Component\Console\Output\OutputInterface; @@ -28,6 +29,66 @@ */ final class ErrorOutputTest extends TestCase { + /** + * @param OutputInterface::VERBOSITY_* $verbosityLevel + * + * @dataProvider provideWorkerErrorOutputCases + */ + public function testWorkerErrorOutput(WorkerError $error, int $verbosityLevel): void + { + $output = $this->createStreamOutput($verbosityLevel); + $errorOutput = new ErrorOutput($output); + $errorOutput->listWorkerErrors([$error]); + + $displayed = $this->readFullStreamOutput($output); + + $startWith = sprintf( + ' +Errors reported from workers (parallel analysis): + 1) %s', + $error->getMessage() + ); + + if ($verbosityLevel >= OutputInterface::VERBOSITY_VERY_VERBOSE) { + $startWith .= sprintf( + ' + in %s on line %d', + $error->getFilePath(), + $error->getLine() + ); + } + + if ($verbosityLevel >= OutputInterface::VERBOSITY_DEBUG) { + $startWith .= sprintf( + ' + Stack trace: +%s', + implode("\n", array_map( + static fn (string $frame) => " {$frame}", + explode("\n", $error->getTrace()) + )) + ); + } + + self::assertStringStartsWith($startWith, $displayed); + } + + /** + * @return iterable + */ + public static function provideWorkerErrorOutputCases(): iterable + { + $error = new WorkerError('Boom!', 'foo.php', 123, 1, '#0 Foo\n#1 Bar\n#2 {main}'); + + yield [$error, OutputInterface::VERBOSITY_NORMAL]; + + yield [$error, OutputInterface::VERBOSITY_VERBOSE]; + + yield [$error, OutputInterface::VERBOSITY_VERY_VERBOSE]; + + yield [$error, OutputInterface::VERBOSITY_DEBUG]; + } + /** * @param OutputInterface::VERBOSITY_* $verbosityLevel * diff --git a/tests/Error/ErrorsManagerTest.php b/tests/Error/ErrorsManagerTest.php index 70ee5657154..f81cdc7e4a3 100644 --- a/tests/Error/ErrorsManagerTest.php +++ b/tests/Error/ErrorsManagerTest.php @@ -16,6 +16,7 @@ use PhpCsFixer\Error\Error; use PhpCsFixer\Error\ErrorsManager; +use PhpCsFixer\Error\WorkerError; use PhpCsFixer\Tests\TestCase; /** @@ -33,6 +34,7 @@ public function testDefaults(): void self::assertEmpty($errorsManager->getInvalidErrors()); self::assertEmpty($errorsManager->getExceptionErrors()); self::assertEmpty($errorsManager->getLintErrors()); + self::assertEmpty($errorsManager->getWorkerErrors()); } public function testThatCanReportAndRetrieveInvalidErrors(): void @@ -55,6 +57,7 @@ public function testThatCanReportAndRetrieveInvalidErrors(): void self::assertCount(0, $errorsManager->getExceptionErrors()); self::assertCount(0, $errorsManager->getLintErrors()); + self::assertCount(0, $errorsManager->getWorkerErrors()); } public function testThatCanReportAndRetrieveExceptionErrors(): void @@ -77,6 +80,7 @@ public function testThatCanReportAndRetrieveExceptionErrors(): void self::assertCount(0, $errorsManager->getInvalidErrors()); self::assertCount(0, $errorsManager->getLintErrors()); + self::assertCount(0, $errorsManager->getWorkerErrors()); } public function testThatCanReportAndRetrieveInvalidFileErrors(): void @@ -99,6 +103,26 @@ public function testThatCanReportAndRetrieveInvalidFileErrors(): void self::assertCount(0, $errorsManager->getInvalidErrors()); self::assertCount(0, $errorsManager->getExceptionErrors()); + self::assertCount(0, $errorsManager->getWorkerErrors()); + } + + public function testThatCanReportAndRetrieveWorkerErrors(): void + { + $error = new WorkerError('Boom!', 'foo.php', 123, 1, '#0 Foo\n#1 Bar'); + $errorsManager = new ErrorsManager(); + + $errorsManager->report($error); + + self::assertFalse($errorsManager->isEmpty()); + + $errors = $errorsManager->getWorkerErrors(); + + self::assertCount(1, $errors); + self::assertSame($error, array_shift($errors)); + + self::assertCount(0, $errorsManager->getInvalidErrors()); + self::assertCount(0, $errorsManager->getExceptionErrors()); + self::assertCount(0, $errorsManager->getLintErrors()); } public function testThatCanReportAndRetrieveErrorsForSpecificPath(): void From e9d5e73acec49cf3869f5c0482e2fb8b2d054821 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Thu, 7 Mar 2024 22:11:00 +0100 Subject: [PATCH 50/77] Test future mode in Config --- tests/ConfigTest.php | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/ConfigTest.php b/tests/ConfigTest.php index 54ebf0de4b6..403c8932a43 100644 --- a/tests/ConfigTest.php +++ b/tests/ConfigTest.php @@ -36,6 +36,18 @@ */ final class ConfigTest extends TestCase { + public function testFutureMode(): void + { + $futureMode = getenv('PHP_CS_FIXER_FUTURE_MODE'); + putenv('PHP_CS_FIXER_FUTURE_MODE=1'); + + $config = new Config('test'); + self::assertSame('test (future mode)', $config->getName()); + self::assertArrayHasKey('@PER-CS', $config->getRules()); + + putenv('PHP_CS_FIXER_FUTURE_MODE='.$futureMode); + } + public function testConfigRulesUsingSeparateMethod(): void { $config = new Config(); From d5f6f476abc256b61b01c11b350c1e7aad2bea21 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Wed, 20 Mar 2024 15:47:17 +0100 Subject: [PATCH 51/77] Skip linting on file iteration in main process of parallel runner The linting is done on the workers' side before fixers are applied, so we will collect any syntax errors anyway. --- src/Runner/Runner.php | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/Runner/Runner.php b/src/Runner/Runner.php index da1d5358b9b..bfc3f2fbb91 100644 --- a/src/Runner/Runner.php +++ b/src/Runner/Runner.php @@ -162,7 +162,7 @@ private function fixParallel(): array } $processPool = new ProcessPool($server); - $fileIterator = $this->getFileIterator(); + $fileIterator = $this->getFilteringFileIterator(); $fileIterator->rewind(); $fileChunk = function () use ($fileIterator): array { @@ -357,7 +357,7 @@ function ($exitCode, string $output) use ($processPool, $identifier): void { private function fixSequential(): array { $changed = []; - $collection = $this->getFileIterator(); + $collection = $this->getLintingFileIterator(); foreach ($collection as $file) { $fixInfo = $this->fixFile($file, $collection->currentLintingResult()); @@ -547,22 +547,27 @@ private function dispatchEvent(string $name, Event $event): void $this->eventDispatcher->dispatch($event, $name); } - private function getFileIterator(): LintingResultAwareFileIteratorInterface + private function getLintingFileIterator(): LintingResultAwareFileIteratorInterface { if (null === $this->fileIterator) { throw new \RuntimeException('File iterator is not configured. Pass paths during Runner initialisation or set them after with `setFileIterator()`.'); } - $fileFilterIterator = new FileFilterIterator( + $fileFilterIterator = $this->getFilteringFileIterator(); + + return $this->linter->isAsync() + ? new FileCachingLintingFileIterator($fileFilterIterator, $this->linter) + : new LintingFileIterator($fileFilterIterator, $this->linter); + } + + private function getFilteringFileIterator(): FileFilterIterator + { + return new FileFilterIterator( $this->fileIterator instanceof \IteratorAggregate ? $this->fileIterator->getIterator() : $this->fileIterator, $this->eventDispatcher, $this->cacheManager ); - - return $this->linter->isAsync() - ? new FileCachingLintingFileIterator($fileFilterIterator, $this->linter) - : new LintingFileIterator($fileFilterIterator, $this->linter); } } From 4d3bf565bfce07cef8a98990371a33d9b862eaef Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Wed, 20 Mar 2024 23:59:07 +0100 Subject: [PATCH 52/77] Add todos for 4.0 --- src/Runner/Runner.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Runner/Runner.php b/src/Runner/Runner.php index bfc3f2fbb91..a29026e3d8a 100644 --- a/src/Runner/Runner.php +++ b/src/Runner/Runner.php @@ -104,6 +104,7 @@ public function __construct( CacheManagerInterface $cacheManager, ?DirectoryInterface $directory = null, bool $stopOnViolation = false, + // @TODO Make these arguments required in 4.0 ?ParallelConfig $parallelConfig = null, ?InputInterface $input = null, ?string $configFile = null @@ -137,6 +138,7 @@ public function setFileIterator(iterable $fileIterator): void */ public function fix(): array { + // @TODO Remove condition for the input argument in 4.0, as it should be required in the constructor return $this->parallelConfig->getMaxProcesses() > 1 && null !== $this->input ? $this->fixParallel() : $this->fixSequential(); @@ -258,7 +260,7 @@ function (array $workerResponse) use ($processPool, $process, $identifier, $file // Worker requests for another file chunk when all files were processed foreach ($workerResponse['errors'] ?? [] as $workerError) { - $error = new Error( + $this->errorsManager->report(new Error( $workerError['type'], $workerError['filePath'], null !== $workerError['source'] @@ -266,9 +268,7 @@ function (array $workerResponse) use ($processPool, $process, $identifier, $file : null, $workerError['appliedFixers'], $workerError['diff'] - ); - - $this->errorsManager->report($error); + )); } return; From 34308f95696a90fa4ccfb17cd97fe94c3160ac0a Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Tue, 26 Mar 2024 00:43:24 +0100 Subject: [PATCH 53/77] Move check for file iterator presence to `getFilteringFileIterator()` --- src/Runner/Runner.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Runner/Runner.php b/src/Runner/Runner.php index a29026e3d8a..f1b16f44878 100644 --- a/src/Runner/Runner.php +++ b/src/Runner/Runner.php @@ -549,10 +549,6 @@ private function dispatchEvent(string $name, Event $event): void private function getLintingFileIterator(): LintingResultAwareFileIteratorInterface { - if (null === $this->fileIterator) { - throw new \RuntimeException('File iterator is not configured. Pass paths during Runner initialisation or set them after with `setFileIterator()`.'); - } - $fileFilterIterator = $this->getFilteringFileIterator(); return $this->linter->isAsync() @@ -562,6 +558,10 @@ private function getLintingFileIterator(): LintingResultAwareFileIteratorInterfa private function getFilteringFileIterator(): FileFilterIterator { + if (null === $this->fileIterator) { + throw new \RuntimeException('File iterator is not configured. Pass paths during Runner initialisation or set them after with `setFileIterator()`.'); + } + return new FileFilterIterator( $this->fileIterator instanceof \IteratorAggregate ? $this->fileIterator->getIterator() From a766504fd4a767c8dccd7c69aed245fa8bfe1620 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Tue, 26 Mar 2024 00:46:58 +0100 Subject: [PATCH 54/77] Fallback to default parallel config only if not defined explicitly during value read --- src/Config.php | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Config.php b/src/Config.php index beead176910..5f3fad32400 100644 --- a/src/Config.php +++ b/src/Config.php @@ -48,7 +48,7 @@ class Config implements ConfigInterface, ParallelRunnerConfigInterface private string $name; - private ParallelConfig $parallelRunnerConfig; + private ?ParallelConfig $parallelRunnerConfig; /** * @var null|string @@ -66,8 +66,6 @@ class Config implements ConfigInterface, ParallelRunnerConfigInterface public function __construct(string $name = 'default') { - $this->parallelRunnerConfig = ParallelConfig::sequential(); - // @TODO 4.0 cleanup if (Utils::isFutureModeEnabled()) { $this->name = $name.' (future mode)'; @@ -125,7 +123,7 @@ public function getName(): string public function getParallelConfig(): ParallelConfig { - return $this->parallelRunnerConfig; + return $this->parallelRunnerConfig ?? ParallelConfig::sequential(); } public function getPhpExecutable(): ?string From eaa1354c873de37f42b7876465a1343879db016e Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Tue, 26 Mar 2024 01:05:27 +0100 Subject: [PATCH 55/77] Always print information about CPU(s) used for analysis --- src/Console/Command/FixCommand.php | 20 +++++++++------ tests/Console/Command/FixCommandTest.php | 31 ++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/src/Console/Command/FixCommand.php b/src/Console/Command/FixCommand.php index 587ee5bb441..71d2e9bafd3 100644 --- a/src/Console/Command/FixCommand.php +++ b/src/Console/Command/FixCommand.php @@ -260,22 +260,26 @@ protected function execute(InputInterface $input, OutputInterface $output): int if (null !== $stdErr) { $stdErr->writeln(Application::getAboutWithRuntime(true)); + $isParallel = $resolver->getParallelConfig()->getMaxProcesses() > 1; + + $stdErr->writeln(sprintf( + 'Running analysis on %d core%s.', + $resolver->getParallelConfig()->getMaxProcesses(), + $isParallel ? sprintf( + 's with %d file%s per process', + $resolver->getParallelConfig()->getFilesPerProcess(), + $resolver->getParallelConfig()->getFilesPerProcess() > 1 ? 's' : '' + ) : ' sequentially' + )); // @TODO remove when parallel runner is mature enough and works as expected - if ($resolver->getParallelConfig()->getMaxProcesses() > 1) { + if ($isParallel) { $stdErr->writeln( sprintf( $stdErr->isDecorated() ? '%s' : '%s', 'Parallel runner is an experimental feature and may be unstable, use it at your own risk. Feedback highly appreciated!' ) ); - - $stdErr->writeln(sprintf( - 'Running analysis on %d cores with %d file%s per process.', - $resolver->getParallelConfig()->getMaxProcesses(), - $resolver->getParallelConfig()->getFilesPerProcess(), - $resolver->getParallelConfig()->getFilesPerProcess() > 1 ? 's' : '' - )); } $configFile = $resolver->getConfigFile(); diff --git a/tests/Console/Command/FixCommandTest.php b/tests/Console/Command/FixCommandTest.php index 179c2631fb3..03d5f4c9917 100644 --- a/tests/Console/Command/FixCommandTest.php +++ b/tests/Console/Command/FixCommandTest.php @@ -72,6 +72,37 @@ public function testEmptyFormatValue(): void $cmdTester->getStatusCode(); } + /** + * @covers \PhpCsFixer\Console\Command\WorkerCommand + * @covers \PhpCsFixer\Runner\Runner::fixSequential + */ + public function testSequentialRun(): void + { + $pathToDistConfig = __DIR__.'/../../../.php-cs-fixer.dist.php'; + $configWithFixedParallelConfig = <<setRules(['header_comment' => ['header' => 'SEQUENTIAL!']]); + \$config->setParallelConfig(\\PhpCsFixer\\Runner\\Parallel\\ParallelConfig::sequential()); + + return \$config; + PHP; + $tmpFile = tempnam(sys_get_temp_dir(), 'php-cs-fixer-parallel-config-').'.php'; + file_put_contents($tmpFile, $configWithFixedParallelConfig); + + $cmdTester = $this->doTestExecute( + [ + '--config' => $tmpFile, + 'path' => [__DIR__], + ] + ); + + self::assertStringContainsString('Running analysis on 1 core sequentially.', $cmdTester->getDisplay()); + self::assertStringContainsString('(header_comment)', $cmdTester->getDisplay()); + self::assertSame(8, $cmdTester->getStatusCode()); + } + /** * There's no simple way to cover parallelisation with tests, because it involves a lot of hardcoded logic under the hood, * like opening server, communicating through sockets, etc. That's why we only test `fix` command with proper From a4cc010109a58c5a037d829a3aed570c44f8cdea Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Tue, 26 Mar 2024 01:07:40 +0100 Subject: [PATCH 56/77] Collection of dispatched events is a list --- src/Console/Command/WorkerCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Console/Command/WorkerCommand.php b/src/Console/Command/WorkerCommand.php index 381e82f2c97..013614db36f 100644 --- a/src/Console/Command/WorkerCommand.php +++ b/src/Console/Command/WorkerCommand.php @@ -61,7 +61,7 @@ final class WorkerCommand extends Command private EventDispatcherInterface $eventDispatcher; private ReadonlyCacheManager $readonlyCacheManager; - /** @var array */ + /** @var list */ private array $events; public function __construct(ToolInfoInterface $toolInfo) From cfbf0bbeb86e9846814f7fc758618964ad9cb818 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Tue, 26 Mar 2024 01:09:21 +0100 Subject: [PATCH 57/77] No need for class property to store `ReadonlyCacheManager` in worker command --- src/Console/Command/WorkerCommand.php | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Console/Command/WorkerCommand.php b/src/Console/Command/WorkerCommand.php index 013614db36f..fd058e124d6 100644 --- a/src/Console/Command/WorkerCommand.php +++ b/src/Console/Command/WorkerCommand.php @@ -59,7 +59,6 @@ final class WorkerCommand extends Command private ConfigurationResolver $configurationResolver; private ErrorsManager $errorsManager; private EventDispatcherInterface $eventDispatcher; - private ReadonlyCacheManager $readonlyCacheManager; /** @var list */ private array $events; @@ -248,8 +247,6 @@ private function createRunner(InputInterface $input): Runner $this->toolInfo ); - $this->readonlyCacheManager = new ReadonlyCacheManager($this->configurationResolver->getCacheManager()); - return new Runner( null, // Paths are known when parallelisation server requests new chunk, not now $this->configurationResolver->getFixers(), @@ -258,7 +255,7 @@ private function createRunner(InputInterface $input): Runner $this->errorsManager, $this->configurationResolver->getLinter(), $this->configurationResolver->isDryRun(), - $this->readonlyCacheManager, + new ReadonlyCacheManager($this->configurationResolver->getCacheManager()), $this->configurationResolver->getDirectory(), $this->configurationResolver->shouldStopOnViolation(), ParallelConfig::sequential(), // IMPORTANT! Worker must run in sequential mode From 2d053412019707246f490c435aa7e98bf8a29040 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Tue, 26 Mar 2024 01:11:21 +0100 Subject: [PATCH 58/77] `ParallelAction` is not instantiable --- src/Runner/Parallel/ParallelAction.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Runner/Parallel/ParallelAction.php b/src/Runner/Parallel/ParallelAction.php index 3b7703c185c..c08db60ed57 100644 --- a/src/Runner/Parallel/ParallelAction.php +++ b/src/Runner/Parallel/ParallelAction.php @@ -30,4 +30,6 @@ final class ParallelAction // Actions handled by the worker public const WORKER_RUN = 'run'; public const WORKER_THANK_YOU = 'thankYou'; + + private function __construct() {} } From 71b089ec14d7a9ad36fa82d5af3845ce86ec0742 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Tue, 26 Mar 2024 01:22:14 +0100 Subject: [PATCH 59/77] Make `RunnerConfig` internal --- src/Runner/RunnerConfig.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Runner/RunnerConfig.php b/src/Runner/RunnerConfig.php index eb3dfc83abf..1d6855eb33d 100644 --- a/src/Runner/RunnerConfig.php +++ b/src/Runner/RunnerConfig.php @@ -18,6 +18,8 @@ /** * @author Greg Korba + * + * @internal */ final class RunnerConfig { From 051b9ee72d348aea548568a99648799c40aa5cbc Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Tue, 26 Mar 2024 01:33:50 +0100 Subject: [PATCH 60/77] Do not expect tests for classes without public methods --- tests/AutoReview/ProjectCodeTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/AutoReview/ProjectCodeTest.php b/tests/AutoReview/ProjectCodeTest.php index feb56d28987..470f9334233 100644 --- a/tests/AutoReview/ProjectCodeTest.php +++ b/tests/AutoReview/ProjectCodeTest.php @@ -686,7 +686,7 @@ public static function provideThatSrcClassHaveTestClassCases(): iterable static function (string $className): bool { $rc = new \ReflectionClass($className); - return !$rc->isTrait() && !$rc->isAbstract() && !$rc->isInterface() && \count($rc->getMethods()) > 0; + return !$rc->isTrait() && !$rc->isAbstract() && !$rc->isInterface() && \count($rc->getMethods(\ReflectionMethod::IS_PUBLIC)) > 0; } ) ); From 763cc8b18b58157ecec556f3a8d201690450e563 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Tue, 26 Mar 2024 02:09:39 +0100 Subject: [PATCH 61/77] Fix smoke tests - always expect CPU(s) usage info in the output - always use dist config for PHAR parallel test (because local config can use sequential analysis) --- tests/Smoke/CiIntegrationTest.php | 3 ++- tests/Smoke/PharTest.php | 11 +++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/Smoke/CiIntegrationTest.php b/tests/Smoke/CiIntegrationTest.php index 0a4f2da3741..77092157e29 100644 --- a/tests/Smoke/CiIntegrationTest.php +++ b/tests/Smoke/CiIntegrationTest.php @@ -173,12 +173,13 @@ public function testIntegration( : 'PHP CS Fixer '.preg_quote(Application::VERSION, '/')." by Fabien Potencier, Dariusz Ruminski and contributors.\nPHP runtime: ".PHP_VERSION; $pattern = sprintf( - '/^(?:%s)?(?:%s)?(?:%s)?(?:%s)?%s\n%s\n([\.S]{%d})%s\n%s$/', + '/^(?:%s)?(?:%s)?(?:%s)?(?:%s)?%s\n%s\n%s\n([\.S]{%d})%s\n%s$/', preg_quote($optionalDeprecatedVersionWarning, '/'), preg_quote($optionalIncompatibilityWarning, '/'), preg_quote($optionalXdebugWarning, '/'), preg_quote($optionalWarningsHelp, '/'), $aboutSubpattern, + 'Running analysis on \d+ core(?: sequentially|s with \d+ files? per process)+\.', preg_quote('Loaded config default from ".php-cs-fixer.dist.php".', '/'), \strlen($expectedResult3FilesDots), preg_quote($expectedResult3FilesPercentage, '/'), diff --git a/tests/Smoke/PharTest.php b/tests/Smoke/PharTest.php index df414fe8b84..568c6285be8 100644 --- a/tests/Smoke/PharTest.php +++ b/tests/Smoke/PharTest.php @@ -92,15 +92,18 @@ public function testDescribe(): void public function testFixSequential(): void { - self::assertSame( - 0, - self::executePharCommand('fix src/Config.php -vvv --dry-run --sequential --diff --using-cache=no 2>&1')->getCode() + $command = self::executePharCommand('fix src/Config.php -vvv --dry-run --sequential --diff --using-cache=no 2>&1'); + + self::assertSame(0, $command->getCode()); + self::assertMatchesRegularExpression( + '/Running analysis on 1 core sequentially/', + $command->getOutput() ); } public function testFixParallel(): void { - $command = self::executePharCommand('fix src/Config.php -vvv --dry-run --diff --using-cache=no 2>&1'); + $command = self::executePharCommand('fix src/Config.php -vvv --dry-run --diff --using-cache=no --config=.php-cs-fixer.dist.php 2>&1'); self::assertSame(0, $command->getCode()); self::assertMatchesRegularExpression( From df35f4f2e7d60971569def2b009383411560c827 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Wed, 1 May 2024 02:42:36 +0200 Subject: [PATCH 62/77] Align with PHPStan level 7 --- src/Console/Command/WorkerCommand.php | 2 +- src/Runner/Parallel/ParallelConfig.php | 4 ++++ src/Runner/Parallel/Process.php | 14 ++++++++------ src/Runner/Runner.php | 18 ++++++++++++------ tests/Console/Command/WorkerCommandTest.php | 2 +- tests/Runner/Parallel/ParallelConfigTest.php | 1 + 6 files changed, 27 insertions(+), 14 deletions(-) diff --git a/src/Console/Command/WorkerCommand.php b/src/Console/Command/WorkerCommand.php index fd058e124d6..09c5131dbb2 100644 --- a/src/Console/Command/WorkerCommand.php +++ b/src/Console/Command/WorkerCommand.php @@ -243,7 +243,7 @@ private function createRunner(InputInterface $input): Runner 'diff' => $input->getOption('diff'), 'stop-on-violation' => false, // @TODO Pass this option to the runner ], - getcwd(), + getcwd(), // @phpstan-ignore-line $this->toolInfo ); diff --git a/src/Runner/Parallel/ParallelConfig.php b/src/Runner/Parallel/ParallelConfig.php index 2268cf83048..8f77e79b196 100644 --- a/src/Runner/Parallel/ParallelConfig.php +++ b/src/Runner/Parallel/ParallelConfig.php @@ -69,6 +69,10 @@ public static function sequential(): self return new self(1); } + /** + * @param positive-int $filesPerProcess + * @param positive-int $processTimeout + */ public static function detect( int $filesPerProcess = self::DEFAULT_FILES_PER_PROCESS, int $processTimeout = self::DEFAULT_PROCESS_TIMEOUT diff --git a/src/Runner/Parallel/Process.php b/src/Runner/Parallel/Process.php index e2514c8bb58..0376df47839 100644 --- a/src/Runner/Parallel/Process.php +++ b/src/Runner/Parallel/Process.php @@ -41,10 +41,10 @@ final class Process private ?ReactProcess $process = null; private ?WritableStreamInterface $in = null; - /** @var false|resource */ + /** @var resource */ private $stdErr; - /** @var false|resource */ + /** @var resource */ private $stdOut; /** @var callable(mixed[]): void */ @@ -69,15 +69,17 @@ public function __construct(string $command, LoopInterface $loop, int $timeoutSe */ public function start(callable $onData, callable $onError, callable $onExit): void { - $this->stdOut = tmpfile(); - if (false === $this->stdOut) { + $stdOut = tmpfile(); + if (false === $stdOut) { throw new ParallelisationException('Failed creating temp file for stdOut.'); } + $this->stdOut = $stdOut; - $this->stdErr = tmpfile(); - if (false === $this->stdErr) { + $stdErr = tmpfile(); + if (false === $stdErr) { throw new ParallelisationException('Failed creating temp file for stdErr.'); } + $this->stdErr = $stdErr; $this->onData = $onData; $this->onError = $onError; diff --git a/src/Runner/Runner.php b/src/Runner/Runner.php index f1b16f44878..a423eca38bb 100644 --- a/src/Runner/Runner.php +++ b/src/Runner/Runner.php @@ -70,7 +70,7 @@ final class Runner private LinterInterface $linter; /** - * @var null|\Traversable<\SplFileInfo> + * @var null|\Traversable */ private $fileIterator; @@ -90,8 +90,8 @@ final class Runner private ?string $configFile; /** - * @param null|\Traversable<\SplFileInfo> $fileIterator - * @param list $fixers + * @param null|\Traversable $fileIterator + * @param list $fixers */ public function __construct( ?\Traversable $fileIterator, @@ -109,7 +109,9 @@ public function __construct( ?InputInterface $input = null, ?string $configFile = null ) { - $this->fileCount = \count($fileIterator ?? []); // Required only for main process (calculating workers count) + // Required only for main process (calculating workers count) + $this->fileCount = null !== $fileIterator ? \count(iterator_to_array($fileIterator)) : 0; + $this->fileIterator = $fileIterator; $this->fixers = $fixers; $this->differ = $differ; @@ -126,7 +128,7 @@ public function __construct( } /** - * @param iterable<\SplFileInfo> $fileIterator + * @param \Traversable $fileIterator */ public function setFileIterator(iterable $fileIterator): void { @@ -255,7 +257,11 @@ function (array $workerResponse) use ($processPool, $process, $identifier, $file && !$this->isDryRun ) ) { - $this->cacheManager->setFile($fileRelativePath, file_get_contents($fileAbsolutePath)); + $fileContent = file_get_contents($fileAbsolutePath); + + if (false !== $fileContent) { + $this->cacheManager->setFile($fileRelativePath, $fileContent); + } } // Worker requests for another file chunk when all files were processed diff --git a/tests/Console/Command/WorkerCommandTest.php b/tests/Console/Command/WorkerCommandTest.php index 288cae7c481..7cb99db4e68 100644 --- a/tests/Console/Command/WorkerCommandTest.php +++ b/tests/Console/Command/WorkerCommandTest.php @@ -88,7 +88,7 @@ public function testWorkerCommunicatesWithTheServer(): void new ArrayInput([], (new FixCommand(new ToolInfo()))->getDefinition()) ); $process = new Process(implode(' ', $processFactory->getCommandArgs( - $serverPort, + $serverPort, // @phpstan-ignore-line $processIdentifier, new RunnerConfig(true, false, ParallelConfig::sequential()) ))); diff --git a/tests/Runner/Parallel/ParallelConfigTest.php b/tests/Runner/Parallel/ParallelConfigTest.php index 1112ca34973..5e3128709c1 100644 --- a/tests/Runner/Parallel/ParallelConfigTest.php +++ b/tests/Runner/Parallel/ParallelConfigTest.php @@ -36,6 +36,7 @@ public function testExceptionIsThrownOnNegativeValues( ): void { $this->expectException(\InvalidArgumentException::class); + // @phpstan-ignore-next-line False-positive, we pass negative values to the constructor on purpose. new ParallelConfig($maxProcesses, $filesPerProcess, $processTimeout); } From ee9def70cc795b29015564a7290814e7f26de9ec Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Mon, 6 May 2024 21:39:01 +0200 Subject: [PATCH 63/77] Align `ReadonlyCacheManager` with changes in main branch --- src/Runner/Parallel/ReadonlyCacheManager.php | 2 ++ tests/Runner/Parallel/ReadonlyCacheManagerTest.php | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/src/Runner/Parallel/ReadonlyCacheManager.php b/src/Runner/Parallel/ReadonlyCacheManager.php index d59cc445b1e..ea6431956fb 100644 --- a/src/Runner/Parallel/ReadonlyCacheManager.php +++ b/src/Runner/Parallel/ReadonlyCacheManager.php @@ -31,4 +31,6 @@ public function needFixing(string $file, string $fileContent): bool } public function setFile(string $file, string $fileContent): void {} + + public function setFileHash(string $file, string $hash): void {} } diff --git a/tests/Runner/Parallel/ReadonlyCacheManagerTest.php b/tests/Runner/Parallel/ReadonlyCacheManagerTest.php index 1e21d833fdb..e4c9c251851 100644 --- a/tests/Runner/Parallel/ReadonlyCacheManagerTest.php +++ b/tests/Runner/Parallel/ReadonlyCacheManagerTest.php @@ -79,6 +79,11 @@ public function setFile(string $file, string $fileContent): void { throw new \LogicException('Should not be called.'); } + + public function setFileHash(string $file, string $hash): void + { + throw new \LogicException('Should not be called.'); + } }; } } From c6ea7843ab7d259e0c2a0a9948a660508c9f8939 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dariusz=20Rumi=C5=84ski?= Date: Tue, 7 May 2024 23:54:29 +0200 Subject: [PATCH 64/77] Refactoring and cosmetic changes * syntax sugar * sequential option - no shortcut, as we do not use shortcut for other any other option * explicitly mark `ProcessFactory::getCommandArgs` as `@private`, to indicate it shall not be called from outside of this class, ref https://github.com/PHP-CS-Fixer/PHP-CS-Fixer/pull/7777#discussion_r1538396175 * `WorkerCommand` options written as single-liners, so unified as all other commands and easier to copy-paste * add explicit comment for hardcoded option * remove unused var * rename `ParallelRunnerConfigInterface` to `ParallelAwareConfigInterface` * make `ParallelConfig` only DTO and externalize factory methods to dedicated class * `Runner` - don't reevaluate `maxFixerPerProcess` in each loop * explicitly call generator, avoid simple 'job' as too similar to job-id * rename variables related to file chunks --- .php-cs-fixer.dist.php | 4 +- doc/usage.rst | 4 +- src/Config.php | 11 ++-- src/Console/Command/FixCommand.php | 2 +- src/Console/Command/WorkerCommand.php | 50 ++++------------- src/Console/ConfigurationResolver.php | 7 +-- ...e.php => ParallelAwareConfigInterface.php} | 4 +- src/Runner/Parallel/ParallelConfig.php | 25 --------- src/Runner/Parallel/ParallelConfigFactory.php | 53 +++++++++++++++++++ src/Runner/Parallel/ProcessFactory.php | 2 + src/Runner/Runner.php | 28 +++++----- tests/Console/Command/FixCommandTest.php | 2 +- tests/Console/Command/WorkerCommandTest.php | 4 +- tests/Console/ConfigurationResolverTest.php | 3 +- .../Parallel/ParallelConfigFactoryTest.php | 51 ++++++++++++++++++ tests/Runner/Parallel/ParallelConfigTest.php | 17 ------ tests/Runner/Parallel/ProcessFactoryTest.php | 4 +- tests/Runner/Parallel/ProcessPoolTest.php | 4 +- 18 files changed, 159 insertions(+), 116 deletions(-) rename src/{ParallelRunnerConfigInterface.php => ParallelAwareConfigInterface.php} (81%) create mode 100644 src/Runner/Parallel/ParallelConfigFactory.php create mode 100644 tests/Runner/Parallel/ParallelConfigFactoryTest.php diff --git a/.php-cs-fixer.dist.php b/.php-cs-fixer.dist.php index c9585edc44e..dac0b4ecb4e 100644 --- a/.php-cs-fixer.dist.php +++ b/.php-cs-fixer.dist.php @@ -14,10 +14,10 @@ use PhpCsFixer\Config; use PhpCsFixer\Finder; -use PhpCsFixer\Runner\Parallel\ParallelConfig; +use PhpCsFixer\Runner\Parallel\ParallelConfigFactory; return (new Config()) - ->setParallelConfig(ParallelConfig::detect()) + ->setParallelConfig(ParallelConfigFactory::detect()) ->setRiskyAllowed(true) ->setRules([ '@PHP74Migration' => true, diff --git a/doc/usage.rst b/doc/usage.rst index 62871b66b0f..0439338dd1b 100644 --- a/doc/usage.rst +++ b/doc/usage.rst @@ -21,14 +21,14 @@ If you do not have config file, you can run following command to fix non-hidden, php php-cs-fixer.phar fix . -You can also fix files in parallel, utilising more CPU cores. You can do this by using config class that implements ``PhpCsFixer\Runner\Parallel\ParallelConfig\ParallelRunnerConfigInterface``, and use ``setParallelConfig()`` method. Recommended way is to utilise auto-detecting parallel configuration: +You can also fix files in parallel, utilising more CPU cores. You can do this by using config class that implements ``PhpCsFixer\Runner\Parallel\ParallelConfig\ParallelAwareConfigInterface``, and use ``setParallelConfig()`` method. Recommended way is to utilise auto-detecting parallel configuration: .. code-block:: php setParallelConfig(ParallelConfig::detect()) + ->setParallelConfig(ParallelConfigFactory::detect()) ; However, in some case you may want to fine-tune parallelisation with explicit values (e.g. in environments where auto-detection does not work properly and suggests more cores than it should): diff --git a/src/Config.php b/src/Config.php index 5f3fad32400..016a4e967bd 100644 --- a/src/Config.php +++ b/src/Config.php @@ -16,13 +16,14 @@ use PhpCsFixer\Fixer\FixerInterface; use PhpCsFixer\Runner\Parallel\ParallelConfig; +use PhpCsFixer\Runner\Parallel\ParallelConfigFactory; /** * @author Fabien Potencier * @author Katsuhiro Ogawa * @author Dariusz Rumiński */ -class Config implements ConfigInterface, ParallelRunnerConfigInterface +class Config implements ConfigInterface, ParallelAwareConfigInterface { private string $cacheFile = '.php-cs-fixer.cache'; @@ -48,7 +49,7 @@ class Config implements ConfigInterface, ParallelRunnerConfigInterface private string $name; - private ?ParallelConfig $parallelRunnerConfig; + private ?ParallelConfig $parallelConfig; /** * @var null|string @@ -123,7 +124,9 @@ public function getName(): string public function getParallelConfig(): ParallelConfig { - return $this->parallelRunnerConfig ?? ParallelConfig::sequential(); + $this->parallelConfig ??= ParallelConfigFactory::sequential(); + + return $this->parallelConfig; } public function getPhpExecutable(): ?string @@ -199,7 +202,7 @@ public function setLineEnding(string $lineEnding): ConfigInterface public function setParallelConfig(ParallelConfig $config): ConfigInterface { - $this->parallelRunnerConfig = $config; + $this->parallelConfig = $config; return $this; } diff --git a/src/Console/Command/FixCommand.php b/src/Console/Command/FixCommand.php index 71d2e9bafd3..39a27ed7c43 100644 --- a/src/Console/Command/FixCommand.php +++ b/src/Console/Command/FixCommand.php @@ -214,7 +214,7 @@ protected function configure(): void new InputOption('format', '', InputOption::VALUE_REQUIRED, 'To output results in other formats.'), new InputOption('stop-on-violation', '', InputOption::VALUE_NONE, 'Stop execution on first violation.'), new InputOption('show-progress', '', InputOption::VALUE_REQUIRED, 'Type of progress indicator (none, dots).'), - new InputOption('sequential', 's', InputOption::VALUE_NONE, 'Enforce sequential analysis.'), + new InputOption('sequential', '', InputOption::VALUE_NONE, 'Enforce sequential analysis.'), ] ); } diff --git a/src/Console/Command/WorkerCommand.php b/src/Console/Command/WorkerCommand.php index 09c5131dbb2..0c71fcf60b9 100644 --- a/src/Console/Command/WorkerCommand.php +++ b/src/Console/Command/WorkerCommand.php @@ -21,7 +21,7 @@ use PhpCsFixer\Error\ErrorsManager; use PhpCsFixer\FixerFileProcessedEvent; use PhpCsFixer\Runner\Parallel\ParallelAction; -use PhpCsFixer\Runner\Parallel\ParallelConfig; +use PhpCsFixer\Runner\Parallel\ParallelConfigFactory; use PhpCsFixer\Runner\Parallel\ParallelisationException; use PhpCsFixer\Runner\Parallel\ReadonlyCacheManager; use PhpCsFixer\Runner\Runner; @@ -77,43 +77,13 @@ protected function configure(): void { $this->setDefinition( [ - new InputOption( - 'port', - null, - InputOption::VALUE_REQUIRED, - 'Specifies parallelisation server\'s port.' - ), - new InputOption( - 'identifier', - null, - InputOption::VALUE_REQUIRED, - 'Specifies parallelisation process\' identifier.' - ), - new InputOption( - 'allow-risky', - '', - InputOption::VALUE_REQUIRED, - 'Are risky fixers allowed (can be `yes` or `no`).' - ), + new InputOption('port', null, InputOption::VALUE_REQUIRED, 'Specifies parallelisation server\'s port.'), + new InputOption('identifier', null, InputOption::VALUE_REQUIRED, 'Specifies parallelisation process\' identifier.'), + new InputOption('allow-risky', '', InputOption::VALUE_REQUIRED, 'Are risky fixers allowed (can be `yes` or `no`).'), new InputOption('config', '', InputOption::VALUE_REQUIRED, 'The path to a config file.'), - new InputOption( - 'dry-run', - '', - InputOption::VALUE_NONE, - 'Only shows which files would have been modified.' - ), - new InputOption( - 'rules', - '', - InputOption::VALUE_REQUIRED, - 'List of rules that should be run against configured paths.' - ), - new InputOption( - 'using-cache', - '', - InputOption::VALUE_REQUIRED, - 'Should cache be used (can be `yes` or `no`).' - ), + new InputOption('dry-run', '', InputOption::VALUE_NONE, 'Only shows which files would have been modified.'), + new InputOption('rules', '', InputOption::VALUE_REQUIRED, 'List of rules that should be run against configured paths.'), + new InputOption('using-cache', '', InputOption::VALUE_REQUIRED, 'Should cache be used (can be `yes` or `no`).'), new InputOption('cache-file', '', InputOption::VALUE_REQUIRED, 'The path to the cache file.'), new InputOption('diff', '', InputOption::VALUE_NONE, 'Prints diff for each file.'), ] @@ -182,7 +152,7 @@ function (ConnectionInterface $connection) use ($loop, $runner, $identifier): vo /** @var iterable $files */ $files = $json['files']; - foreach ($files as $i => $absolutePath) { + foreach ($files as $absolutePath) { $relativePath = $this->configurationResolver->getDirectory()->getRelativePathTo($absolutePath); // Reset events because we want to collect only those coming from analysed files chunk @@ -237,7 +207,7 @@ private function createRunner(InputInterface $input): Runner 'dry-run' => $input->getOption('dry-run'), 'rules' => $passedRules, 'path' => [], - 'path-mode' => ConfigurationResolver::PATH_MODE_OVERRIDE, + 'path-mode' => ConfigurationResolver::PATH_MODE_OVERRIDE, // IMPORTANT! WorkerCommand is called with file that already passed filtering, so here we can rely on PATH_MODE_OVERRIDE. 'using-cache' => $input->getOption('using-cache'), 'cache-file' => $input->getOption('cache-file'), 'diff' => $input->getOption('diff'), @@ -258,7 +228,7 @@ private function createRunner(InputInterface $input): Runner new ReadonlyCacheManager($this->configurationResolver->getCacheManager()), $this->configurationResolver->getDirectory(), $this->configurationResolver->shouldStopOnViolation(), - ParallelConfig::sequential(), // IMPORTANT! Worker must run in sequential mode + ParallelConfigFactory::sequential(), // IMPORTANT! Worker must run in sequential mode. null, $this->configurationResolver->getConfigFile() ); diff --git a/src/Console/ConfigurationResolver.php b/src/Console/ConfigurationResolver.php index 211f19990e3..7e11e8affca 100644 --- a/src/Console/ConfigurationResolver.php +++ b/src/Console/ConfigurationResolver.php @@ -35,10 +35,11 @@ use PhpCsFixer\FixerFactory; use PhpCsFixer\Linter\Linter; use PhpCsFixer\Linter\LinterInterface; -use PhpCsFixer\ParallelRunnerConfigInterface; +use PhpCsFixer\ParallelAwareConfigInterface; use PhpCsFixer\RuleSet\RuleSet; use PhpCsFixer\RuleSet\RuleSetInterface; use PhpCsFixer\Runner\Parallel\ParallelConfig; +use PhpCsFixer\Runner\Parallel\ParallelConfigFactory; use PhpCsFixer\StdinFileInfo; use PhpCsFixer\ToolInfoInterface; use PhpCsFixer\Utils; @@ -281,9 +282,9 @@ public function getParallelConfig(): ParallelConfig { $config = $this->getConfig(); - return true !== $this->options['sequential'] && $config instanceof ParallelRunnerConfigInterface + return true !== $this->options['sequential'] && $config instanceof ParallelAwareConfigInterface ? $config->getParallelConfig() - : ParallelConfig::sequential(); + : ParallelConfigFactory::sequential(); } public function getConfigFile(): ?string diff --git a/src/ParallelRunnerConfigInterface.php b/src/ParallelAwareConfigInterface.php similarity index 81% rename from src/ParallelRunnerConfigInterface.php rename to src/ParallelAwareConfigInterface.php index f6a0ce1c837..6716cf3ce1d 100644 --- a/src/ParallelRunnerConfigInterface.php +++ b/src/ParallelAwareConfigInterface.php @@ -19,9 +19,9 @@ /** * @author Greg Korba * - * @TODO 4.0 Include parallel runner config in main config + * @TODO 4.0 Include parallel runner config in main ConfigInterface */ -interface ParallelRunnerConfigInterface extends ConfigInterface +interface ParallelAwareConfigInterface extends ConfigInterface { public function getParallelConfig(): ParallelConfig; diff --git a/src/Runner/Parallel/ParallelConfig.php b/src/Runner/Parallel/ParallelConfig.php index 8f77e79b196..2cfa9acef7f 100644 --- a/src/Runner/Parallel/ParallelConfig.php +++ b/src/Runner/Parallel/ParallelConfig.php @@ -14,10 +14,6 @@ namespace PhpCsFixer\Runner\Parallel; -use Fidry\CpuCoreCounter\CpuCoreCounter; -use Fidry\CpuCoreCounter\Finder\DummyCpuCoreFinder; -use Fidry\CpuCoreCounter\Finder\FinderRegistry; - /** * @author Greg Korba */ @@ -63,25 +59,4 @@ public function getProcessTimeout(): int { return $this->processTimeout; } - - public static function sequential(): self - { - return new self(1); - } - - /** - * @param positive-int $filesPerProcess - * @param positive-int $processTimeout - */ - public static function detect( - int $filesPerProcess = self::DEFAULT_FILES_PER_PROCESS, - int $processTimeout = self::DEFAULT_PROCESS_TIMEOUT - ): self { - $counter = new CpuCoreCounter([ - ...FinderRegistry::getDefaultLogicalFinders(), - new DummyCpuCoreFinder(1), - ]); - - return new self($counter->getCount(), $filesPerProcess, $processTimeout); - } } diff --git a/src/Runner/Parallel/ParallelConfigFactory.php b/src/Runner/Parallel/ParallelConfigFactory.php new file mode 100644 index 00000000000..01c44a0292b --- /dev/null +++ b/src/Runner/Parallel/ParallelConfigFactory.php @@ -0,0 +1,53 @@ + + * Dariusz Rumiński + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace PhpCsFixer\Runner\Parallel; + +use Fidry\CpuCoreCounter\CpuCoreCounter; +use Fidry\CpuCoreCounter\Finder\DummyCpuCoreFinder; +use Fidry\CpuCoreCounter\Finder\FinderRegistry; + +/** + * @author Dariusz Rumiński + */ +final class ParallelConfigFactory +{ + private function __construct() {} + + public static function sequential(): ParallelConfig + { + return new ParallelConfig(1); + } + + /** + * @param null|positive-int $filesPerProcess + * @param null|positive-int $processTimeout + */ + public static function detect( + ?int $filesPerProcess = null, + ?int $processTimeout = null + ): ParallelConfig { + $counter = new CpuCoreCounter([ + ...FinderRegistry::getDefaultLogicalFinders(), + new DummyCpuCoreFinder(1), + ]); + + return new ParallelConfig( + ...array_filter( + [$counter->getCount(), $filesPerProcess, $processTimeout], + static fn ($value): bool => null !== $value + ) + ); + } +} diff --git a/src/Runner/Parallel/ProcessFactory.php b/src/Runner/Parallel/ProcessFactory.php index a50f9680bcd..d737b5c3880 100644 --- a/src/Runner/Parallel/ProcessFactory.php +++ b/src/Runner/Parallel/ProcessFactory.php @@ -49,6 +49,8 @@ public function create( } /** + * @private + * * @return list */ public function getCommandArgs(int $serverPort, ProcessIdentifier $identifier, RunnerConfig $runnerConfig): array diff --git a/src/Runner/Runner.php b/src/Runner/Runner.php index a423eca38bb..aa98e7ba6a3 100644 --- a/src/Runner/Runner.php +++ b/src/Runner/Runner.php @@ -34,6 +34,7 @@ use PhpCsFixer\Preg; use PhpCsFixer\Runner\Parallel\ParallelAction; use PhpCsFixer\Runner\Parallel\ParallelConfig; +use PhpCsFixer\Runner\Parallel\ParallelConfigFactory; use PhpCsFixer\Runner\Parallel\ParallelisationException; use PhpCsFixer\Runner\Parallel\ProcessFactory; use PhpCsFixer\Runner\Parallel\ProcessIdentifier; @@ -122,7 +123,7 @@ public function __construct( $this->cacheManager = $cacheManager; $this->directory = $directory ?? new Directory(''); $this->stopOnViolation = $stopOnViolation; - $this->parallelConfig = $parallelConfig ?? ParallelConfig::sequential(); + $this->parallelConfig = $parallelConfig ?? ParallelConfigFactory::sequential(); $this->input = $input; $this->configFile = $configFile; } @@ -132,6 +133,8 @@ public function __construct( */ public function setFileIterator(iterable $fileIterator): void { + // @TODO consider to drop this method and make iterator parameter obligatory in constructor, + // more in https://github.com/PHP-CS-Fixer/PHP-CS-Fixer/pull/7777/files#r1590447581 $this->fileIterator = $fileIterator; } @@ -166,13 +169,14 @@ private function fixParallel(): array } $processPool = new ProcessPool($server); + $maxFilesPerProcess = $this->parallelConfig->getFilesPerProcess(); $fileIterator = $this->getFilteringFileIterator(); $fileIterator->rewind(); - $fileChunk = function () use ($fileIterator): array { + $getFileChunk = static function () use ($fileIterator, $maxFilesPerProcess): array { $files = []; - while (\count($files) < $this->parallelConfig->getFilesPerProcess()) { + while (\count($files) < $maxFilesPerProcess) { $current = $fileIterator->current(); if (null === $current) { @@ -188,13 +192,13 @@ private function fixParallel(): array }; // [REACT] Handle worker's handshake (init connection) - $server->on('connection', static function (ConnectionInterface $connection) use ($processPool, $fileChunk): void { + $server->on('connection', static function (ConnectionInterface $connection) use ($processPool, $getFileChunk): void { $jsonInvalidUtf8Ignore = \defined('JSON_INVALID_UTF8_IGNORE') ? JSON_INVALID_UTF8_IGNORE : 0; $decoder = new Decoder($connection, true, 512, $jsonInvalidUtf8Ignore); $encoder = new Encoder($connection, $jsonInvalidUtf8Ignore); // [REACT] Bind connection when worker's process requests "hello" action (enables 2-way communication) - $decoder->on('data', static function (array $data) use ($processPool, $fileChunk, $decoder, $encoder): void { + $decoder->on('data', static function (array $data) use ($processPool, $getFileChunk, $decoder, $encoder): void { if (ParallelAction::RUNNER_HELLO !== $data['action']) { return; } @@ -202,16 +206,16 @@ private function fixParallel(): array $identifier = ProcessIdentifier::fromRaw($data['identifier']); $process = $processPool->getProcess($identifier); $process->bindConnection($decoder, $encoder); - $job = $fileChunk(); + $fileChunk = $getFileChunk(); - if (0 === \count($job)) { + if (0 === \count($fileChunk)) { $process->request(['action' => ParallelAction::WORKER_THANK_YOU]); $processPool->endProcessIfKnown($identifier); return; } - $process->request(['action' => ParallelAction::WORKER_RUN, 'files' => $job]); + $process->request(['action' => ParallelAction::WORKER_RUN, 'files' => $fileChunk]); }); }); @@ -237,7 +241,7 @@ private function fixParallel(): array $processPool->addProcess($identifier, $process); $process->start( // [REACT] Handle workers' responses (multiple actions possible) - function (array $workerResponse) use ($processPool, $process, $identifier, $fileChunk, &$changed): void { + function (array $workerResponse) use ($processPool, $process, $identifier, $getFileChunk, &$changed): void { // File analysis result (we want close-to-realtime progress with frequent cache savings) if (ParallelAction::RUNNER_RESULT === $workerResponse['action']) { $fileAbsolutePath = $workerResponse['file']; @@ -282,16 +286,16 @@ function (array $workerResponse) use ($processPool, $process, $identifier, $file if (ParallelAction::RUNNER_GET_FILE_CHUNK === $workerResponse['action']) { // Request another chunk of files, if still available - $job = $fileChunk(); + $fileChunk = $getFileChunk(); - if (0 === \count($job)) { + if (0 === \count($fileChunk)) { $process->request(['action' => ParallelAction::WORKER_THANK_YOU]); $processPool->endProcessIfKnown($identifier); return; } - $process->request(['action' => ParallelAction::WORKER_RUN, 'files' => $job]); + $process->request(['action' => ParallelAction::WORKER_RUN, 'files' => $fileChunk]); return; } diff --git a/tests/Console/Command/FixCommandTest.php b/tests/Console/Command/FixCommandTest.php index 03d5f4c9917..47b32772097 100644 --- a/tests/Console/Command/FixCommandTest.php +++ b/tests/Console/Command/FixCommandTest.php @@ -84,7 +84,7 @@ public function testSequentialRun(): void \$config = require '{$pathToDistConfig}'; \$config->setRules(['header_comment' => ['header' => 'SEQUENTIAL!']]); - \$config->setParallelConfig(\\PhpCsFixer\\Runner\\Parallel\\ParallelConfig::sequential()); + \$config->setParallelConfig(\\PhpCsFixer\\Runner\\Parallel\\ParallelConfigFactory::sequential()); return \$config; PHP; diff --git a/tests/Console/Command/WorkerCommandTest.php b/tests/Console/Command/WorkerCommandTest.php index 7cb99db4e68..3e688594840 100644 --- a/tests/Console/Command/WorkerCommandTest.php +++ b/tests/Console/Command/WorkerCommandTest.php @@ -21,7 +21,7 @@ use PhpCsFixer\Console\Command\WorkerCommand; use PhpCsFixer\FixerFileProcessedEvent; use PhpCsFixer\Runner\Parallel\ParallelAction; -use PhpCsFixer\Runner\Parallel\ParallelConfig; +use PhpCsFixer\Runner\Parallel\ParallelConfigFactory; use PhpCsFixer\Runner\Parallel\ParallelisationException; use PhpCsFixer\Runner\Parallel\ProcessFactory; use PhpCsFixer\Runner\Parallel\ProcessIdentifier; @@ -90,7 +90,7 @@ public function testWorkerCommunicatesWithTheServer(): void $process = new Process(implode(' ', $processFactory->getCommandArgs( $serverPort, // @phpstan-ignore-line $processIdentifier, - new RunnerConfig(true, false, ParallelConfig::sequential()) + new RunnerConfig(true, false, ParallelConfigFactory::sequential()) ))); /** diff --git a/tests/Console/ConfigurationResolverTest.php b/tests/Console/ConfigurationResolverTest.php index 06d1c3a4d91..2a390ad2dcf 100644 --- a/tests/Console/ConfigurationResolverTest.php +++ b/tests/Console/ConfigurationResolverTest.php @@ -32,6 +32,7 @@ use PhpCsFixer\FixerConfiguration\FixerOptionBuilder; use PhpCsFixer\FixerDefinition\FixerDefinitionInterface; use PhpCsFixer\Runner\Parallel\ParallelConfig; +use PhpCsFixer\Runner\Parallel\ParallelConfigFactory; use PhpCsFixer\Tests\TestCase; use PhpCsFixer\Tokenizer\Tokens; use PhpCsFixer\ToolInfoInterface; @@ -60,7 +61,7 @@ public function testResolveParallelConfig(): void public function testDefaultParallelConfigFallbacksToSequential(): void { $parallelConfig = $this->createConfigurationResolver([])->getParallelConfig(); - $defaultParallelConfig = ParallelConfig::sequential(); + $defaultParallelConfig = ParallelConfigFactory::sequential(); self::assertSame($defaultParallelConfig->getMaxProcesses(), $parallelConfig->getMaxProcesses()); self::assertSame($defaultParallelConfig->getFilesPerProcess(), $parallelConfig->getFilesPerProcess()); diff --git a/tests/Runner/Parallel/ParallelConfigFactoryTest.php b/tests/Runner/Parallel/ParallelConfigFactoryTest.php new file mode 100644 index 00000000000..0fa0f72c80c --- /dev/null +++ b/tests/Runner/Parallel/ParallelConfigFactoryTest.php @@ -0,0 +1,51 @@ + + * Dariusz Rumiński + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace PhpCsFixer\Tests\Runner\Parallel; + +use PhpCsFixer\Runner\Parallel\ParallelConfigFactory; +use PhpCsFixer\Tests\TestCase; + +/** + * @internal + * + * @covers \PhpCsFixer\Runner\Parallel\ParallelConfigFactory + * + * @TODO Test `detect()` method, but first discuss the best way to do it. + */ +final class ParallelConfigFactoryTest extends TestCase +{ + public function testSequentialConfigHasExactlyOneProcess(): void + { + $config = ParallelConfigFactory::sequential(); + + self::assertSame(1, $config->getMaxProcesses()); + } + + public function testDetectConfigurationWithoutParams(): void + { + $config = ParallelConfigFactory::detect(); + + self::assertSame(10, $config->getFilesPerProcess()); + self::assertSame(120, $config->getProcessTimeout()); + } + + public function testDetectConfigurationWithParams(): void + { + $config = ParallelConfigFactory::detect(22, 2_200); + + self::assertSame(22, $config->getFilesPerProcess()); + self::assertSame(2_200, $config->getProcessTimeout()); + } +} diff --git a/tests/Runner/Parallel/ParallelConfigTest.php b/tests/Runner/Parallel/ParallelConfigTest.php index 5e3128709c1..3c5cedff0db 100644 --- a/tests/Runner/Parallel/ParallelConfigTest.php +++ b/tests/Runner/Parallel/ParallelConfigTest.php @@ -21,8 +21,6 @@ * @internal * * @covers \PhpCsFixer\Runner\Parallel\ParallelConfig - * - * @TODO Test `detect()` method, but first discuss the best way to do it. */ final class ParallelConfigTest extends TestCase { @@ -60,19 +58,4 @@ public function testGettersAreReturningProperValues(): void self::assertSame(10, $config->getFilesPerProcess()); self::assertSame(120, $config->getProcessTimeout()); } - - public function testSequentialConfigHasExactlyOneProcess(): void - { - $config = ParallelConfig::sequential(); - - self::assertSame(1, $config->getMaxProcesses()); - } - - public function testDetectConfiguration(): void - { - $config = ParallelConfig::detect(1, 100); - - self::assertSame(1, $config->getFilesPerProcess()); - self::assertSame(100, $config->getProcessTimeout()); - } } diff --git a/tests/Runner/Parallel/ProcessFactoryTest.php b/tests/Runner/Parallel/ProcessFactoryTest.php index 0e1a41fed47..b49e73fbe1f 100644 --- a/tests/Runner/Parallel/ProcessFactoryTest.php +++ b/tests/Runner/Parallel/ProcessFactoryTest.php @@ -16,7 +16,7 @@ use PhpCsFixer\Console\Command\FixCommand; use PhpCsFixer\Preg; -use PhpCsFixer\Runner\Parallel\ParallelConfig; +use PhpCsFixer\Runner\Parallel\ParallelConfigFactory; use PhpCsFixer\Runner\Parallel\ProcessFactory; use PhpCsFixer\Runner\Parallel\ProcessIdentifier; use PhpCsFixer\Runner\RunnerConfig; @@ -128,6 +128,6 @@ public static function provideCreateCases(): iterable private static function createRunnerConfig(bool $dryRun): RunnerConfig { - return new RunnerConfig($dryRun, false, ParallelConfig::sequential()); + return new RunnerConfig($dryRun, false, ParallelConfigFactory::sequential()); } } diff --git a/tests/Runner/Parallel/ProcessPoolTest.php b/tests/Runner/Parallel/ProcessPoolTest.php index cab6bccd13d..9dab7011e6f 100644 --- a/tests/Runner/Parallel/ProcessPoolTest.php +++ b/tests/Runner/Parallel/ProcessPoolTest.php @@ -15,7 +15,7 @@ namespace PhpCsFixer\Tests\Runner\Parallel; use PhpCsFixer\Console\Command\FixCommand; -use PhpCsFixer\Runner\Parallel\ParallelConfig; +use PhpCsFixer\Runner\Parallel\ParallelConfigFactory; use PhpCsFixer\Runner\Parallel\ParallelisationException; use PhpCsFixer\Runner\Parallel\Process; use PhpCsFixer\Runner\Parallel\ProcessFactory; @@ -124,7 +124,7 @@ private function createProcess(ProcessIdentifier $identifier): Process new RunnerConfig( true, false, - ParallelConfig::sequential() + ParallelConfigFactory::sequential() ), $identifier, 10_000 From 97a2764faa17c94b354f02e29fb4d2f79a8f0088 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dariusz=20Rumi=C5=84ski?= Date: Wed, 8 May 2024 00:08:30 +0200 Subject: [PATCH 65/77] More strict approach in `WorkerCommand` * explicitly crash on unexpected parallel action * explicitly crash on unexpected events count, i.e. when someone wrongly refactors EventBus * no need to calculate relative path + validate if result has proper size * introduce `ErrorsManager::reportWorkerError()`: don't report `WorkerError` via `report(Error)` method --- src/Console/Command/WorkerCommand.php | 19 ++++++++++++------- src/Error/ErrorsManager.php | 16 ++++++---------- src/Runner/Runner.php | 6 +++--- tests/Error/ErrorsManagerTest.php | 2 +- 4 files changed, 22 insertions(+), 21 deletions(-) diff --git a/src/Console/Command/WorkerCommand.php b/src/Console/Command/WorkerCommand.php index 0c71fcf60b9..821f3deb92e 100644 --- a/src/Console/Command/WorkerCommand.php +++ b/src/Console/Command/WorkerCommand.php @@ -144,28 +144,33 @@ function (ConnectionInterface $connection) use ($loop, $runner, $identifier): vo return; } - // At this point we only expect analysis requests, so let's return early for any other message if (ParallelAction::WORKER_RUN !== $action) { - return; + // At this point we only expect analysis requests, if any other action happen, we need to fix the code. + throw new \LogicException(sprintf('Unexpected action ParallelAction::%s.', $action)); } /** @var iterable $files */ $files = $json['files']; foreach ($files as $absolutePath) { - $relativePath = $this->configurationResolver->getDirectory()->getRelativePathTo($absolutePath); - // Reset events because we want to collect only those coming from analysed files chunk $this->events = []; $runner->setFileIterator(new \ArrayIterator([new \SplFileInfo($absolutePath)])); $analysisResult = $runner->fix(); + if (1 !== \count($this->events)) { + throw new ParallelisationException('Runner did not report a fixing event or reported too many.'); + } + + if (1 < \count($analysisResult)) { + throw new ParallelisationException('Runner returned more analysis results than expected.'); + } + $out->write([ 'action' => ParallelAction::RUNNER_RESULT, 'file' => $absolutePath, - // @phpstan-ignore-next-line False-positive caused by assigning empty array to $events property - 'status' => isset($this->events[0]) ? $this->events[0]->getStatus() : null, - 'fixInfo' => $analysisResult[$relativePath] ?? null, + 'status' => $this->events[0]->getStatus(), + 'fixInfo' => array_pop($analysisResult), 'errors' => $this->errorsManager->forPath($absolutePath), ]); } diff --git a/src/Error/ErrorsManager.php b/src/Error/ErrorsManager.php index 2a42b3c7ff6..6f060b813b0 100644 --- a/src/Error/ErrorsManager.php +++ b/src/Error/ErrorsManager.php @@ -91,17 +91,13 @@ public function isEmpty(): bool return [] === $this->errors && [] === $this->workerErrors; } - /** - * @param Error|WorkerError $error - */ - public function report($error): void + public function report(Error $error): void { - if ($error instanceof WorkerError) { - $this->workerErrors[] = $error; - - return; - } - $this->errors[] = $error; } + + public function reportWorkerError(WorkerError $error): void + { + $this->workerErrors[] = $error; + } } diff --git a/src/Runner/Runner.php b/src/Runner/Runner.php index aa98e7ba6a3..e3d3e7009a0 100644 --- a/src/Runner/Runner.php +++ b/src/Runner/Runner.php @@ -301,7 +301,7 @@ function (array $workerResponse) use ($processPool, $process, $identifier, $getF } if (ParallelAction::RUNNER_ERROR_REPORT === $workerResponse['action']) { - $this->errorsManager->report(new WorkerError( + $this->errorsManager->reportWorkerError(new WorkerError( $workerResponse['message'], $workerResponse['file'], (int) $workerResponse['line'], @@ -317,7 +317,7 @@ function (array $workerResponse) use ($processPool, $process, $identifier, $getF // [REACT] Handle errors encountered during worker's execution function (\Throwable $error) use ($processPool): void { - $this->errorsManager->report(new WorkerError( + $this->errorsManager->reportWorkerError(new WorkerError( $error->getMessage(), $error->getFile(), $error->getLine(), @@ -344,7 +344,7 @@ function ($exitCode, string $output) use ($processPool, $identifier): void { if ($errorsReported > 0) { $error = json_decode($matches[1][0], true); - $this->errorsManager->report(new WorkerError( + $this->errorsManager->reportWorkerError(new WorkerError( $error['message'], $error['file'], (int) $error['line'], diff --git a/tests/Error/ErrorsManagerTest.php b/tests/Error/ErrorsManagerTest.php index f81cdc7e4a3..420b6e3bf6e 100644 --- a/tests/Error/ErrorsManagerTest.php +++ b/tests/Error/ErrorsManagerTest.php @@ -111,7 +111,7 @@ public function testThatCanReportAndRetrieveWorkerErrors(): void $error = new WorkerError('Boom!', 'foo.php', 123, 1, '#0 Foo\n#1 Bar'); $errorsManager = new ErrorsManager(); - $errorsManager->report($error); + $errorsManager->reportWorkerError($error); self::assertFalse($errorsManager->isEmpty()); From ab01e66724fee7c08ca6743fbe6ac7233157eb28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dariusz=20Rumi=C5=84ski?= Date: Wed, 8 May 2024 00:19:29 +0200 Subject: [PATCH 66/77] Auto-detected parallel config by default for future mode or as an opt-in via env variable --- .php-cs-fixer.dist.php | 2 +- src/Config.php | 11 ++++++++--- src/Console/Command/FixCommand.php | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.php-cs-fixer.dist.php b/.php-cs-fixer.dist.php index dac0b4ecb4e..36ccbb37e6d 100644 --- a/.php-cs-fixer.dist.php +++ b/.php-cs-fixer.dist.php @@ -17,7 +17,7 @@ use PhpCsFixer\Runner\Parallel\ParallelConfigFactory; return (new Config()) - ->setParallelConfig(ParallelConfigFactory::detect()) + ->setParallelConfig(ParallelConfigFactory::detect()) // @TODO 4.0 no need to call this manually ->setRiskyAllowed(true) ->setRules([ '@PHP74Migration' => true, diff --git a/src/Config.php b/src/Config.php index 016a4e967bd..29ef94c4894 100644 --- a/src/Config.php +++ b/src/Config.php @@ -49,7 +49,7 @@ class Config implements ConfigInterface, ParallelAwareConfigInterface private string $name; - private ?ParallelConfig $parallelConfig; + private ParallelConfig $parallelConfig; /** * @var null|string @@ -75,6 +75,13 @@ public function __construct(string $name = 'default') $this->name = $name; $this->rules = ['@PSR12' => true]; } + + // @TODO 4.0 cleanup + if (Utils::isFutureModeEnabled() || filter_var(getenv('PHP_CS_FIXER_PARALLEL'), FILTER_VALIDATE_BOOL)) { + $this->parallelConfig = ParallelConfigFactory::detect(); + } else { + $this->parallelConfig = ParallelConfigFactory::sequential(); + } } public function getCacheFile(): string @@ -124,8 +131,6 @@ public function getName(): string public function getParallelConfig(): ParallelConfig { - $this->parallelConfig ??= ParallelConfigFactory::sequential(); - return $this->parallelConfig; } diff --git a/src/Console/Command/FixCommand.php b/src/Console/Command/FixCommand.php index 39a27ed7c43..a25f844052c 100644 --- a/src/Console/Command/FixCommand.php +++ b/src/Console/Command/FixCommand.php @@ -272,7 +272,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int ) : ' sequentially' )); - // @TODO remove when parallel runner is mature enough and works as expected + // @TODO v4 remove the warning if ($isParallel) { $stdErr->writeln( sprintf( From 69b64beb2690b4d5944d944b5ad790f9470f9b66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dariusz=20Rumi=C5=84ski?= Date: Wed, 8 May 2024 00:29:37 +0200 Subject: [PATCH 67/77] Reuse existing cache hash to avoid IO read operation * use cache nicely, without recalculation of hash * no need for read-only cache, anymore --- src/Console/Command/WorkerCommand.php | 5 +- src/FixerFileProcessedEvent.php | 17 +++- src/Runner/Parallel/ReadonlyCacheManager.php | 36 -------- src/Runner/Runner.php | 21 ++--- .../Parallel/ReadonlyCacheManagerTest.php | 89 ------------------- 5 files changed, 25 insertions(+), 143 deletions(-) delete mode 100644 src/Runner/Parallel/ReadonlyCacheManager.php delete mode 100644 tests/Runner/Parallel/ReadonlyCacheManagerTest.php diff --git a/src/Console/Command/WorkerCommand.php b/src/Console/Command/WorkerCommand.php index 821f3deb92e..406d37133fb 100644 --- a/src/Console/Command/WorkerCommand.php +++ b/src/Console/Command/WorkerCommand.php @@ -16,6 +16,7 @@ use Clue\React\NDJson\Decoder; use Clue\React\NDJson\Encoder; +use PhpCsFixer\Cache\NullCacheManager; use PhpCsFixer\Config; use PhpCsFixer\Console\ConfigurationResolver; use PhpCsFixer\Error\ErrorsManager; @@ -23,7 +24,6 @@ use PhpCsFixer\Runner\Parallel\ParallelAction; use PhpCsFixer\Runner\Parallel\ParallelConfigFactory; use PhpCsFixer\Runner\Parallel\ParallelisationException; -use PhpCsFixer\Runner\Parallel\ReadonlyCacheManager; use PhpCsFixer\Runner\Runner; use PhpCsFixer\ToolInfoInterface; use React\EventLoop\StreamSelectLoop; @@ -169,6 +169,7 @@ function (ConnectionInterface $connection) use ($loop, $runner, $identifier): vo $out->write([ 'action' => ParallelAction::RUNNER_RESULT, 'file' => $absolutePath, + 'fileHash' => $this->events[0]->getFileHash(), 'status' => $this->events[0]->getStatus(), 'fixInfo' => array_pop($analysisResult), 'errors' => $this->errorsManager->forPath($absolutePath), @@ -230,7 +231,7 @@ private function createRunner(InputInterface $input): Runner $this->errorsManager, $this->configurationResolver->getLinter(), $this->configurationResolver->isDryRun(), - new ReadonlyCacheManager($this->configurationResolver->getCacheManager()), + new NullCacheManager(), // IMPORTANT! We pass null cache, as cache is read&write in main process and we do not need to do it again. $this->configurationResolver->getDirectory(), $this->configurationResolver->shouldStopOnViolation(), ParallelConfigFactory::sequential(), // IMPORTANT! Worker must run in sequential mode. diff --git a/src/FixerFileProcessedEvent.php b/src/FixerFileProcessedEvent.php index b609da49985..c9e9cb9fb9d 100644 --- a/src/FixerFileProcessedEvent.php +++ b/src/FixerFileProcessedEvent.php @@ -39,13 +39,28 @@ final class FixerFileProcessedEvent extends Event private int $status; - public function __construct(int $status) + private ?string $fileRelativePath; + private ?string $fileHash; + + public function __construct(int $status, ?string $fileRelativePath = null, ?string $fileHash = null) { $this->status = $status; + $this->fileRelativePath = $fileRelativePath; + $this->fileHash = $fileHash; } public function getStatus(): int { return $this->status; } + + public function getFileRelativePath(): ?string + { + return $this->fileRelativePath; + } + + public function getFileHash(): ?string + { + return $this->fileHash; + } } diff --git a/src/Runner/Parallel/ReadonlyCacheManager.php b/src/Runner/Parallel/ReadonlyCacheManager.php deleted file mode 100644 index ea6431956fb..00000000000 --- a/src/Runner/Parallel/ReadonlyCacheManager.php +++ /dev/null @@ -1,36 +0,0 @@ - - * Dariusz Rumiński - * - * This source file is subject to the MIT license that is bundled - * with this source code in the file LICENSE. - */ - -namespace PhpCsFixer\Runner\Parallel; - -use PhpCsFixer\Cache\CacheManagerInterface; - -final class ReadonlyCacheManager implements CacheManagerInterface -{ - private CacheManagerInterface $cacheManager; - - public function __construct(CacheManagerInterface $cacheManager) - { - $this->cacheManager = $cacheManager; - } - - public function needFixing(string $file, string $fileContent): bool - { - return $this->cacheManager->needFixing($file, $fileContent); - } - - public function setFile(string $file, string $fileContent): void {} - - public function setFileHash(string $file, string $hash): void {} -} diff --git a/src/Runner/Runner.php b/src/Runner/Runner.php index e3d3e7009a0..25a6afabebf 100644 --- a/src/Runner/Runner.php +++ b/src/Runner/Runner.php @@ -251,23 +251,14 @@ function (array $workerResponse) use ($processPool, $process, $identifier, $getF if (isset($workerResponse['fixInfo'])) { $changed[$fileRelativePath] = $workerResponse['fixInfo']; } - // Dispatch an event for each file processed and dispatch its status (required for progress output) - $this->dispatchEvent(FixerFileProcessedEvent::NAME, new FixerFileProcessedEvent($workerResponse['status'])); - if ( - FixerFileProcessedEvent::STATUS_NO_CHANGES === (int) $workerResponse['status'] - || ( - FixerFileProcessedEvent::STATUS_FIXED === (int) $workerResponse['status'] - && !$this->isDryRun - ) - ) { - $fileContent = file_get_contents($fileAbsolutePath); - - if (false !== $fileContent) { - $this->cacheManager->setFile($fileRelativePath, $fileContent); - } + if (isset($workerResponse['fileHash'])) { + $this->cacheManager->setFileHash($fileRelativePath, $workerResponse['fileHash']); } + // Dispatch an event for each file processed and dispatch its status (required for progress output) + $this->dispatchEvent(FixerFileProcessedEvent::NAME, new FixerFileProcessedEvent($workerResponse['status'])); + // Worker requests for another file chunk when all files were processed foreach ($workerResponse['errors'] ?? [] as $workerError) { $this->errorsManager->report(new Error( @@ -529,7 +520,7 @@ private function fixFile(\SplFileInfo $file, LintingResultInterface $lintingResu $this->dispatchEvent( FixerFileProcessedEvent::NAME, - new FixerFileProcessedEvent(null !== $fixInfo ? FixerFileProcessedEvent::STATUS_FIXED : FixerFileProcessedEvent::STATUS_NO_CHANGES) + new FixerFileProcessedEvent(null !== $fixInfo ? FixerFileProcessedEvent::STATUS_FIXED : FixerFileProcessedEvent::STATUS_NO_CHANGES, $name, $newHash) ); return $fixInfo; diff --git a/tests/Runner/Parallel/ReadonlyCacheManagerTest.php b/tests/Runner/Parallel/ReadonlyCacheManagerTest.php deleted file mode 100644 index e4c9c251851..00000000000 --- a/tests/Runner/Parallel/ReadonlyCacheManagerTest.php +++ /dev/null @@ -1,89 +0,0 @@ - - * Dariusz Rumiński - * - * This source file is subject to the MIT license that is bundled - * with this source code in the file LICENSE. - */ - -namespace PhpCsFixer\Tests\Runner\Parallel; - -use PhpCsFixer\Cache\CacheManagerInterface; -use PhpCsFixer\Runner\Parallel\ReadonlyCacheManager; -use PhpCsFixer\Tests\TestCase; - -/** - * @internal - * - * @covers \PhpCsFixer\Runner\Parallel\ReadonlyCacheManager - */ -final class ReadonlyCacheManagerTest extends TestCase -{ - /** - * @dataProvider provideNeedFixingCases - */ - public function testNeedFixing(bool $needsFixing): void - { - $cacheManager = new ReadonlyCacheManager($this->getInnerCacheManager($needsFixing)); - - self::assertSame($needsFixing, $cacheManager->needFixing('foo.php', 'getInnerCacheManager(false)); - $cacheManager->setFile('foo.php', ' - */ - public static function provideNeedFixingCases(): iterable - { - yield [true]; - - yield [false]; - } - - private function getInnerCacheManager(bool $needsFixing): CacheManagerInterface - { - return new class($needsFixing) implements CacheManagerInterface { - private bool $needsFixing; - - public function __construct(bool $needsFixing) - { - $this->needsFixing = $needsFixing; - } - - public function needFixing(string $file, string $fileContent): bool - { - return $this->needsFixing; - } - - public function setFile(string $file, string $fileContent): void - { - throw new \LogicException('Should not be called.'); - } - - public function setFileHash(string $file, string $hash): void - { - throw new \LogicException('Should not be called.'); - } - }; - } -} From f347f13327dc9bd688d8e61e5680abcff39d1f5f Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Wed, 8 May 2024 00:45:36 +0200 Subject: [PATCH 68/77] Fix "well-defined arrays" CI step --- src/Console/Output/ErrorOutput.php | 2 +- src/Runner/Parallel/Process.php | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Console/Output/ErrorOutput.php b/src/Console/Output/ErrorOutput.php index 168d6719705..138645b06a5 100644 --- a/src/Console/Output/ErrorOutput.php +++ b/src/Console/Output/ErrorOutput.php @@ -120,7 +120,7 @@ public function listErrors(string $process, array $errors): void } /** - * @param array $errors + * @param list $errors */ public function listWorkerErrors(array $errors): void { diff --git a/src/Runner/Parallel/Process.php b/src/Runner/Parallel/Process.php index 0376df47839..9958d767530 100644 --- a/src/Runner/Parallel/Process.php +++ b/src/Runner/Parallel/Process.php @@ -47,7 +47,7 @@ final class Process /** @var resource */ private $stdOut; - /** @var callable(mixed[]): void */ + /** @var callable(array): void */ private $onData; /** @var callable(\Throwable): void */ @@ -63,7 +63,7 @@ public function __construct(string $command, LoopInterface $loop, int $timeoutSe } /** - * @param callable(mixed[] $json): void $onData callback to be called when data is received from the parallelisation operator + * @param callable(array $json): void $onData callback to be called when data is received from the parallelisation operator * @param callable(\Throwable $exception): void $onError callback to be called when an exception occurs * @param callable(?int $exitCode, string $output): void $onExit callback to be called when the process exits */ @@ -115,7 +115,7 @@ public function start(callable $onData, callable $onError, callable $onExit): vo /** * Handles requests from parallelisation operator to its worker (spawned process). * - * @param mixed[] $data + * @param array $data */ public function request(array $data): void { From 077fd6b6421e7bc88d31b4aba77f1f17e82cff36 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Wed, 8 May 2024 00:56:55 +0200 Subject: [PATCH 69/77] Add explanation why some tests are skipped on Windows --- tests/Console/Command/WorkerCommandTest.php | 4 ++++ tests/Runner/Parallel/ProcessFactoryTest.php | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/tests/Console/Command/WorkerCommandTest.php b/tests/Console/Command/WorkerCommandTest.php index 3e688594840..0b1e0e42e92 100644 --- a/tests/Console/Command/WorkerCommandTest.php +++ b/tests/Console/Command/WorkerCommandTest.php @@ -76,6 +76,10 @@ public function testWorkerCantConnectToServerWhenExecutedDirectly(): void } /** + * This test is not executed on Windows because process pipes are not supported there, due to their blocking nature + * on this particular OS. The cause of this lays in `react/child-process` component, but it's related only to tests, + * as parallel runner works properly on Windows too. Feel free to fiddle with it and add testing support for Windows. + * * @requires OS Linux|Darwin */ public function testWorkerCommunicatesWithTheServer(): void diff --git a/tests/Runner/Parallel/ProcessFactoryTest.php b/tests/Runner/Parallel/ProcessFactoryTest.php index b49e73fbe1f..8e561c2e539 100644 --- a/tests/Runner/Parallel/ProcessFactoryTest.php +++ b/tests/Runner/Parallel/ProcessFactoryTest.php @@ -51,6 +51,10 @@ protected function setUp(): void } /** + * This test is not executed on Windows because process pipes are not supported there, due to their blocking nature + * on this particular OS. The cause of this lays in `react/child-process` component, but it's related only to tests, + * as parallel runner works properly on Windows too. Feel free to fiddle with it and add testing support for Windows. + * * @requires OS Linux|Darwin * * @param array $input From b7ac50dab7c93c64ab10d17739ea5d697d172b61 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Wed, 8 May 2024 01:08:40 +0200 Subject: [PATCH 70/77] Clean up loop that re-reports errors from worker to main process' error manager - Remove invalid comment (no idea why it was here) - rename loop's variable to not confuse it with `WorkerError` class that represents unrecoverable errors on worker side --- src/Runner/Runner.php | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/Runner/Runner.php b/src/Runner/Runner.php index 25a6afabebf..1b66d4ea19c 100644 --- a/src/Runner/Runner.php +++ b/src/Runner/Runner.php @@ -259,16 +259,15 @@ function (array $workerResponse) use ($processPool, $process, $identifier, $getF // Dispatch an event for each file processed and dispatch its status (required for progress output) $this->dispatchEvent(FixerFileProcessedEvent::NAME, new FixerFileProcessedEvent($workerResponse['status'])); - // Worker requests for another file chunk when all files were processed - foreach ($workerResponse['errors'] ?? [] as $workerError) { + foreach ($workerResponse['errors'] ?? [] as $error) { $this->errorsManager->report(new Error( - $workerError['type'], - $workerError['filePath'], - null !== $workerError['source'] - ? ParallelisationException::forWorkerError($workerError['source']) + $error['type'], + $error['filePath'], + null !== $error['source'] + ? ParallelisationException::forWorkerError($error['source']) : null, - $workerError['appliedFixers'], - $workerError['diff'] + $error['appliedFixers'], + $error['diff'] )); } From 5a0cc3e187b092c86d55adb3d365b4205bfa5982 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Wed, 8 May 2024 01:15:40 +0200 Subject: [PATCH 71/77] Workaround for `ParallelConfigFactory` testability around CPU detection --- src/Runner/Parallel/ParallelConfigFactory.php | 14 +++++++++----- .../Parallel/ParallelConfigFactoryTest.php | 17 +++++++++++++++-- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/Runner/Parallel/ParallelConfigFactory.php b/src/Runner/Parallel/ParallelConfigFactory.php index 01c44a0292b..cd6a56be63d 100644 --- a/src/Runner/Parallel/ParallelConfigFactory.php +++ b/src/Runner/Parallel/ParallelConfigFactory.php @@ -23,6 +23,8 @@ */ final class ParallelConfigFactory { + private static ?CpuCoreCounter $cpuDetector = null; + private function __construct() {} public static function sequential(): ParallelConfig @@ -38,14 +40,16 @@ public static function detect( ?int $filesPerProcess = null, ?int $processTimeout = null ): ParallelConfig { - $counter = new CpuCoreCounter([ - ...FinderRegistry::getDefaultLogicalFinders(), - new DummyCpuCoreFinder(1), - ]); + if (null === self::$cpuDetector) { + self::$cpuDetector = new CpuCoreCounter([ + ...FinderRegistry::getDefaultLogicalFinders(), + new DummyCpuCoreFinder(1), + ]); + } return new ParallelConfig( ...array_filter( - [$counter->getCount(), $filesPerProcess, $processTimeout], + [self::$cpuDetector->getCount(), $filesPerProcess, $processTimeout], static fn ($value): bool => null !== $value ) ); diff --git a/tests/Runner/Parallel/ParallelConfigFactoryTest.php b/tests/Runner/Parallel/ParallelConfigFactoryTest.php index 0fa0f72c80c..720b7fa8b6a 100644 --- a/tests/Runner/Parallel/ParallelConfigFactoryTest.php +++ b/tests/Runner/Parallel/ParallelConfigFactoryTest.php @@ -14,6 +14,8 @@ namespace PhpCsFixer\Tests\Runner\Parallel; +use Fidry\CpuCoreCounter\CpuCoreCounter; +use Fidry\CpuCoreCounter\Finder\DummyCpuCoreFinder; use PhpCsFixer\Runner\Parallel\ParallelConfigFactory; use PhpCsFixer\Tests\TestCase; @@ -21,8 +23,6 @@ * @internal * * @covers \PhpCsFixer\Runner\Parallel\ParallelConfigFactory - * - * @TODO Test `detect()` method, but first discuss the best way to do it. */ final class ParallelConfigFactoryTest extends TestCase { @@ -33,12 +33,25 @@ public function testSequentialConfigHasExactlyOneProcess(): void self::assertSame(1, $config->getMaxProcesses()); } + /** + * @see https://github.com/PHP-CS-Fixer/PHP-CS-Fixer/pull/7777#discussion_r1591623367 + */ public function testDetectConfigurationWithoutParams(): void { + $parallelConfigFactoryReflection = new \ReflectionClass(ParallelConfigFactory::class); + $cpuDetector = $parallelConfigFactoryReflection->getProperty('cpuDetector'); + $cpuDetector->setAccessible(true); + $cpuDetector->setValue($parallelConfigFactoryReflection, new CpuCoreCounter([ + new DummyCpuCoreFinder(7), + ])); + $config = ParallelConfigFactory::detect(); + self::assertSame(7, $config->getMaxProcesses()); self::assertSame(10, $config->getFilesPerProcess()); self::assertSame(120, $config->getProcessTimeout()); + + $cpuDetector->setValue($parallelConfigFactoryReflection, null); } public function testDetectConfigurationWithParams(): void From 50695498e0bf9a5e1b4a7578c44a6cdb02fb0420 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Wed, 8 May 2024 01:29:48 +0200 Subject: [PATCH 72/77] Flip naming convention for `ParallelAction` consts and improve them It's actually more natural to name these actions after initiating side - when we're writing to the output, we should use action related to the place where we're writing, and when we're reading the message, action should point to the callee. See: https://github.com/PHP-CS-Fixer/PHP-CS-Fixer/pull/7777#discussion_r1592635801 --- src/Console/Command/WorkerCommand.php | 12 ++++++------ src/Runner/Parallel/ParallelAction.php | 16 ++++++++-------- src/Runner/Runner.php | 16 ++++++++-------- tests/Console/Command/WorkerCommandTest.php | 12 ++++++------ 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/src/Console/Command/WorkerCommand.php b/src/Console/Command/WorkerCommand.php index 406d37133fb..9113685d752 100644 --- a/src/Console/Command/WorkerCommand.php +++ b/src/Console/Command/WorkerCommand.php @@ -118,11 +118,11 @@ function (ConnectionInterface $connection) use ($loop, $runner, $identifier): vo $in = new Decoder($connection, true, 512, $jsonInvalidUtf8Ignore); // [REACT] Initialise connection with the parallelisation operator - $out->write(['action' => ParallelAction::RUNNER_HELLO, 'identifier' => $identifier]); + $out->write(['action' => ParallelAction::WORKER_HELLO, 'identifier' => $identifier]); $handleError = static function (\Throwable $error) use ($out): void { $out->write([ - 'action' => ParallelAction::RUNNER_ERROR_REPORT, + 'action' => ParallelAction::WORKER_ERROR_REPORT, 'message' => $error->getMessage(), 'file' => $error->getFile(), 'line' => $error->getLine(), @@ -138,13 +138,13 @@ function (ConnectionInterface $connection) use ($loop, $runner, $identifier): vo $action = $json['action'] ?? null; // Parallelisation operator does not have more to do, let's close the connection - if (ParallelAction::WORKER_THANK_YOU === $action) { + if (ParallelAction::RUNNER_THANK_YOU === $action) { $loop->stop(); return; } - if (ParallelAction::WORKER_RUN !== $action) { + if (ParallelAction::RUNNER_REQUEST_ANALYSIS !== $action) { // At this point we only expect analysis requests, if any other action happen, we need to fix the code. throw new \LogicException(sprintf('Unexpected action ParallelAction::%s.', $action)); } @@ -167,7 +167,7 @@ function (ConnectionInterface $connection) use ($loop, $runner, $identifier): vo } $out->write([ - 'action' => ParallelAction::RUNNER_RESULT, + 'action' => ParallelAction::WORKER_RESULT, 'file' => $absolutePath, 'fileHash' => $this->events[0]->getFileHash(), 'status' => $this->events[0]->getStatus(), @@ -177,7 +177,7 @@ function (ConnectionInterface $connection) use ($loop, $runner, $identifier): vo } // Request another file chunk (if available, the parallelisation operator will request new "run" action) - $out->write(['action' => ParallelAction::RUNNER_GET_FILE_CHUNK]); + $out->write(['action' => ParallelAction::WORKER_GET_FILE_CHUNK]); }); }, static function (\Throwable $error) use ($errorOutput): void { diff --git a/src/Runner/Parallel/ParallelAction.php b/src/Runner/Parallel/ParallelAction.php index c08db60ed57..42585ee28e7 100644 --- a/src/Runner/Parallel/ParallelAction.php +++ b/src/Runner/Parallel/ParallelAction.php @@ -21,15 +21,15 @@ */ final class ParallelAction { - // Actions handled by the runner - public const RUNNER_ERROR_REPORT = 'errorReport'; - public const RUNNER_HELLO = 'hello'; - public const RUNNER_RESULT = 'result'; - public const RUNNER_GET_FILE_CHUNK = 'getFileChunk'; + // Actions executed by the runner (main process) + public const RUNNER_REQUEST_ANALYSIS = 'requestAnalysis'; + public const RUNNER_THANK_YOU = 'thankYou'; - // Actions handled by the worker - public const WORKER_RUN = 'run'; - public const WORKER_THANK_YOU = 'thankYou'; + // Actions executed by the worker + public const WORKER_ERROR_REPORT = 'errorReport'; + public const WORKER_GET_FILE_CHUNK = 'getFileChunk'; + public const WORKER_HELLO = 'hello'; + public const WORKER_RESULT = 'result'; private function __construct() {} } diff --git a/src/Runner/Runner.php b/src/Runner/Runner.php index 1b66d4ea19c..8af02a90184 100644 --- a/src/Runner/Runner.php +++ b/src/Runner/Runner.php @@ -199,7 +199,7 @@ private function fixParallel(): array // [REACT] Bind connection when worker's process requests "hello" action (enables 2-way communication) $decoder->on('data', static function (array $data) use ($processPool, $getFileChunk, $decoder, $encoder): void { - if (ParallelAction::RUNNER_HELLO !== $data['action']) { + if (ParallelAction::WORKER_HELLO !== $data['action']) { return; } @@ -209,13 +209,13 @@ private function fixParallel(): array $fileChunk = $getFileChunk(); if (0 === \count($fileChunk)) { - $process->request(['action' => ParallelAction::WORKER_THANK_YOU]); + $process->request(['action' => ParallelAction::RUNNER_THANK_YOU]); $processPool->endProcessIfKnown($identifier); return; } - $process->request(['action' => ParallelAction::WORKER_RUN, 'files' => $fileChunk]); + $process->request(['action' => ParallelAction::RUNNER_REQUEST_ANALYSIS, 'files' => $fileChunk]); }); }); @@ -243,7 +243,7 @@ private function fixParallel(): array // [REACT] Handle workers' responses (multiple actions possible) function (array $workerResponse) use ($processPool, $process, $identifier, $getFileChunk, &$changed): void { // File analysis result (we want close-to-realtime progress with frequent cache savings) - if (ParallelAction::RUNNER_RESULT === $workerResponse['action']) { + if (ParallelAction::WORKER_RESULT === $workerResponse['action']) { $fileAbsolutePath = $workerResponse['file']; $fileRelativePath = $this->directory->getRelativePathTo($fileAbsolutePath); @@ -274,23 +274,23 @@ function (array $workerResponse) use ($processPool, $process, $identifier, $getF return; } - if (ParallelAction::RUNNER_GET_FILE_CHUNK === $workerResponse['action']) { + if (ParallelAction::WORKER_GET_FILE_CHUNK === $workerResponse['action']) { // Request another chunk of files, if still available $fileChunk = $getFileChunk(); if (0 === \count($fileChunk)) { - $process->request(['action' => ParallelAction::WORKER_THANK_YOU]); + $process->request(['action' => ParallelAction::RUNNER_THANK_YOU]); $processPool->endProcessIfKnown($identifier); return; } - $process->request(['action' => ParallelAction::WORKER_RUN, 'files' => $fileChunk]); + $process->request(['action' => ParallelAction::RUNNER_REQUEST_ANALYSIS, 'files' => $fileChunk]); return; } - if (ParallelAction::RUNNER_ERROR_REPORT === $workerResponse['action']) { + if (ParallelAction::WORKER_ERROR_REPORT === $workerResponse['action']) { $this->errorsManager->reportWorkerError(new WorkerError( $workerResponse['message'], $workerResponse['file'], diff --git a/tests/Console/Command/WorkerCommandTest.php b/tests/Console/Command/WorkerCommandTest.php index 0b1e0e42e92..04ade0f6a6f 100644 --- a/tests/Console/Command/WorkerCommandTest.php +++ b/tests/Console/Command/WorkerCommandTest.php @@ -127,8 +127,8 @@ static function (array $data) use ($encoder, &$workerScope): void { $workerScope['messages'][] = $data; $ds = \DIRECTORY_SEPARATOR; - if (ParallelAction::RUNNER_HELLO === $data['action']) { - $encoder->write(['action' => ParallelAction::WORKER_RUN, 'files' => [ + if (ParallelAction::WORKER_HELLO === $data['action']) { + $encoder->write(['action' => ParallelAction::RUNNER_REQUEST_ANALYSIS, 'files' => [ realpath(__DIR__.$ds.'..'.$ds.'..').$ds.'Fixtures'.$ds.'FixerTest'.$ds.'fix'.$ds.'somefile.php', ]]); @@ -136,7 +136,7 @@ static function (array $data) use ($encoder, &$workerScope): void { } if (3 === \count($workerScope['messages'])) { - $encoder->write(['action' => ParallelAction::WORKER_THANK_YOU]); + $encoder->write(['action' => ParallelAction::RUNNER_THANK_YOU]); } } ); @@ -152,10 +152,10 @@ static function (array $data) use ($encoder, &$workerScope): void { self::assertSame(Command::SUCCESS, $process->getExitCode()); self::assertCount(3, $workerScope['messages']); - self::assertSame(ParallelAction::RUNNER_HELLO, $workerScope['messages'][0]['action']); - self::assertSame(ParallelAction::RUNNER_RESULT, $workerScope['messages'][1]['action']); + self::assertSame(ParallelAction::WORKER_HELLO, $workerScope['messages'][0]['action']); + self::assertSame(ParallelAction::WORKER_RESULT, $workerScope['messages'][1]['action']); self::assertSame(FixerFileProcessedEvent::STATUS_FIXED, $workerScope['messages'][1]['status']); - self::assertSame(ParallelAction::RUNNER_GET_FILE_CHUNK, $workerScope['messages'][2]['action']); + self::assertSame(ParallelAction::WORKER_GET_FILE_CHUNK, $workerScope['messages'][2]['action']); $server->close(); } From d613aded70f54995de93b0eeaef50f5a5f573649 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Wed, 8 May 2024 09:34:31 +0200 Subject: [PATCH 73/77] Explicit `->toString()` conversion for process identifiers See: https://github.com/Wirone/PHP-CS-Fixer/pull/5#discussion_r1591588344 --- src/Runner/Parallel/ParallelisationException.php | 2 +- src/Runner/Parallel/ProcessFactory.php | 2 +- src/Runner/Parallel/ProcessIdentifier.php | 4 ++-- src/Runner/Parallel/ProcessPool.php | 10 +++++----- tests/Console/Command/WorkerCommandTest.php | 6 +++--- tests/Runner/Parallel/ProcessFactoryTest.php | 2 +- tests/Runner/Parallel/ProcessIdentifierTest.php | 4 ++-- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/Runner/Parallel/ParallelisationException.php b/src/Runner/Parallel/ParallelisationException.php index 28a4b8ab9b7..4b7d2842698 100644 --- a/src/Runner/Parallel/ParallelisationException.php +++ b/src/Runner/Parallel/ParallelisationException.php @@ -25,7 +25,7 @@ final class ParallelisationException extends \RuntimeException { public static function forUnknownIdentifier(ProcessIdentifier $identifier): self { - return new self('Unknown process identifier: '.(string) $identifier); + return new self('Unknown process identifier: '.$identifier->toString()); } /** diff --git a/src/Runner/Parallel/ProcessFactory.php b/src/Runner/Parallel/ProcessFactory.php index d737b5c3880..64de299f93b 100644 --- a/src/Runner/Parallel/ProcessFactory.php +++ b/src/Runner/Parallel/ProcessFactory.php @@ -80,7 +80,7 @@ public function getCommandArgs(int $serverPort, ProcessIdentifier $identifier, R '--port', (string) $serverPort, '--identifier', - escapeshellarg((string) $identifier), + escapeshellarg($identifier->toString()), ]; if ($runnerConfig->isDryRun()) { diff --git a/src/Runner/Parallel/ProcessIdentifier.php b/src/Runner/Parallel/ProcessIdentifier.php index 03bb11df433..469c665d582 100644 --- a/src/Runner/Parallel/ProcessIdentifier.php +++ b/src/Runner/Parallel/ProcessIdentifier.php @@ -21,7 +21,7 @@ * * @internal */ -final class ProcessIdentifier implements \Stringable +final class ProcessIdentifier { private const IDENTIFIER_PREFIX = 'php-cs-fixer_parallel_'; @@ -32,7 +32,7 @@ private function __construct(string $identifier) $this->identifier = $identifier; } - public function __toString(): string + public function toString(): string { return $this->identifier; } diff --git a/src/Runner/Parallel/ProcessPool.php b/src/Runner/Parallel/ProcessPool.php index d48f9edcd18..ef4bcf3802c 100644 --- a/src/Runner/Parallel/ProcessPool.php +++ b/src/Runner/Parallel/ProcessPool.php @@ -45,21 +45,21 @@ public function __construct(ServerInterface $server, ?callable $onServerClose = public function getProcess(ProcessIdentifier $identifier): Process { - if (!isset($this->processes[(string) $identifier])) { + if (!isset($this->processes[$identifier->toString()])) { throw ParallelisationException::forUnknownIdentifier($identifier); } - return $this->processes[(string) $identifier]; + return $this->processes[$identifier->toString()]; } public function addProcess(ProcessIdentifier $identifier, Process $process): void { - $this->processes[(string) $identifier] = $process; + $this->processes[$identifier->toString()] = $process; } public function endProcessIfKnown(ProcessIdentifier $identifier): void { - if (!isset($this->processes[(string) $identifier])) { + if (!isset($this->processes[$identifier->toString()])) { return; } @@ -77,7 +77,7 @@ private function endProcess(ProcessIdentifier $identifier): void { $this->getProcess($identifier)->quit(); - unset($this->processes[(string) $identifier]); + unset($this->processes[$identifier->toString()]); if (0 === \count($this->processes)) { $this->server->close(); diff --git a/tests/Console/Command/WorkerCommandTest.php b/tests/Console/Command/WorkerCommandTest.php index 04ade0f6a6f..049a238f062 100644 --- a/tests/Console/Command/WorkerCommandTest.php +++ b/tests/Console/Command/WorkerCommandTest.php @@ -59,13 +59,13 @@ public function testMissingPortCausesFailure(): void self::expectException(ParallelisationException::class); self::expectExceptionMessage('Missing parallelisation options'); - $commandTester = $this->doTestExecute(['--identifier' => (string) ProcessIdentifier::create()]); + $commandTester = $this->doTestExecute(['--identifier' => ProcessIdentifier::create()->toString()]); } public function testWorkerCantConnectToServerWhenExecutedDirectly(): void { $commandTester = $this->doTestExecute([ - '--identifier' => (string) ProcessIdentifier::create(), + '--identifier' => ProcessIdentifier::create()->toString(), '--port' => 12_345, ]); @@ -107,7 +107,7 @@ public function testWorkerCommunicatesWithTheServer(): void * } $workerScope */ $workerScope = [ - 'identifier' => (string) $processIdentifier, + 'identifier' => $processIdentifier->toString(), 'messages' => [], 'connected' => false, 'chunkRequested' => false, diff --git a/tests/Runner/Parallel/ProcessFactoryTest.php b/tests/Runner/Parallel/ProcessFactoryTest.php index 8e561c2e539..32f583f92cc 100644 --- a/tests/Runner/Parallel/ProcessFactoryTest.php +++ b/tests/Runner/Parallel/ProcessFactoryTest.php @@ -80,7 +80,7 @@ public function testCreate(array $input, RunnerConfig $config, string $expectedA trim( sprintf( 'worker --port 1234 --identifier \'%s\' %s', - (string) $identifier, + $identifier->toString(), trim($expectedAdditionalArgs) ) ), diff --git a/tests/Runner/Parallel/ProcessIdentifierTest.php b/tests/Runner/Parallel/ProcessIdentifierTest.php index c3d6450dbad..c90c0951c17 100644 --- a/tests/Runner/Parallel/ProcessIdentifierTest.php +++ b/tests/Runner/Parallel/ProcessIdentifierTest.php @@ -29,7 +29,7 @@ public function testCreateIdentifier(): void { $identifier = ProcessIdentifier::create(); - self::assertStringStartsWith('php-cs-fixer_parallel_', (string) $identifier); + self::assertStringStartsWith('php-cs-fixer_parallel_', $identifier->toString()); } /** @@ -42,7 +42,7 @@ public function testFromRaw(string $rawIdentifier, bool $valid): void } $identifier = ProcessIdentifier::fromRaw($rawIdentifier); - self::assertSame($rawIdentifier, (string) $identifier); + self::assertSame($rawIdentifier, $identifier->toString()); } /** From 7aa838b635fb9f9d0ed7b670b818ec7b9e7782c5 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Wed, 8 May 2024 10:30:21 +0200 Subject: [PATCH 74/77] Support for `--stop-on-violation` in parallel mode --- src/Console/Command/WorkerCommand.php | 3 +- src/Runner/Parallel/ProcessFactory.php | 4 ++ src/Runner/Parallel/ProcessPool.php | 2 +- src/Runner/Runner.php | 23 +++++++--- tests/Runner/Parallel/ProcessFactoryTest.php | 5 ++- tests/Runner/RunnerTest.php | 47 ++++++++++++++++++++ 6 files changed, 74 insertions(+), 10 deletions(-) diff --git a/src/Console/Command/WorkerCommand.php b/src/Console/Command/WorkerCommand.php index 9113685d752..4b4e2964c74 100644 --- a/src/Console/Command/WorkerCommand.php +++ b/src/Console/Command/WorkerCommand.php @@ -86,6 +86,7 @@ protected function configure(): void new InputOption('using-cache', '', InputOption::VALUE_REQUIRED, 'Should cache be used (can be `yes` or `no`).'), new InputOption('cache-file', '', InputOption::VALUE_REQUIRED, 'The path to the cache file.'), new InputOption('diff', '', InputOption::VALUE_NONE, 'Prints diff for each file.'), + new InputOption('stop-on-violation', '', InputOption::VALUE_NONE, 'Stop execution on first violation.'), ] ); } @@ -217,7 +218,7 @@ private function createRunner(InputInterface $input): Runner 'using-cache' => $input->getOption('using-cache'), 'cache-file' => $input->getOption('cache-file'), 'diff' => $input->getOption('diff'), - 'stop-on-violation' => false, // @TODO Pass this option to the runner + 'stop-on-violation' => $input->getOption('stop-on-violation'), ], getcwd(), // @phpstan-ignore-line $this->toolInfo diff --git a/src/Runner/Parallel/ProcessFactory.php b/src/Runner/Parallel/ProcessFactory.php index 64de299f93b..af37e05890d 100644 --- a/src/Runner/Parallel/ProcessFactory.php +++ b/src/Runner/Parallel/ProcessFactory.php @@ -91,6 +91,10 @@ public function getCommandArgs(int $serverPort, ProcessIdentifier $identifier, R $commandArgs[] = '--diff'; } + if (filter_var($this->input->getOption('stop-on-violation'), FILTER_VALIDATE_BOOLEAN)) { + $commandArgs[] = '--stop-on-violation'; + } + foreach (['allow-risky', 'config', 'rules', 'using-cache', 'cache-file'] as $option) { $optionValue = $this->input->getOption($option); diff --git a/src/Runner/Parallel/ProcessPool.php b/src/Runner/Parallel/ProcessPool.php index ef4bcf3802c..fe13d0faa09 100644 --- a/src/Runner/Parallel/ProcessPool.php +++ b/src/Runner/Parallel/ProcessPool.php @@ -69,7 +69,7 @@ public function endProcessIfKnown(ProcessIdentifier $identifier): void public function endAll(): void { foreach (array_keys($this->processes) as $identifier) { - $this->endProcess(ProcessIdentifier::fromRaw($identifier)); + $this->endProcessIfKnown(ProcessIdentifier::fromRaw($identifier)); } } diff --git a/src/Runner/Runner.php b/src/Runner/Runner.php index 8af02a90184..d88f1b7a88d 100644 --- a/src/Runner/Runner.php +++ b/src/Runner/Runner.php @@ -247,18 +247,16 @@ function (array $workerResponse) use ($processPool, $process, $identifier, $getF $fileAbsolutePath = $workerResponse['file']; $fileRelativePath = $this->directory->getRelativePathTo($fileAbsolutePath); - // Pass-back information about applied changes (only if there are any) - if (isset($workerResponse['fixInfo'])) { - $changed[$fileRelativePath] = $workerResponse['fixInfo']; - } + // Dispatch an event for each file processed and dispatch its status (required for progress output) + $this->dispatchEvent( + FixerFileProcessedEvent::NAME, + new FixerFileProcessedEvent($workerResponse['status']) + ); if (isset($workerResponse['fileHash'])) { $this->cacheManager->setFileHash($fileRelativePath, $workerResponse['fileHash']); } - // Dispatch an event for each file processed and dispatch its status (required for progress output) - $this->dispatchEvent(FixerFileProcessedEvent::NAME, new FixerFileProcessedEvent($workerResponse['status'])); - foreach ($workerResponse['errors'] ?? [] as $error) { $this->errorsManager->report(new Error( $error['type'], @@ -271,6 +269,17 @@ function (array $workerResponse) use ($processPool, $process, $identifier, $getF )); } + // Pass-back information about applied changes (only if there are any) + if (isset($workerResponse['fixInfo'])) { + $changed[$fileRelativePath] = $workerResponse['fixInfo']; + + if ($this->stopOnViolation) { + $processPool->endAll(); + + return; + } + } + return; } diff --git a/tests/Runner/Parallel/ProcessFactoryTest.php b/tests/Runner/Parallel/ProcessFactoryTest.php index 32f583f92cc..c20949e99ee 100644 --- a/tests/Runner/Parallel/ProcessFactoryTest.php +++ b/tests/Runner/Parallel/ProcessFactoryTest.php @@ -105,6 +105,8 @@ public static function provideCreateCases(): iterable yield 'diff enabled' => [['--diff' => true], self::createRunnerConfig(false), '--diff']; + yield 'stop-on-violation enabled' => [['--stop-on-violation' => true], self::createRunnerConfig(false), '--stop-on-violation']; + yield 'allow risky' => [['--allow-risky' => 'yes'], self::createRunnerConfig(false), '--allow-risky \'yes\'']; yield 'config' => [['--config' => 'foo.php'], self::createRunnerConfig(false), '--config \'foo.php\'']; @@ -124,9 +126,10 @@ public static function provideCreateCases(): iterable '--config' => 'conf.php', '--diff' => true, '--using-cache' => 'yes', + '--stop-on-violation' => true, ], self::createRunnerConfig(true), - '--dry-run --diff --config \'conf.php\' --using-cache \'yes\'', + '--dry-run --diff --stop-on-violation --config \'conf.php\' --using-cache \'yes\'', ]; } diff --git a/tests/Runner/RunnerTest.php b/tests/Runner/RunnerTest.php index ea6887bf5dd..57eaf90326a 100644 --- a/tests/Runner/RunnerTest.php +++ b/tests/Runner/RunnerTest.php @@ -181,6 +181,53 @@ public function testThatParallelFixOfInvalidFileReportsToErrorManager(): void self::assertSame($pathToInvalidFile, $error->getFilePath()); } + /** + * @requires OS Darwin|Windows + * + * @TODO v4 do not switch on parallel execution by default while this test is not passing on Linux. + * + * @covers \PhpCsFixer\Runner\Runner::fix + * @covers \PhpCsFixer\Runner\Runner::fixFile + * @covers \PhpCsFixer\Runner\Runner::fixParallel + * + * @dataProvider provideParallelFixStopsOnFirstViolationIfSuchOptionIsEnabledCases + */ + public function testParallelFixStopsOnFirstViolationIfSuchOptionIsEnabled(bool $stopOnViolation, int $expectedChanges): void + { + $errorsManager = new ErrorsManager(); + + $path = realpath(__DIR__.\DIRECTORY_SEPARATOR.'..').\DIRECTORY_SEPARATOR.'Fixtures'.\DIRECTORY_SEPARATOR.'FixerTest'.\DIRECTORY_SEPARATOR.'fix'; + $runner = new Runner( + Finder::create()->in($path), + [ + new Fixer\ClassNotation\VisibilityRequiredFixer(), + new Fixer\Import\NoUnusedImportsFixer(), // will be ignored cause of test keyword in namespace + ], + new NullDiffer(), + null, + $errorsManager, + new Linter(), + true, + new NullCacheManager(), + null, + $stopOnViolation, + new ParallelConfig(2, 1, 3), + new ArrayInput([], (new FixCommand(new ToolInfo()))->getDefinition()) + ); + + self::assertCount($expectedChanges, $runner->fix()); + } + + /** + * @return iterable + */ + public static function provideParallelFixStopsOnFirstViolationIfSuchOptionIsEnabledCases(): iterable + { + yield 'do NOT stop on violation' => [false, 2]; + + yield 'stop on violation' => [true, 1]; + } + /** * @covers \PhpCsFixer\Runner\Runner::fix * @covers \PhpCsFixer\Runner\Runner::fixFile From f63b4d5526ea3748f0b352aaad557fac3972ab85 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Tue, 14 May 2024 00:14:21 +0200 Subject: [PATCH 75/77] Rework parallel-related warnings displayed during analysis --- src/Console/Command/FixCommand.php | 21 +++++++++++++-------- tests/Console/Command/FixCommandTest.php | 2 ++ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/Console/Command/FixCommand.php b/src/Console/Command/FixCommand.php index a25f844052c..b8246488353 100644 --- a/src/Console/Command/FixCommand.php +++ b/src/Console/Command/FixCommand.php @@ -30,6 +30,7 @@ use PhpCsFixer\ToolInfoInterface; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Formatter\OutputFormatter; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; @@ -272,15 +273,19 @@ protected function execute(InputInterface $input, OutputInterface $output): int ) : ' sequentially' )); - // @TODO v4 remove the warning - if ($isParallel) { - $stdErr->writeln( - sprintf( - $stdErr->isDecorated() ? '%s' : '%s', - 'Parallel runner is an experimental feature and may be unstable, use it at your own risk. Feedback highly appreciated!' + /** @TODO v4 remove warnings related to parallel runner */ + $usageDocs = 'https://cs.symfony.com/doc/usage.html'; + $stdErr->writeln(sprintf( + $stdErr->isDecorated() ? '%s' : '%s', + $isParallel + ? 'Parallel runner is an experimental feature and may be unstable, use it at your own risk. Feedback highly appreciated!' + : sprintf( + 'You can enable parallel runner and speed up the analysis! Please see %s for more information.', + $stdErr->isDecorated() + ? sprintf('usage docs', OutputFormatter::escape($usageDocs)) + : $usageDocs ) - ); - } + )); $configFile = $resolver->getConfigFile(); $stdErr->writeln(sprintf('Loaded config %s%s.', $resolver->getConfig()->getName(), null === $configFile ? '' : ' from "'.$configFile.'"')); diff --git a/tests/Console/Command/FixCommandTest.php b/tests/Console/Command/FixCommandTest.php index 47b32772097..75dfdfb3317 100644 --- a/tests/Console/Command/FixCommandTest.php +++ b/tests/Console/Command/FixCommandTest.php @@ -99,6 +99,7 @@ public function testSequentialRun(): void ); self::assertStringContainsString('Running analysis on 1 core sequentially.', $cmdTester->getDisplay()); + self::assertStringContainsString('You can enable parallel runner and speed up the analysis!', $cmdTester->getDisplay()); self::assertStringContainsString('(header_comment)', $cmdTester->getDisplay()); self::assertSame(8, $cmdTester->getStatusCode()); } @@ -134,6 +135,7 @@ public function testParallelRun(): void ); self::assertStringContainsString('Running analysis on 2 cores with 1 file per process.', $cmdTester->getDisplay()); + self::assertStringContainsString('Parallel runner is an experimental feature and may be unstable, use it at your own risk. Feedback highly appreciated!', $cmdTester->getDisplay()); self::assertStringContainsString('(header_comment)', $cmdTester->getDisplay()); self::assertSame(8, $cmdTester->getStatusCode()); } From 0ad99f63fc57657c2e6085311373320ecd03c756 Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Tue, 14 May 2024 02:59:05 +0200 Subject: [PATCH 76/77] Rework error handling in workers - handled process errors (linting, failed fixes etc.) are now collected with ~original exception as source when possible. These don't break the main process, unless `--stop-on-violation` is used, it was like that before too, but `ParallelisationException` was used for recreating error's source, which was not correct. - other errors (socket in/out errors caught by React, or unhandled errors that kill the worker) are re-thrown in the main process that effectively stop further execution --- src/Console/Application.php | 9 +++ src/Console/Command/FixCommand.php | 8 +- .../FixCommandExitStatusCalculator.php | 5 +- src/Console/Command/WorkerCommand.php | 2 + src/Console/Output/ErrorOutput.php | 37 --------- src/Error/Error.php | 3 +- src/Error/ErrorsManager.php | 22 +----- src/Error/SourceExceptionFactory.php | 60 ++++++++++++++ src/Error/WorkerError.php | 63 --------------- .../Parallel/ParallelisationException.php | 12 --- src/Runner/Parallel/WorkerException.php | 66 ++++++++++++++++ src/Runner/Runner.php | 38 +++------ tests/Console/ApplicationTest.php | 2 +- .../FixCommandExitStatusCalculatorTest.php | 34 ++++---- tests/Console/Output/ErrorOutputTest.php | 61 -------------- tests/Error/ErrorsManagerTest.php | 24 ------ tests/Error/SourceExceptionFactoryTest.php | 79 +++++++++++++++++++ tests/Error/WorkerErrorTest.php | 47 ----------- .../Parallel/ParallelisationExceptionTest.php | 15 ---- tests/Runner/Parallel/WorkerExceptionTest.php | 44 +++++++++++ tests/Runner/RunnerTest.php | 4 +- 21 files changed, 293 insertions(+), 342 deletions(-) create mode 100644 src/Error/SourceExceptionFactory.php delete mode 100644 src/Error/WorkerError.php create mode 100644 src/Runner/Parallel/WorkerException.php create mode 100644 tests/Error/SourceExceptionFactoryTest.php delete mode 100644 tests/Error/WorkerErrorTest.php create mode 100644 tests/Runner/Parallel/WorkerExceptionTest.php diff --git a/src/Console/Application.php b/src/Console/Application.php index c0044436207..3ed1b16cbf2 100644 --- a/src/Console/Application.php +++ b/src/Console/Application.php @@ -25,6 +25,7 @@ use PhpCsFixer\Console\SelfUpdate\GithubClient; use PhpCsFixer\Console\SelfUpdate\NewVersionChecker; use PhpCsFixer\PharChecker; +use PhpCsFixer\Runner\Parallel\WorkerException; use PhpCsFixer\ToolInfo; use PhpCsFixer\Utils; use Symfony\Component\Console\Application as BaseApplication; @@ -184,6 +185,7 @@ protected function doRenderThrowable(\Throwable $e, OutputInterface $output): vo if ($this->executedCommand instanceof WorkerCommand) { $output->writeln(WorkerCommand::ERROR_PREFIX.json_encode( [ + 'class' => \get_class($e), 'message' => $e->getMessage(), 'file' => $e->getFile(), 'line' => $e->getLine(), @@ -196,5 +198,12 @@ protected function doRenderThrowable(\Throwable $e, OutputInterface $output): vo } parent::doRenderThrowable($e, $output); + + if ($output->isVeryVerbose() && $e instanceof WorkerException) { + $output->writeln('Original trace from worker:'); + $output->writeln(''); + $output->writeln($e->getOriginalTraceAsString()); + $output->writeln(''); + } } } diff --git a/src/Console/Command/FixCommand.php b/src/Console/Command/FixCommand.php index b8246488353..f05fc123151 100644 --- a/src/Console/Command/FixCommand.php +++ b/src/Console/Command/FixCommand.php @@ -357,7 +357,6 @@ protected function execute(InputInterface $input, OutputInterface $output): int ? $output->write($reporter->generate($reportSummary)) : $output->write($reporter->generate($reportSummary), false, OutputInterface::OUTPUT_RAW); - $workerErrors = $this->errorsManager->getWorkerErrors(); $invalidErrors = $this->errorsManager->getInvalidErrors(); $exceptionErrors = $this->errorsManager->getExceptionErrors(); $lintErrors = $this->errorsManager->getLintErrors(); @@ -365,10 +364,6 @@ protected function execute(InputInterface $input, OutputInterface $output): int if (null !== $stdErr) { $errorOutput = new ErrorOutput($stdErr); - if (\count($workerErrors) > 0) { - $errorOutput->listWorkerErrors($workerErrors); - } - if (\count($invalidErrors) > 0) { $errorOutput->listErrors('linting before fixing', $invalidErrors); } @@ -389,8 +384,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int \count($changed) > 0, \count($invalidErrors) > 0, \count($exceptionErrors) > 0, - \count($lintErrors) > 0, - \count($workerErrors) > 0 + \count($lintErrors) > 0 ); } diff --git a/src/Console/Command/FixCommandExitStatusCalculator.php b/src/Console/Command/FixCommandExitStatusCalculator.php index 61831ac6e64..28d6b258fbb 100644 --- a/src/Console/Command/FixCommandExitStatusCalculator.php +++ b/src/Console/Command/FixCommandExitStatusCalculator.php @@ -33,8 +33,7 @@ public function calculate( bool $hasChangedFiles, bool $hasInvalidErrors, bool $hasExceptionErrors, - bool $hasLintErrorsAfterFixing, - bool $hasWorkerErrors + bool $hasLintErrorsAfterFixing ): int { $exitStatus = 0; @@ -48,7 +47,7 @@ public function calculate( } } - if ($hasExceptionErrors || $hasLintErrorsAfterFixing || $hasWorkerErrors) { + if ($hasExceptionErrors || $hasLintErrorsAfterFixing) { $exitStatus |= self::EXIT_STATUS_FLAG_EXCEPTION_IN_APP; } diff --git a/src/Console/Command/WorkerCommand.php b/src/Console/Command/WorkerCommand.php index 4b4e2964c74..64a57a15f08 100644 --- a/src/Console/Command/WorkerCommand.php +++ b/src/Console/Command/WorkerCommand.php @@ -124,6 +124,7 @@ function (ConnectionInterface $connection) use ($loop, $runner, $identifier): vo $handleError = static function (\Throwable $error) use ($out): void { $out->write([ 'action' => ParallelAction::WORKER_ERROR_REPORT, + 'class' => \get_class($error), 'message' => $error->getMessage(), 'file' => $error->getFile(), 'line' => $error->getLine(), @@ -182,6 +183,7 @@ function (ConnectionInterface $connection) use ($loop, $runner, $identifier): vo }); }, static function (\Throwable $error) use ($errorOutput): void { + // @TODO Verify onRejected behaviour → https://github.com/PHP-CS-Fixer/PHP-CS-Fixer/pull/7777#discussion_r1590399285 $errorOutput->writeln($error->getMessage()); } ) diff --git a/src/Console/Output/ErrorOutput.php b/src/Console/Output/ErrorOutput.php index 138645b06a5..79bb57b7320 100644 --- a/src/Console/Output/ErrorOutput.php +++ b/src/Console/Output/ErrorOutput.php @@ -16,7 +16,6 @@ use PhpCsFixer\Differ\DiffConsoleFormatter; use PhpCsFixer\Error\Error; -use PhpCsFixer\Error\WorkerError; use PhpCsFixer\Linter\LintingException; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Formatter\OutputFormatter; @@ -119,42 +118,6 @@ public function listErrors(string $process, array $errors): void } } - /** - * @param list $errors - */ - public function listWorkerErrors(array $errors): void - { - $this->output->writeln(['', 'Errors reported from workers (parallel analysis):']); - - $showDetails = $this->output->getVerbosity() >= OutputInterface::VERBOSITY_VERY_VERBOSE; - $showTrace = $this->output->getVerbosity() >= OutputInterface::VERBOSITY_DEBUG; - - foreach ($errors as $i => $error) { - $this->output->writeln(sprintf('%4d) %s', $i + 1, $error->getMessage())); - - if (!$showDetails) { - continue; - } - - $this->output->writeln(sprintf( - ' in %s on line %d', - $error->getFilePath(), - $error->getLine() - )); - - if ($showTrace) { - $this->output->writeln([ - ' Stack trace:', - ...array_map( - static fn (string $frame) => " {$frame}", - explode("\n", $error->getTrace()) - ), - '', - ]); - } - } - } - /** * @param array{ * function?: string, diff --git a/src/Error/Error.php b/src/Error/Error.php index cc209aca238..d0c5ac7ea18 100644 --- a/src/Error/Error.php +++ b/src/Error/Error.php @@ -97,7 +97,7 @@ public function getDiff(): ?string * @return array{ * type: self::TYPE_*, * filePath: string, - * source: null|array{message: string, code: int, file: string, line: int}, + * source: null|array{class: class-string, message: string, code: int, file: string, line: int}, * appliedFixers: list, * diff: null|string * } @@ -109,6 +109,7 @@ public function jsonSerialize(): array 'filePath' => $this->filePath, 'source' => null !== $this->source ? [ + 'class' => \get_class($this->source), 'message' => $this->source->getMessage(), 'code' => $this->source->getCode(), 'file' => $this->source->getFile(), diff --git a/src/Error/ErrorsManager.php b/src/Error/ErrorsManager.php index 6f060b813b0..e6333db80f1 100644 --- a/src/Error/ErrorsManager.php +++ b/src/Error/ErrorsManager.php @@ -28,21 +28,6 @@ final class ErrorsManager */ private array $errors = []; - /** - * @var list - */ - private array $workerErrors = []; - - /** - * Returns worker errors reported during processing files in parallel. - * - * @return list - */ - public function getWorkerErrors(): array - { - return $this->workerErrors; - } - /** * Returns errors reported during linting before fixing. * @@ -88,16 +73,11 @@ public function forPath(string $path): array */ public function isEmpty(): bool { - return [] === $this->errors && [] === $this->workerErrors; + return [] === $this->errors; } public function report(Error $error): void { $this->errors[] = $error; } - - public function reportWorkerError(WorkerError $error): void - { - $this->workerErrors[] = $error; - } } diff --git a/src/Error/SourceExceptionFactory.php b/src/Error/SourceExceptionFactory.php new file mode 100644 index 00000000000..390cd3b7d57 --- /dev/null +++ b/src/Error/SourceExceptionFactory.php @@ -0,0 +1,60 @@ + + * Dariusz Rumiński + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace PhpCsFixer\Error; + +/** + * @internal + */ +final class SourceExceptionFactory +{ + /** + * @param array{class: class-string<\Throwable>, message: string, code: int, file: string, line: int} $error + */ + public static function fromArray(array $error): \Throwable + { + $exceptionClass = $error['class']; + + try { + $exception = new $exceptionClass($error['message'], $error['code']); + + if ( + $exception->getMessage() !== $error['message'] + || $exception->getCode() !== $error['code'] + ) { + throw new \RuntimeException('Failed to create exception from array. Message and code are not the same.'); + } + } catch (\Throwable $e) { + $exception = new \RuntimeException( + sprintf('[%s] %s', $exceptionClass, $error['message']), + $error['code'] + ); + } + + try { + $exceptionReflection = new \ReflectionClass($exception); + foreach (['file', 'line'] as $property) { + $propertyReflection = $exceptionReflection->getProperty($property); + $propertyReflection->setAccessible(true); + $propertyReflection->setValue($exception, $error[$property]); + $propertyReflection->setAccessible(false); + } + } catch (\Throwable $reflectionException) { + // Ignore if we were not able to set file/line properties. In most cases it should be fine, + // we just need to make sure nothing is broken when we recreate errors from raw data passed from worker. + } + + return $exception; + } +} diff --git a/src/Error/WorkerError.php b/src/Error/WorkerError.php deleted file mode 100644 index e5d21c34add..00000000000 --- a/src/Error/WorkerError.php +++ /dev/null @@ -1,63 +0,0 @@ - - * Dariusz Rumiński - * - * This source file is subject to the MIT license that is bundled - * with this source code in the file LICENSE. - */ - -namespace PhpCsFixer\Error; - -/** - * @author Greg Korba - * - * @internal - */ -final class WorkerError -{ - private string $message; - private string $filePath; - private int $line; - private int $code; - private string $trace; - - public function __construct(string $message, string $filePath, int $line, int $code, string $trace) - { - $this->message = $message; - $this->filePath = $filePath; - $this->line = $line; - $this->code = $code; - $this->trace = $trace; - } - - public function getMessage(): string - { - return $this->message; - } - - public function getFilePath(): string - { - return $this->filePath; - } - - public function getLine(): int - { - return $this->line; - } - - public function getCode(): int - { - return $this->code; - } - - public function getTrace(): string - { - return $this->trace; - } -} diff --git a/src/Runner/Parallel/ParallelisationException.php b/src/Runner/Parallel/ParallelisationException.php index 4b7d2842698..8c24e6a4a5c 100644 --- a/src/Runner/Parallel/ParallelisationException.php +++ b/src/Runner/Parallel/ParallelisationException.php @@ -27,16 +27,4 @@ public static function forUnknownIdentifier(ProcessIdentifier $identifier): self { return new self('Unknown process identifier: '.$identifier->toString()); } - - /** - * @param array{message: string, code: int, file: string, line: int} $error - */ - public static function forWorkerError(array $error): self - { - $exception = new self($error['message'], $error['code']); - $exception->file = $error['file']; - $exception->line = $error['line']; - - return $exception; - } } diff --git a/src/Runner/Parallel/WorkerException.php b/src/Runner/Parallel/WorkerException.php new file mode 100644 index 00000000000..f526ecdd8f1 --- /dev/null +++ b/src/Runner/Parallel/WorkerException.php @@ -0,0 +1,66 @@ + + * Dariusz Rumiński + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace PhpCsFixer\Runner\Parallel; + +use Throwable; + +/** + * @author Greg Korba + * + * @internal + */ +final class WorkerException extends \RuntimeException +{ + private string $originalTraceAsString; + + private function __construct(string $message, int $code) + { + parent::__construct($message, $code); + } + + /** + * @param array{ + * class: class-string, + * message: string, + * file: string, + * line: int, + * code: int, + * trace: string + * } $data + */ + public static function fromRaw(array $data): self + { + $exception = new self( + sprintf('[%s] %s', $data['class'], $data['message']), + $data['code'] + ); + $exception->file = $data['file']; + $exception->line = $data['line']; + $exception->originalTraceAsString = sprintf( + '## %s(%d)%s%s', + $data['file'], + $data['line'], + PHP_EOL, + $data['trace'] + ); + + return $exception; + } + + public function getOriginalTraceAsString(): string + { + return $this->originalTraceAsString; + } +} diff --git a/src/Runner/Runner.php b/src/Runner/Runner.php index d88f1b7a88d..3a9c7aad029 100644 --- a/src/Runner/Runner.php +++ b/src/Runner/Runner.php @@ -24,7 +24,7 @@ use PhpCsFixer\Differ\DifferInterface; use PhpCsFixer\Error\Error; use PhpCsFixer\Error\ErrorsManager; -use PhpCsFixer\Error\WorkerError; +use PhpCsFixer\Error\SourceExceptionFactory; use PhpCsFixer\FileReader; use PhpCsFixer\Fixer\FixerInterface; use PhpCsFixer\FixerFileProcessedEvent; @@ -39,6 +39,7 @@ use PhpCsFixer\Runner\Parallel\ProcessFactory; use PhpCsFixer\Runner\Parallel\ProcessIdentifier; use PhpCsFixer\Runner\Parallel\ProcessPool; +use PhpCsFixer\Runner\Parallel\WorkerException; use PhpCsFixer\Tokenizer\Tokens; use React\EventLoop\StreamSelectLoop; use React\Socket\ConnectionInterface; @@ -262,7 +263,7 @@ function (array $workerResponse) use ($processPool, $process, $identifier, $getF $error['type'], $error['filePath'], null !== $error['source'] - ? ParallelisationException::forWorkerError($error['source']) + ? SourceExceptionFactory::fromArray($error['source']) : null, $error['appliedFixers'], $error['diff'] @@ -300,35 +301,21 @@ function (array $workerResponse) use ($processPool, $process, $identifier, $getF } if (ParallelAction::WORKER_ERROR_REPORT === $workerResponse['action']) { - $this->errorsManager->reportWorkerError(new WorkerError( - $workerResponse['message'], - $workerResponse['file'], - (int) $workerResponse['line'], - (int) $workerResponse['code'], - $workerResponse['trace'] - )); - - return; + throw WorkerException::fromRaw($workerResponse); // @phpstan-ignore-line } throw new ParallelisationException('Unsupported action: '.($workerResponse['action'] ?? 'n/a')); }, // [REACT] Handle errors encountered during worker's execution - function (\Throwable $error) use ($processPool): void { - $this->errorsManager->reportWorkerError(new WorkerError( - $error->getMessage(), - $error->getFile(), - $error->getLine(), - $error->getCode(), - $error->getTraceAsString() - )); - + static function (\Throwable $error) use ($processPool): void { $processPool->endAll(); + + throw new ParallelisationException($error->getMessage(), $error->getCode(), $error); }, // [REACT] Handle worker's shutdown - function ($exitCode, string $output) use ($processPool, $identifier): void { + static function ($exitCode, string $output) use ($processPool, $identifier): void { $processPool->endProcessIfKnown($identifier); if (0 === $exitCode || null === $exitCode) { @@ -342,14 +329,7 @@ function ($exitCode, string $output) use ($processPool, $identifier): void { ); if ($errorsReported > 0) { - $error = json_decode($matches[1][0], true); - $this->errorsManager->reportWorkerError(new WorkerError( - $error['message'], - $error['file'], - (int) $error['line'], - (int) $error['code'], - $error['trace'] - )); + throw WorkerException::fromRaw(json_decode($matches[1][0], true)); } } ); diff --git a/tests/Console/ApplicationTest.php b/tests/Console/ApplicationTest.php index bf00a337cff..67f81f5d2d8 100644 --- a/tests/Console/ApplicationTest.php +++ b/tests/Console/ApplicationTest.php @@ -51,7 +51,7 @@ public function testWorkerExceptionsAreRenderedInMachineFriendlyWay(): void $appTester->run(['worker']); self::assertStringContainsString( - WorkerCommand::ERROR_PREFIX.'{"message":"Missing parallelisation options"', + WorkerCommand::ERROR_PREFIX.'{"class":"PhpCsFixer\\\Runner\\\Parallel\\\ParallelisationException","message":"Missing parallelisation options"', $appTester->getDisplay() ); } diff --git a/tests/Console/Command/FixCommandExitStatusCalculatorTest.php b/tests/Console/Command/FixCommandExitStatusCalculatorTest.php index 60244b66f01..90e30b97821 100644 --- a/tests/Console/Command/FixCommandExitStatusCalculatorTest.php +++ b/tests/Console/Command/FixCommandExitStatusCalculatorTest.php @@ -36,8 +36,7 @@ public function testCalculate( bool $hasChangedFiles, bool $hasInvalidErrors, bool $hasExceptionErrors, - bool $hasLintErrorsAfterFixing, - bool $hasWorkerErrors + bool $hasLintErrorsAfterFixing ): void { $calculator = new FixCommandExitStatusCalculator(); @@ -48,40 +47,37 @@ public function testCalculate( $hasChangedFiles, $hasInvalidErrors, $hasExceptionErrors, - $hasLintErrorsAfterFixing, - $hasWorkerErrors + $hasLintErrorsAfterFixing ) ); } public static function provideCalculateCases(): iterable { - yield [0, true, false, false, false, false, false]; + yield [0, true, false, false, false, false]; - yield [0, false, false, false, false, false, false]; + yield [0, false, false, false, false, false]; - yield [8, true, true, false, false, false, false]; + yield [8, true, true, false, false, false]; - yield [0, false, true, false, false, false, false]; + yield [0, false, true, false, false, false]; - yield [4, true, false, true, false, false, false]; + yield [4, true, false, true, false, false]; - yield [0, false, false, true, false, false, false]; + yield [0, false, false, true, false, false]; - yield [12, true, true, true, false, false, false]; + yield [12, true, true, true, false, false]; - yield [0, false, true, true, false, false, false]; + yield [0, false, true, true, false, false]; - yield [76, true, true, true, true, false, false]; + yield [76, true, true, true, true, false]; - yield [64, false, false, false, false, true, false]; + yield [64, false, false, false, false, true]; - yield [64, false, false, false, true, false, false]; + yield [64, false, false, false, true, false]; - yield [64, false, false, false, true, true, false]; + yield [64, false, false, false, true, true]; - yield [64, false, false, false, false, false, true]; - - yield [8 | 64, true, true, false, true, true, false]; + yield [8 | 64, true, true, false, true, true]; } } diff --git a/tests/Console/Output/ErrorOutputTest.php b/tests/Console/Output/ErrorOutputTest.php index bd9e396ab2a..82c71c4608d 100644 --- a/tests/Console/Output/ErrorOutputTest.php +++ b/tests/Console/Output/ErrorOutputTest.php @@ -16,7 +16,6 @@ use PhpCsFixer\Console\Output\ErrorOutput; use PhpCsFixer\Error\Error; -use PhpCsFixer\Error\WorkerError; use PhpCsFixer\Linter\LintingException; use PhpCsFixer\Tests\TestCase; use Symfony\Component\Console\Output\OutputInterface; @@ -29,66 +28,6 @@ */ final class ErrorOutputTest extends TestCase { - /** - * @param OutputInterface::VERBOSITY_* $verbosityLevel - * - * @dataProvider provideWorkerErrorOutputCases - */ - public function testWorkerErrorOutput(WorkerError $error, int $verbosityLevel): void - { - $output = $this->createStreamOutput($verbosityLevel); - $errorOutput = new ErrorOutput($output); - $errorOutput->listWorkerErrors([$error]); - - $displayed = $this->readFullStreamOutput($output); - - $startWith = sprintf( - ' -Errors reported from workers (parallel analysis): - 1) %s', - $error->getMessage() - ); - - if ($verbosityLevel >= OutputInterface::VERBOSITY_VERY_VERBOSE) { - $startWith .= sprintf( - ' - in %s on line %d', - $error->getFilePath(), - $error->getLine() - ); - } - - if ($verbosityLevel >= OutputInterface::VERBOSITY_DEBUG) { - $startWith .= sprintf( - ' - Stack trace: -%s', - implode("\n", array_map( - static fn (string $frame) => " {$frame}", - explode("\n", $error->getTrace()) - )) - ); - } - - self::assertStringStartsWith($startWith, $displayed); - } - - /** - * @return iterable - */ - public static function provideWorkerErrorOutputCases(): iterable - { - $error = new WorkerError('Boom!', 'foo.php', 123, 1, '#0 Foo\n#1 Bar\n#2 {main}'); - - yield [$error, OutputInterface::VERBOSITY_NORMAL]; - - yield [$error, OutputInterface::VERBOSITY_VERBOSE]; - - yield [$error, OutputInterface::VERBOSITY_VERY_VERBOSE]; - - yield [$error, OutputInterface::VERBOSITY_DEBUG]; - } - /** * @param OutputInterface::VERBOSITY_* $verbosityLevel * diff --git a/tests/Error/ErrorsManagerTest.php b/tests/Error/ErrorsManagerTest.php index 420b6e3bf6e..70ee5657154 100644 --- a/tests/Error/ErrorsManagerTest.php +++ b/tests/Error/ErrorsManagerTest.php @@ -16,7 +16,6 @@ use PhpCsFixer\Error\Error; use PhpCsFixer\Error\ErrorsManager; -use PhpCsFixer\Error\WorkerError; use PhpCsFixer\Tests\TestCase; /** @@ -34,7 +33,6 @@ public function testDefaults(): void self::assertEmpty($errorsManager->getInvalidErrors()); self::assertEmpty($errorsManager->getExceptionErrors()); self::assertEmpty($errorsManager->getLintErrors()); - self::assertEmpty($errorsManager->getWorkerErrors()); } public function testThatCanReportAndRetrieveInvalidErrors(): void @@ -57,7 +55,6 @@ public function testThatCanReportAndRetrieveInvalidErrors(): void self::assertCount(0, $errorsManager->getExceptionErrors()); self::assertCount(0, $errorsManager->getLintErrors()); - self::assertCount(0, $errorsManager->getWorkerErrors()); } public function testThatCanReportAndRetrieveExceptionErrors(): void @@ -80,7 +77,6 @@ public function testThatCanReportAndRetrieveExceptionErrors(): void self::assertCount(0, $errorsManager->getInvalidErrors()); self::assertCount(0, $errorsManager->getLintErrors()); - self::assertCount(0, $errorsManager->getWorkerErrors()); } public function testThatCanReportAndRetrieveInvalidFileErrors(): void @@ -103,26 +99,6 @@ public function testThatCanReportAndRetrieveInvalidFileErrors(): void self::assertCount(0, $errorsManager->getInvalidErrors()); self::assertCount(0, $errorsManager->getExceptionErrors()); - self::assertCount(0, $errorsManager->getWorkerErrors()); - } - - public function testThatCanReportAndRetrieveWorkerErrors(): void - { - $error = new WorkerError('Boom!', 'foo.php', 123, 1, '#0 Foo\n#1 Bar'); - $errorsManager = new ErrorsManager(); - - $errorsManager->reportWorkerError($error); - - self::assertFalse($errorsManager->isEmpty()); - - $errors = $errorsManager->getWorkerErrors(); - - self::assertCount(1, $errors); - self::assertSame($error, array_shift($errors)); - - self::assertCount(0, $errorsManager->getInvalidErrors()); - self::assertCount(0, $errorsManager->getExceptionErrors()); - self::assertCount(0, $errorsManager->getLintErrors()); } public function testThatCanReportAndRetrieveErrorsForSpecificPath(): void diff --git a/tests/Error/SourceExceptionFactoryTest.php b/tests/Error/SourceExceptionFactoryTest.php new file mode 100644 index 00000000000..9cb1685f1e5 --- /dev/null +++ b/tests/Error/SourceExceptionFactoryTest.php @@ -0,0 +1,79 @@ + + * Dariusz Rumiński + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace PhpCsFixer\Tests\Error; + +use PhpCsFixer\Error\SourceExceptionFactory; +use PhpCsFixer\Linter\LintingException; +use PhpCsFixer\Runner\Parallel\WorkerException; +use PhpCsFixer\Tests\TestCase; + +/** + * @covers \PhpCsFixer\Error\SourceExceptionFactory + * + * @internal + */ +final class SourceExceptionFactoryTest extends TestCase +{ + public function testFromArrayWithInstantiableException(): void + { + $exception = SourceExceptionFactory::fromArray([ + 'class' => LintingException::class, + 'message' => 'foo', + 'code' => 1, + 'file' => 'foo.php', + 'line' => 1, + ]); + + self::assertInstanceOf(LintingException::class, $exception); + self::assertSame('foo', $exception->getMessage()); + self::assertSame(1, $exception->getCode()); + self::assertSame('foo.php', $exception->getFile()); + self::assertSame(1, $exception->getLine()); + } + + public function testFromArrayWithInstantiableError(): void + { + $error = SourceExceptionFactory::fromArray([ + 'class' => \ParseError::class, + 'message' => 'foo', + 'code' => 1, + 'file' => 'foo.php', + 'line' => 1, + ]); + + self::assertInstanceOf(\ParseError::class, $error); + self::assertSame('foo', $error->getMessage()); + self::assertSame(1, $error->getCode()); + self::assertSame('foo.php', $error->getFile()); + self::assertSame(1, $error->getLine()); + } + + public function testFromArrayWithNonInstantiableException(): void + { + $exception = SourceExceptionFactory::fromArray([ + 'class' => WorkerException::class, + 'message' => 'foo', + 'code' => 1, + 'file' => 'foo.php', + 'line' => 1, + ]); + + self::assertInstanceOf(\RuntimeException::class, $exception); + self::assertSame('[PhpCsFixer\Runner\Parallel\WorkerException] foo', $exception->getMessage()); + self::assertSame(1, $exception->getCode()); + self::assertSame('foo.php', $exception->getFile()); + self::assertSame(1, $exception->getLine()); + } +} diff --git a/tests/Error/WorkerErrorTest.php b/tests/Error/WorkerErrorTest.php deleted file mode 100644 index bc54f01a7e3..00000000000 --- a/tests/Error/WorkerErrorTest.php +++ /dev/null @@ -1,47 +0,0 @@ - - * Dariusz Rumiński - * - * This source file is subject to the MIT license that is bundled - * with this source code in the file LICENSE. - */ - -namespace PhpCsFixer\Tests\Error; - -use PhpCsFixer\Error\WorkerError; -use PhpCsFixer\Tests\TestCase; - -/** - * @covers \PhpCsFixer\Error\WorkerError - * - * @internal - */ -final class WorkerErrorTest extends TestCase -{ - public function testConstructorDataCanBeAccessed(): void - { - $message = 'BOOM!'; - $filePath = '/path/to/file.php'; - $line = 10; - $code = 100; - $trace = <<<'TRACE' - #0 Foo - #1 Bar - #2 {main} - TRACE; - - $error = new WorkerError($message, $filePath, $line, $code, $trace); - - self::assertSame($message, $error->getMessage()); - self::assertSame($filePath, $error->getFilePath()); - self::assertSame($line, $error->getLine()); - self::assertSame($code, $error->getCode()); - self::assertSame($trace, $error->getTrace()); - } -} diff --git a/tests/Runner/Parallel/ParallelisationExceptionTest.php b/tests/Runner/Parallel/ParallelisationExceptionTest.php index b3e8f83601f..b0a0075867e 100644 --- a/tests/Runner/Parallel/ParallelisationExceptionTest.php +++ b/tests/Runner/Parallel/ParallelisationExceptionTest.php @@ -33,19 +33,4 @@ public function testCreateForUnknownIdentifier(): void self::assertSame('Unknown process identifier: php-cs-fixer_parallel_foo', $exception->getMessage()); self::assertSame(0, $exception->getCode()); } - - public function testCreateForWorkerError(): void - { - $exception = ParallelisationException::forWorkerError([ - 'message' => 'foo', - 'code' => 1, - 'file' => 'foo.php', - 'line' => 1, - ]); - - self::assertSame('foo', $exception->getMessage()); - self::assertSame(1, $exception->getCode()); - self::assertSame('foo.php', $exception->getFile()); - self::assertSame(1, $exception->getLine()); - } } diff --git a/tests/Runner/Parallel/WorkerExceptionTest.php b/tests/Runner/Parallel/WorkerExceptionTest.php new file mode 100644 index 00000000000..cbad8680481 --- /dev/null +++ b/tests/Runner/Parallel/WorkerExceptionTest.php @@ -0,0 +1,44 @@ + + * Dariusz Rumiński + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace PhpCsFixer\Tests\Runner\Parallel; + +use PhpCsFixer\Runner\Parallel\WorkerException; +use PhpCsFixer\Tests\TestCase; + +/** + * @covers \PhpCsFixer\Runner\Parallel\WorkerException + * + * @internal + */ +final class WorkerExceptionTest extends TestCase +{ + public function testFromRaw(): void + { + $exception = WorkerException::fromRaw([ + 'class' => \RuntimeException::class, + 'message' => 'foo', + 'file' => 'foo.php', + 'line' => 1, + 'code' => 1, + 'trace' => '#0 bar', + ]); + + self::assertSame('[RuntimeException] foo', $exception->getMessage()); + self::assertSame('foo.php', $exception->getFile()); + self::assertSame(1, $exception->getLine()); + self::assertSame(1, $exception->getCode()); + self::assertSame('## foo.php(1)'.PHP_EOL.'#0 bar', $exception->getOriginalTraceAsString()); + } +} diff --git a/tests/Runner/RunnerTest.php b/tests/Runner/RunnerTest.php index 57eaf90326a..5e1623c491a 100644 --- a/tests/Runner/RunnerTest.php +++ b/tests/Runner/RunnerTest.php @@ -25,9 +25,9 @@ use PhpCsFixer\Fixer; use PhpCsFixer\Linter\Linter; use PhpCsFixer\Linter\LinterInterface; +use PhpCsFixer\Linter\LintingException; use PhpCsFixer\Linter\LintingResultInterface; use PhpCsFixer\Runner\Parallel\ParallelConfig; -use PhpCsFixer\Runner\Parallel\ParallelisationException; use PhpCsFixer\Runner\Runner; use PhpCsFixer\Tests\TestCase; use PhpCsFixer\ToolInfo; @@ -175,7 +175,7 @@ public function testThatParallelFixOfInvalidFileReportsToErrorManager(): void $error = $errors[0]; - self::assertInstanceOf(ParallelisationException::class, $error->getSource()); + self::assertInstanceOf(LintingException::class, $error->getSource()); self::assertSame(Error::TYPE_INVALID, $error->getType()); self::assertSame($pathToInvalidFile, $error->getFilePath()); From 1bae68bbde6bec0c1796fc81f6a2303d81cb475f Mon Sep 17 00:00:00 2001 From: Greg Korba Date: Tue, 14 May 2024 03:48:43 +0200 Subject: [PATCH 77/77] Fix smoke tests --- tests/Smoke/CiIntegrationTest.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/Smoke/CiIntegrationTest.php b/tests/Smoke/CiIntegrationTest.php index 77092157e29..947bdb57f32 100644 --- a/tests/Smoke/CiIntegrationTest.php +++ b/tests/Smoke/CiIntegrationTest.php @@ -173,13 +173,14 @@ public function testIntegration( : 'PHP CS Fixer '.preg_quote(Application::VERSION, '/')." by Fabien Potencier, Dariusz Ruminski and contributors.\nPHP runtime: ".PHP_VERSION; $pattern = sprintf( - '/^(?:%s)?(?:%s)?(?:%s)?(?:%s)?%s\n%s\n%s\n([\.S]{%d})%s\n%s$/', + '/^(?:%s)?(?:%s)?(?:%s)?(?:%s)?%s\n%s\n%s\n%s\n([\.S]{%d})%s\n%s$/', preg_quote($optionalDeprecatedVersionWarning, '/'), preg_quote($optionalIncompatibilityWarning, '/'), preg_quote($optionalXdebugWarning, '/'), preg_quote($optionalWarningsHelp, '/'), $aboutSubpattern, 'Running analysis on \d+ core(?: sequentially|s with \d+ files? per process)+\.', + preg_quote('You can enable parallel runner and speed up the analysis! Please see https://cs.symfony.com/doc/usage.html for more information.', '/'), preg_quote('Loaded config default from ".php-cs-fixer.dist.php".', '/'), \strlen($expectedResult3FilesDots), preg_quote($expectedResult3FilesPercentage, '/'),