Dump dependency container and re-use it - 35% faster test-suite (#3809)

.github/workflows/tests.yaml

@@ -43,4 +43,11 @@ jobs:
 
             -   uses: "ramsey/composer-install@v2"
 
+            - name: Cache rector kernels temporary files
+              uses: actions/cache@v3
+              with:
+                  path: /tmp/rector/
+                  key: ${{ matrix.php }}-${{ matrix.path }}-rector-kernel-${{ github.run_id }}
+                  restore-keys: ${{ matrix.php }}-${{ matrix.path }}-rector-kernel-
+
             -   run: vendor/bin/phpunit ${{ matrix.path }} --colors

packages-tests/Parallel/Command/WorkerCommandLineFactoryTest.php

@@ -35,10 +35,10 @@ final class WorkerCommandLineFactoryTest extends TestCase
     protected function setUp(): void
     {
         $rectorKernel = new RectorKernel();
-        $container = $rectorKernel->create();
+        $containerBuilder = $rectorKernel->createBuilder();
 
-        $this->workerCommandLineFactory = $container->get(WorkerCommandLineFactory::class);
-        $this->processCommand = $container->get(ProcessCommand::class);
+        $this->workerCommandLineFactory = $containerBuilder->get(WorkerCommandLineFactory::class);
+        $this->processCommand = $containerBuilder->get(ProcessCommand::class);
     }
 
     /**

packages-tests/Skipper/FileSystem/FnMatchPathNormalizerTest.php

@@ -17,9 +17,9 @@ final class FnMatchPathNormalizerTest extends TestCase
     protected function setUp(): void
     {
         $rectorKernel = new RectorKernel();
-        $container = $rectorKernel->create();
+        $containerBuilder = $rectorKernel->createBuilder();
 
-        $this->fnMatchPathNormalizer = $container->get(FnMatchPathNormalizer::class);
+        $this->fnMatchPathNormalizer = $containerBuilder->get(FnMatchPathNormalizer::class);
     }
 
     #[DataProvider('providePaths')]

packages/Testing/PHPUnit/AbstractTestCase.php

@@ -9,6 +9,7 @@
 use Rector\Core\Exception\ShouldNotHappenException;
 use Rector\Core\Kernel\RectorKernel;
 use Rector\Core\Util\FileHasher;
+use Throwable;
 
 abstract class AbstractTestCase extends TestCase
 {
@@ -59,8 +60,17 @@ protected function getService(string $type): object
             );
         }
 
-        $object = self::$currentContainer->get($type);
+        try {
+            $object = self::$currentContainer->get($type);
+        } catch (Throwable $e) {
+            // clear compiled container cache, to trigger re-discovery
+            RectorKernel::clearCache();
+
+            throw $e;
+        }
+
         if ($object === null) {
+
             $message = sprintf('Service "%s" was not found', $type);
             throw new ShouldNotHappenException($message);
         }

phpstan.neon

@@ -795,3 +795,8 @@ parameters:
         # deprecated
         - '#Register "Rector\\Php80\\Rector\\FunctionLike\\UnionTypesRector" service to "php80\.php" config set#'
         - '#Fetching class constant class of deprecated class Rector\\CodeQuality\\Rector\\ClassMethod\\NarrowUnionTypeDocRector#'
+
+        # statics are required in the kernel for performance reasons
+        -
+            message: '#Do not use static property#'
+            path: src/Kernel/RectorKernel.php

src/Console/Command/ProcessCommand.php

@@ -14,6 +14,7 @@
 use Rector\Core\Console\Output\OutputFormatterCollector;
 use Rector\Core\Contract\Console\OutputStyleInterface;
 use Rector\Core\Exception\ShouldNotHappenException;
+use Rector\Core\Kernel\RectorKernel;
 use Rector\Core\StaticReflection\DynamicSourceLocatorDecorator;
 use Rector\Core\Util\MemoryLimiter;
 use Rector\Core\Validation\EmptyConfigurableRectorChecker;
@@ -108,6 +109,7 @@ protected function initialize(InputInterface $input, OutputInterface $output): v
         $optionClearCache = (bool) $input->getOption(Option::CLEAR_CACHE);
         if ($optionDebug || $optionClearCache) {
             $this->changedFilesDetector->clear();
+            RectorKernel::clearCache();
         }
     }
 

src/Kernel/CachedContainerBuilder.php

@@ -0,0 +1,76 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Rector\Core\Kernel;
+
+use Rector\Core\Exception\ShouldNotHappenException;
+use Symfony\Component\DependencyInjection\ContainerBuilder;
+use Symfony\Component\DependencyInjection\ContainerInterface;
+use Symfony\Component\DependencyInjection\Dumper\PhpDumper;
+use Symplify\SmartFileSystem\SmartFileSystem;
+
+/**
+ * see https://symfony.com/doc/current/components/dependency_injection/compilation.html#dumping-the-configuration-for-performance
+ */
+final class CachedContainerBuilder
+{
+    public function __construct(
+        private readonly string $cacheDir,
+        private readonly string $cacheKey,
+    ) {
+        if (! str_ends_with($cacheDir, '/')) {
+            throw new ShouldNotHappenException(sprintf('Cache dir "%s" must end with "/"', $cacheDir));
+        }
+    }
+
+    /**
+     * @param string[] $configFiles
+     * @param callable(string[] $configFiles):ContainerBuilder $containerBuilderCallback
+     */
+    public function build(array $configFiles, string $hash, callable $containerBuilderCallback): ContainerInterface
+    {
+        $smartFileSystem = new SmartFileSystem();
+        $className = 'RectorKernel' . $hash;
+        $file = $this->cacheDir . 'kernel-' . $this->cacheKey . '-' . $hash . '.php';
+
+        if (file_exists($file)) {
+            require_once $file;
+            $className = '\\' . __NAMESPACE__ . '\\' . $className;
+            $container = new $className();
+            if (! $container instanceof ContainerInterface) {
+                throw new ShouldNotHappenException();
+            }
+        } else {
+            $container = ($containerBuilderCallback)($configFiles);
+
+            $phpDumper = new PhpDumper($container);
+            $dumpedContainer = $phpDumper->dump([
+                'class' => $className,
+                'namespace' => __NAMESPACE__,
+            ]);
+            if (! is_string($dumpedContainer)) {
+                throw new ShouldNotHappenException();
+            }
+
+            $smartFileSystem->dumpFile($file, $dumpedContainer);
+        }
+
+        return $container;
+    }
+
+    public function clearCache(): void
+    {
+        if (! is_writable($this->cacheDir)) {
+            return;
+        }
+
+        $cacheFiles = glob($this->cacheDir . 'kernel-*.php');
+        if ($cacheFiles === false) {
+            return;
+        }
+
+        $smartFileSystem = new SmartFileSystem();
+        $smartFileSystem->remove($cacheFiles);
+    }
+}

src/Kernel/ContainerBuilderBuilder.php

@@ -0,0 +1,67 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Rector\Core\Kernel;
+
+use Rector\Core\Config\Loader\ConfigureCallMergingLoaderFactory;
+use Rector\Core\DependencyInjection\Collector\ConfigureCallValuesCollector;
+use Rector\Core\DependencyInjection\CompilerPass\AutowireArrayParameterCompilerPass;
+use Rector\Core\DependencyInjection\CompilerPass\AutowireRectorCompilerPass;
+use Rector\Core\DependencyInjection\CompilerPass\MakeRectorsPublicCompilerPass;
+use Rector\Core\DependencyInjection\CompilerPass\MergeImportedRectorConfigureCallValuesCompilerPass;
+use Rector\Core\DependencyInjection\CompilerPass\RemoveSkippedRectorsCompilerPass;
+use Symfony\Component\DependencyInjection\Compiler\CompilerPassInterface;
+use Symfony\Component\DependencyInjection\ContainerBuilder;
+
+final class ContainerBuilderBuilder
+{
+    private readonly ConfigureCallValuesCollector $configureCallValuesCollector;
+
+    public function __construct()
+    {
+        $this->configureCallValuesCollector = new ConfigureCallValuesCollector();
+    }
+
+    /**
+     * @param string[] $configFiles
+     */
+    public function build(array $configFiles): ContainerBuilder
+    {
+        $compilerPasses = $this->createCompilerPasses();
+
+        $configureCallMergingLoaderFactory = new ConfigureCallMergingLoaderFactory($this->configureCallValuesCollector);
+        $containerBuilderFactory = new ContainerBuilderFactory($configureCallMergingLoaderFactory);
+
+        $containerBuilder = $containerBuilderFactory->create($configFiles, $compilerPasses);
+
+        // @see https://symfony.com/blog/new-in-symfony-4-4-dependency-injection-improvements-part-1
+        $containerBuilder->setParameter('container.dumper.inline_factories', true);
+        // to fix reincluding files again
+        $containerBuilder->setParameter('container.dumper.inline_class_loader', false);
+
+        $containerBuilder->compile();
+
+        return $containerBuilder;
+    }
+
+    /**
+     * @return CompilerPassInterface[]
+     */
+    private function createCompilerPasses(): array
+    {
+        return [
+
+            // must run before AutowireArrayParameterCompilerPass, as the autowired array cannot contain removed services
+            new RemoveSkippedRectorsCompilerPass(),
+
+            // autowire Rectors by default (mainly for tests)
+            new AutowireRectorCompilerPass(),
+            new MakeRectorsPublicCompilerPass(),
+
+            // add all merged arguments of Rector services
+            new MergeImportedRectorConfigureCallValuesCompilerPass($this->configureCallValuesCollector),
+            new AutowireArrayParameterCompilerPass(),
+        ];
+    }
+}