Skip to content

Commit

Permalink
[TASK] Improve dependency injection container caching
Browse files Browse the repository at this point in the history 
Disallow disabling and flushing the DI cache and
base the cache identifier as well on the currently
installed extensions.

A disabled DI cache creates an unbearable performance hit,
so that disabling won't make sense anyway.
Disallowing flushing that cache will not flush the DI cache
when caches are flushed using the regular backend UI,
but only when caches in install tool are flushed.

Last but not least, the install tool cache flushing is
changed to not bypass the caching API any more by removing complete
caching folders or caching database tables.
Instead the CacheManager is now used twice, once with basic
caching configuration and a second time with caching configuration
that is provided by extensions (if any).

Releases: master
Resolves: #90418
Change-Id: Idc3d053e181c909ccd662065a9c1ab7a893fa9ac
Reviewed-on: https://review.typo3.org/c/Packages/TYPO3.CMS/+/63288
Tested-by: TYPO3com <noreply@typo3.com>
Tested-by: Georg Ringer <georg.ringer@gmail.com>
Tested-by: Benni Mack <benni@typo3.org>
Reviewed-by: Georg Ringer <georg.ringer@gmail.com>
Reviewed-by: Benni Mack <benni@typo3.org>
  • Loading branch information
@helhum @bmack
helhum authored and bmack committed Feb 20, 2020
1 parent 17534d8 commit e8d2e37
Show file tree
Hide file tree
Showing 7 changed files with 107 additions and 61 deletions.
12 changes: 10 additions & 2 deletions typo3/sysext/core/Classes/Core/Bootstrap.php
View file
Expand Up @@ -26,8 +26,10 @@
use TYPO3\CMS\Core\Cache\Exception\InvalidBackendException;
use TYPO3\CMS\Core\Cache\Exception\InvalidCacheException;
use TYPO3\CMS\Core\Cache\Frontend\FrontendInterface;
use TYPO3\CMS\Core\Cache\Frontend\PhpFrontend;
use TYPO3\CMS\Core\Cache\Frontend\VariableFrontend;
use TYPO3\CMS\Core\Configuration\ConfigurationManager;
use TYPO3\CMS\Core\DependencyInjection\Cache\ContainerBackend;
use TYPO3\CMS\Core\DependencyInjection\ContainerBuilder;
use TYPO3\CMS\Core\Imaging\IconRegistry;
use TYPO3\CMS\Core\IO\PharStreamWrapperInterceptor;
Expand Down Expand Up @@ -111,6 +113,7 @@ public static function init(
static::setMemoryLimit();

$assetsCache = static::createCache('assets', $disableCaching);
$dependencyInjectionContainerCache = static::createCache('di');

$bootState = new \stdClass;
$bootState->done = false;
Expand All @@ -121,6 +124,7 @@ public static function init(
ApplicationContext::class => Environment::getContext(),
ConfigurationManager::class => $configurationManager,
LogManager::class => $logManager,
'cache.di' => $dependencyInjectionContainerCache,
'cache.core' => $coreCache,
'cache.assets' => $assetsCache,
PackageManager::class => $packageManager,
Expand All @@ -129,7 +133,7 @@ public static function init(
'boot.state' => $bootState,
]);

$container = $builder->createDependencyInjectionContainer($packageManager, $coreCache, $failsafe);
$container = $builder->createDependencyInjectionContainer($packageManager, $dependencyInjectionContainerCache, $failsafe);

// Push the container to GeneralUtility as we want to make sure its
// makeInstance() method creates classes using the container from now on.
Expand Down Expand Up @@ -310,7 +314,11 @@ protected static function populateLocalConfiguration(ConfigurationManager $confi
*/
public static function createCache(string $identifier, bool $disableCaching = false): FrontendInterface
{
$configuration = $GLOBALS['TYPO3_CONF_VARS']['SYS']['caching']['cacheConfigurations'][$identifier] ?? [];
$cacheConfigurations = $GLOBALS['TYPO3_CONF_VARS']['SYS']['caching']['cacheConfigurations'] ?? [];
$cacheConfigurations['di']['frontend'] = PhpFrontend::class;
$cacheConfigurations['di']['backend'] = ContainerBackend::class;
$cacheConfigurations['di']['options'] = [];
$configuration = $cacheConfigurations[$identifier] ?? [];

$frontend = $configuration['frontend'] ?? VariableFrontend::class;
$backend = $configuration['backend'] ?? Typo3DatabaseBackend::class;
Expand Down
36 changes: 36 additions & 0 deletions typo3/sysext/core/Classes/DependencyInjection/Cache/ContainerBackend.php
View file
@@ -0,0 +1,36 @@
<?php
declare(strict_types = 1);
namespace TYPO3\CMS\Core\DependencyInjection\Cache;

/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/

use TYPO3\CMS\Core\Cache\Backend\SimpleFileBackend;

/**
* @internal
*/
class ContainerBackend extends SimpleFileBackend
{
public function flush()
{
// disable cache flushing
}

public function set($entryIdentifier, $data, array $tags = [], $lifetime = null)
{
// Remove stale cache files, once a new DI container was built
parent::flush();
parent::set($entryIdentifier, $data, $tags, $lifetime);
}
}
41 changes: 17 additions & 24 deletions typo3/sysext/core/Classes/DependencyInjection/ContainerBuilder.php
View file
Expand Up @@ -22,6 +22,7 @@
use Symfony\Component\DependencyInjection\Loader\PhpFileLoader;
use Symfony\Component\DependencyInjection\Loader\YamlFileLoader;
use TYPO3\CMS\Core\Cache\Frontend\FrontendInterface;
use TYPO3\CMS\Core\Cache\Frontend\PhpFrontend;
use TYPO3\CMS\Core\Core\Environment;
use TYPO3\CMS\Core\Information\Typo3Version;
use TYPO3\CMS\Core\Package\PackageManager;
Expand Down Expand Up @@ -62,6 +63,9 @@ public function __construct(array $earlyInstances)
*/
public function createDependencyInjectionContainer(PackageManager $packageManager, FrontendInterface $cache, bool $failsafe = false): ContainerInterface
{
if (!$cache instanceof PhpFrontend) {
throw new \RuntimeException('Cache must be instance of PhpFrontend', 1582022226);
}
$serviceProviderRegistry = new ServiceProviderRegistry($packageManager, $failsafe);

if ($failsafe) {
Expand All @@ -70,29 +74,16 @@ public function createDependencyInjectionContainer(PackageManager $packageManage

$container = null;

$cacheIdentifier = $this->getCacheIdentifier();
$cacheIdentifier = $this->getCacheIdentifier($packageManager);
$containerClassName = $cacheIdentifier;

$hasCache = $cache->requireOnce($cacheIdentifier) !== false;
if (!$hasCache) {
$containerBuilder = $this->buildContainer($packageManager, $serviceProviderRegistry);
$code = $this->dumpContainer($containerBuilder, $cache);

// In theory we could use the $containerBuilder directly as $container,
// but as we patch the compiled source to use
// GeneralUtility::makeInstanceForDi, we need to use the compiled container.
// Once we remove support for singletons configured in ext_localconf.php
// and $GLOBALS['TYPO_CONF_VARS']['SYS']['Objects'], we can remove this,
// and use `$container = $containerBuilder` directly
$hasCache = $cache->requireOnce($cacheIdentifier) !== false;
if (!$hasCache) {
// $cacheIdentifier may be unavailable if the 'core' cache iis configured to
// use the NullBackend
eval($code);
}
$this->dumpContainer($containerBuilder, $cache, $cacheIdentifier);
$cache->requireOnce($cacheIdentifier);
}
$fullyQualifiedContainerClassName = '\\' . $containerClassName;
$container = new $fullyQualifiedContainerClassName();
$container = new $containerClassName();

foreach ($this->defaultServices as $id => $service) {
$container->set('_early.' . $id, $service);
Expand Down Expand Up @@ -129,7 +120,7 @@ protected function buildContainer(PackageManager $packageManager, ServiceProvide
// Store defaults entries in the DIC container
// We need to use a workaround using aliases for synthetic services
// But that's common in symfony (same technique is used to provide the
// symfony container interface as well.
// Symfony container interface as well).
foreach (array_keys($this->defaultServices) as $id) {
$syntheticId = '_early.' . $id;
$containerBuilder->register($syntheticId)->setSynthetic(true)->setPublic(true);
Expand All @@ -144,11 +135,11 @@ protected function buildContainer(PackageManager $packageManager, ServiceProvide
/**
* @param SymfonyContainerBuilder $containerBuilder
* @param FrontendInterface $cache
* @param string $cacheIdentifier
* @return string
*/
protected function dumpContainer(SymfonyContainerBuilder $containerBuilder, FrontendInterface $cache): string
protected function dumpContainer(SymfonyContainerBuilder $containerBuilder, FrontendInterface $cache, string $cacheIdentifier): string
{
$cacheIdentifier = $this->getCacheIdentifier();
$containerClassName = $cacheIdentifier;

$phpDumper = new PhpDumper($containerBuilder);
Expand All @@ -165,18 +156,20 @@ protected function dumpContainer(SymfonyContainerBuilder $containerBuilder, Fron
}

/**
* @param PackageManager $packageManager
* @return string
*/
protected function getCacheIdentifier(): string
protected function getCacheIdentifier(PackageManager $packageManager): string
{
return $this->cacheIdentifier ?? $this->createCacheIdentifier();
return $this->cacheIdentifier ?? $this->createCacheIdentifier($packageManager->getCacheIdentifier());
}

/**
* @param string|null $additionalIdentifier
* @return string
*/
protected function createCacheIdentifier(): string
protected function createCacheIdentifier(string $additionalIdentifier = null): string
{
return $this->cacheIdentifier = 'DependencyInjectionContainer_' . sha1((string)(new Typo3Version()) . Environment::getProjectPath() . 'DependencyInjectionContainer');
return $this->cacheIdentifier = 'DependencyInjectionContainer_' . sha1((string)(new Typo3Version()) . Environment::getProjectPath() . ($additionalIdentifier ?? '') . 'DependencyInjectionContainer');
}
}
5 changes: 3 additions & 2 deletions typo3/sysext/core/Classes/Package/PackageManager.php
View file
Expand Up @@ -139,9 +139,10 @@ public function initialize()
}

/**
* @return string
* @internal
* @return string | null
*/
protected function getCacheIdentifier()
public function getCacheIdentifier()
{
if ($this->cacheIdentifier === null) {
$mTime = @filemtime($this->packageStatesPathAndFilename);
Expand Down
2 changes: 2 additions & 0 deletions typo3/sysext/core/Classes/ServiceProvider.php
View file
Expand Up @@ -65,10 +65,12 @@ public static function getCacheManager(ContainerInterface $container): Cache\Cac
$defaultCaches = [
$container->get('cache.core'),
$container->get('cache.assets'),
$container->get('cache.di'),
];

$cacheManager = self::new($container, Cache\CacheManager::class, [$disableCaching]);
$cacheManager->setCacheConfigurations($cacheConfigurations);
$cacheConfigurations['di']['groups'] = ['system'];
foreach ($defaultCaches as $cache) {
$cacheManager->registerCache($cache, $cacheConfigurations[$cache->getIdentifier()]['groups'] ?? ['all']);
}
Expand Down
65 changes: 37 additions & 28 deletions typo3/sysext/install/Classes/Service/ClearCacheService.php
View file
Expand Up @@ -14,7 +14,6 @@
* The TYPO3 project - inspiring people to share!
*/

use TYPO3\CMS\Core\Core\Environment;
use TYPO3\CMS\Core\Database\ConnectionPool;
use TYPO3\CMS\Core\Utility\GeneralUtility;

Expand All @@ -24,6 +23,10 @@
*/
class ClearCacheService
{
private const legacyDatabaseCacheTables = [
'cache_treelist',
];

/**
* @var LateBootService
*/
Expand All @@ -49,41 +52,47 @@ public function __construct(LateBootService $lateBootService)
*/
public function clearAll()
{
// Delete typo3temp/Cache
GeneralUtility::flushDirectory(Environment::getVarPath() . '/cache', true, true);

// Get all table names from Default connection starting with 'cf_' and truncate them
// Low level flush of legacy database cache tables
$connectionPool = GeneralUtility::makeInstance(ConnectionPool::class);
$connection = $connectionPool->getConnectionByName('Default');
$tableNames = $connection->getSchemaManager()->listTableNames();
foreach ($tableNames as $tableName) {
if (strpos($tableName, 'cf_') === 0 || $tableName === 'cache_treelist') {
$connection->truncate($tableName);
}
foreach (self::legacyDatabaseCacheTables as $tableName) {
$connection = $connectionPool->getConnectionForTable($tableName);
$connection->truncate($tableName);
}

// check tables on other connections
$remappedTables = isset($GLOBALS['TYPO3_CONF_VARS']['DB']['TableMapping'])
? array_keys((array)$GLOBALS['TYPO3_CONF_VARS']['DB']['TableMapping'])
: [];
foreach ($remappedTables as $tableName) {
if (strpos((string)$tableName, 'cf_') === 0 || $tableName === 'cache_treelist') {
$connectionPool->getConnectionForTable($tableName)->truncate($tableName);
}
}
// Flush all caches defined in TYPO3_CONF_VARS, but not the ones defined by extensions in ext_localconf.php
$baseCaches = $GLOBALS['TYPO3_CONF_VARS']['SYS']['caching']['cacheConfigurations'] ?? [];
$this->flushCaches($baseCaches);

// Remove DI container cache (this might be removed in preference of functionality to rebuild this cache)
// We need to remove using the remove method because the DI cache backend disables the flush method
$container = $this->lateBootService->getContainer();
$container->get('cache.di')->remove(get_class($container));

// From this point on, the code may fatal, if some broken extension is loaded.
$this->lateBootService->loadExtLocalconfDatabaseAndExtTables();

// The cache manager is already instantiated in the install tool
// (both in the failsafe and the late boot container), but
// with some hacked settings to disable caching of extbase and fluid.
// We want a "fresh" object here to operate on a different cache setup.
// cacheManager implements SingletonInterface, so the only way to get a "fresh"
// instance is by circumventing makeInstance and/or the objectManager and
// using new directly!
$extensionCaches = $GLOBALS['TYPO3_CONF_VARS']['SYS']['caching']['cacheConfigurations'] ?? [];
// Loose comparison on purpose to allow changed ordering of the array
if ($baseCaches != $extensionCaches) {
// When configuration has changed during loading of extensions (due to ext_localconf.php), flush all caches again
$this->flushCaches($GLOBALS['TYPO3_CONF_VARS']['SYS']['caching']['cacheConfigurations']);
}
}

/**
* The cache manager is already instantiated in the install tool
* (both in the failsafe and the late boot container), but
* with settings to disable caching (all caches using NullBackend).
* We want a "fresh" object here to operate with the really configured cache backends.
* CacheManager implements SingletonInterface, so the only way to get a "fresh"
* instance is by circumventing makeInstance and using new directly!
*
* @param array $cacheConfiguration
*/
private function flushCaches(array $cacheConfiguration): void
{
$cacheManager = new \TYPO3\CMS\Core\Cache\CacheManager();
$cacheManager->setCacheConfigurations($GLOBALS['TYPO3_CONF_VARS']['SYS']['caching']['cacheConfigurations']);
$cacheManager->setCacheConfigurations($cacheConfiguration);
$cacheManager->flushCaches();
}
}
7 changes: 2 additions & 5 deletions typo3/sysext/install/Classes/Service/LateBootService.php
View file
Expand Up @@ -69,15 +69,12 @@ public function getContainer(): ContainerInterface
private function prepareContainer(): ContainerInterface
{
$packageManager = $this->failsafeContainer->get(PackageManager::class);

// Use caching for the full boot – uncached symfony autowiring for every install-tool lateboot request would be too slow.
$disableCaching = false;
$coreCache = Bootstrap::createCache('core', $disableCaching);
$dependencyInjectionContainerCache = $this->failsafeContainer->get('cache.di');

$failsafe = false;

// Build a non-failsafe container which is required for loading ext_localconf
return $this->container = $this->containerBuilder->createDependencyInjectionContainer($packageManager, $coreCache, $failsafe);
return $this->container = $this->containerBuilder->createDependencyInjectionContainer($packageManager, $dependencyInjectionContainerCache, $failsafe);
}

/**
Expand Down

0 comments on commit e8d2e37

Please sign in to comment.