From a8b30320ce99ef7a1a1f4ca01f814afd3e767e1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andre=CC=81=20R?= Date: Mon, 25 Feb 2019 08:36:28 +0100 Subject: [PATCH] Update to align with Symfony PR --- README.md | 34 ++++--- doc/NativeTagAwareAdapters.md | 20 ++-- .../TagAware/AbstractTagAwareAdapter.php | 22 +++-- .../TagAware/FilesystemTagAwareAdapter.php | 69 ++++---------- .../TagAware/RedisTagAwareAdapter.php | 94 ++++++++----------- 5 files changed, 102 insertions(+), 137 deletions(-) rename src/lib/{Incubator/Cache => Symfony/Components/Cache/Adapter}/TagAware/AbstractTagAwareAdapter.php (93%) rename src/lib/{Incubator/Cache => Symfony/Components/Cache/Adapter}/TagAware/FilesystemTagAwareAdapter.php (70%) rename src/lib/{Incubator/Cache => Symfony/Components/Cache/Adapter}/TagAware/RedisTagAwareAdapter.php (59%) diff --git a/README.md b/README.md index d12478a..69ccfea 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # ezsystems/symfony-tools -Collection of polyfill (backport) and incubator features for Symfony. +Collection of polyfill (backported) and incubator (proposed) features for Symfony. Backports Symfony features so they can be used in earlier versions of Symfony, and -aims to serve as incubator for ideas to improve Symfony in the future. +proposed features improving Symfony further. This bundle is first and foremost aiming to cover needs of [eZ Platform](https://ezplatform.com), but is placed in own bundle under MIT as we think others can benefit and help collaborate, and @@ -10,37 +10,45 @@ to simplify forward and backport ports to and from Symfony itself. ### Requriments -- Symfony 3.4 _(4.3+ planned spring 2019, 2.8 support might happen if we need it)_ -- PHP 7.1+ 3.x branch for Symfony3 _(PHP 5.6+ for 2.x for Symfony2 if we ever add that)_ +- Symfony 3.4 +- PHP 7.1+ _(due to backported Symfony 4 code being written for PHP 7.1+)_ #### Semantic Versioning exception Bundle follows [SemVer](https://semver.org/) with one exception: -- Incubator features are allowed to break BC also in Minor versions (x.Y.z), __when__ needed in order to align with changes to the feature when it gets contributed to Symfony. +- Incubator features are allowed to break BC also in Minor versions (x.Y.z), __when__ needed in order to align with +changes to the feature when it gets accepted to Symfony. -!! Tip: As such if you rely on incubator features, make sure to require specific minor versions in composer, like `~1.1.0` or `~1.1.2 || ~1.2.0` +!! Tip: As such if you rely on incubator features, make sure to require specific minor versions in composer, like +`~1.1.0` or `~1.1.2 || ~1.2.0` ### Features **Polyfill (backport) features:** - [Redis session handler](doc/RedisSessionHandler.md) _(for Symfony3, native in Symfony4)_ -**Incubator features** -- +**Incubator (proposed) features** +- [NativeTagAwareAdapters](doc/NativeTagAwareAdapters.md) ### Contributing -Make sure as much as possible the feature is forward compatible for users, so when they upgrade to Symfony version where it's included, they should ideally not need to adapt their code/config. _(see `Semantic Versioning exception` for how this works for incubators)_ +Make sure as much as possible the feature is forward compatible for users, so when they upgrade to Symfony version where +it's included, they should ideally not need to adapt their code/config. _(see `Semantic Versioning exception` for how +this works for incubators)_ **Polyfill (Backports)** -When contributing Symfony backports to this bundle, be aware you commit to help maintain that feature in case there are bug fixes or improvements to that feature in Symfony itself. +When contributing Symfony backports to this bundle, be aware you commit to help maintain that feature in case there are +bug fixes or improvements to that feature in Symfony itself. -**Incubator** -Incubator features should only be proposed here if you intend to contribute this to Symfony itself, and there is at least some certainty it will be accepted. And you also commit to adapt the feature here, if changes are requested once proposed to Symfony. Essentially aiming for the feature here becoming a polyfill/backport feature in the end. +**Incubator (Proposed)** +Incubator features should only be proposed here if also proposed against Symfony itself, and there is at least some +certainty it will be accepted. And you also commit to adapt the feature here, if changes are requested once proposed to +Symfony. Essentially aiming for the feature here becoming a polyfill/backport feature in the end. -As such it's only applicable for smaller features _(e.g. new cache adapter(s))_, not a complete new component or larger changes across Symfony itself for instance. +As such it's only applicable for smaller features _(e.g. new cache adapter(s))_, not a complete new component or larger +changes across Symfony itself for instance. ### License diff --git a/doc/NativeTagAwareAdapters.md b/doc/NativeTagAwareAdapters.md index 5721f45..92d40a8 100644 --- a/doc/NativeTagAwareAdapters.md +++ b/doc/NativeTagAwareAdapters.md @@ -15,23 +15,23 @@ See: https://github.com/symfony/symfony/commits/master/src/Symfony/Component/Cac ## Requirements - Symfony 3.4, PHP 7.1+ - For usage eZ Platform v2: `ezsystems/ezpublish-kernel` v7.3.5, v7.4.3 or higher. -- For `FilesystemTagAwareAdapter` usage: _No particular needs._ - For `RedisTagAwareAdapter` usage: - [PHP Redis](https://pecl.php.net/package/redis) extension v3.1.3 or higher, _or_ [Predis](https://packagist.org/packages/predis/predis) - Redis 3.2 or higher, configured with `noeviction` or any `volatile-*` eviction policy ## Configuration After installing the bundle, you have to configure proper services in order to use this. -Here is an example on how to do that with eZ Platform: + +**Here is an example on how to do that with eZ Platform:** ### File system cache -In `app/config/cache_pool/app.cache.filesystem.yml`, place the following: +In `app/config/cache_pool/app.cache.tagaware.filesystem.yml`, place the following: ```yaml services: - app.cache.filesystem: - class: EzSystems\SymfonyTools\Incubator\Cache\TagAware\FilesystemTagAwareAdapter + app.cache.tagaware.filesystem: + class: Symfony\Component\Cache\Adapter\TagAware\FilesystemTagAwareAdapter parent: cache.adapter.filesystem tags: - name: cache.pool @@ -47,7 +47,7 @@ services: Once that is done you can enable the handler, for instance by setting the following environment variable for PHP: ```bash -export CACHE_POOL="app.cache.filesystem" +export CACHE_POOL="app.cache.tagaware.filesystem" ``` _Then clear cache and restart web server, you'll be able to verify it's in use on Symfony's web debug toolbar._ @@ -55,11 +55,11 @@ _Then clear cache and restart web server, you'll be able to verify it's in use o ### Redis cache -In `app/config/cache_pool/app.cache.redis.yml`, place the following: +In `app/config/cache_pool/app.cache.tagaware.redis.yml`, place the following: ```yaml services: - app.cache.redis: - class: EzSystems\SymfonyTools\Incubator\Cache\TagAware\RedisTagAwareAdapter + app.cache.tagaware.redis: + class: Symfony\Component\Cache\Adapter\TagAware\RedisTagAwareAdapter parent: cache.adapter.redis tags: - name: cache.pool @@ -81,7 +81,7 @@ services: Once that is done you can enable the handler, for instance by setting the following environment variable for PHP: ```bash -export CACHE_POOL="app.cache.redis" +export CACHE_POOL="app.cache.tagaware.redis" ``` If you don't have redis, for testing you can use: - Run: `docker run --name my-redis -p 6379:6379 -d redis`. diff --git a/src/lib/Incubator/Cache/TagAware/AbstractTagAwareAdapter.php b/src/lib/Symfony/Components/Cache/Adapter/TagAware/AbstractTagAwareAdapter.php similarity index 93% rename from src/lib/Incubator/Cache/TagAware/AbstractTagAwareAdapter.php rename to src/lib/Symfony/Components/Cache/Adapter/TagAware/AbstractTagAwareAdapter.php index f02aa99..e9a1725 100644 --- a/src/lib/Incubator/Cache/TagAware/AbstractTagAwareAdapter.php +++ b/src/lib/Symfony/Components/Cache/Adapter/TagAware/AbstractTagAwareAdapter.php @@ -1,11 +1,17 @@ + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. */ -namespace EzSystems\SymfonyTools\Incubator\Cache\TagAware; +declare(strict_types=1); + +namespace Symfony\Component\Cache\Adapter\TagAware; use Psr\Cache\CacheItemInterface; use Psr\Log\LoggerAwareInterface; @@ -35,7 +41,7 @@ abstract class AbstractTagAwareAdapter implements AdapterInterface, LoggerAwareI * @var \Symfony\Component\Cache\Marshaller\MarshallerInterface * NOTE: Not relevant in this way in Symfony 4+ where Abstract trait already uses this */ - protected $marshaller; + protected static $marshaller; /** * @param string $namespace @@ -44,9 +50,9 @@ abstract class AbstractTagAwareAdapter implements AdapterInterface, LoggerAwareI * * @throws \Symfony\Component\Cache\Exception\CacheException */ - protected function __construct($namespace = '', $defaultLifetime = 0, MarshallerInterface $marshaller = null) + protected function __construct(string $namespace = '', int $defaultLifetime = 0, MarshallerInterface $marshaller = null) { - $this->marshaller = $marshaller ?? new DefaultMarshaller(); + self::$marshaller = $marshaller ?? new DefaultMarshaller(); $this->namespace = '' === $namespace ? '' : CacheItem::validateKey($namespace) . ':'; if (null !== $this->maxIdLength && \strlen($namespace) > $this->maxIdLength - 24) { @@ -261,6 +267,6 @@ private function generateItems($items, &$keys) */ protected static function unserialize($value) { - throw new \Exception('Not in use, use $this->marshaller'); + return self::$marshaller->unmarshall($value); } } diff --git a/src/lib/Incubator/Cache/TagAware/FilesystemTagAwareAdapter.php b/src/lib/Symfony/Components/Cache/Adapter/TagAware/FilesystemTagAwareAdapter.php similarity index 70% rename from src/lib/Incubator/Cache/TagAware/FilesystemTagAwareAdapter.php rename to src/lib/Symfony/Components/Cache/Adapter/TagAware/FilesystemTagAwareAdapter.php index 14a3f6a..a599464 100644 --- a/src/lib/Incubator/Cache/TagAware/FilesystemTagAwareAdapter.php +++ b/src/lib/Symfony/Components/Cache/Adapter/TagAware/FilesystemTagAwareAdapter.php @@ -1,13 +1,17 @@ + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. */ declare(strict_types=1); -namespace EzSystems\SymfonyTools\Incubator\Cache\TagAware; +namespace Symfony\Component\Cache\Adapter\TagAware; use Symfony\Component\Cache\Adapter\TagAwareAdapterInterface; use Symfony\Component\Cache\Traits\FilesystemTrait; @@ -35,7 +39,7 @@ final class FilesystemTagAwareAdapter extends AbstractTagAwareAdapter implements */ private $fs; - public function __construct($namespace = '', $defaultLifetime = 0, $directory = null) + public function __construct(string $namespace = '', int $defaultLifetime = 0, string $directory = null) { parent::__construct('', $defaultLifetime); $this->init($namespace, $directory); @@ -53,7 +57,7 @@ public function __construct($namespace = '', $defaultLifetime = 0, $directory = protected function doSave(array $values, $lifetime) { $failed = []; - $serialized = $this->marshaller->marshall($values, $failed); + $serialized = self::$marshaller->marshall($values, $failed); if (empty($serialized)) { return $failed; } @@ -88,39 +92,6 @@ protected function doSave(array $values, $lifetime) return $failed; } - /** - * This method overrides {@see \Symfony\Component\Cache\Traits\FilesystemTrait::doFetch}. - * - * It needs to be overridden due to the usage of `parent::unserialize()` in the original method. - * - * {@inheritdoc} - */ - protected function doFetch(array $ids) - { - $values = []; - $now = time(); - - foreach ($ids as $id) { - $file = $this->getFile($id); - if (!file_exists($file) || !$h = @fopen($file, 'rb')) { - continue; - } - if (($expiresAt = (int) fgets($h)) && $now >= $expiresAt) { - fclose($h); - @unlink($file); - } else { - $i = rawurldecode(rtrim(fgets($h))); - $value = stream_get_contents($h); - fclose($h); - if ($i === $id) { - $values[$id] = $this->marshaller->unmarshall($value); - } - } - } - - return $values; - } - /** * {@inheritdoc} */ @@ -138,7 +109,7 @@ public function invalidateTags(array $tags) foreach (new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($this->getTagFolder($tag), \FilesystemIterator::SKIP_DOTS)) as $itemLink) { if (!$itemLink->isLink()) { - throw new \Exception('Tag link is not a link: ' . $itemLink); + throw new \Exception('Tag link is not a link: '.$itemLink); } $valueFile = $itemLink->getRealPath(); @@ -146,7 +117,7 @@ public function invalidateTags(array $tags) @unlink($valueFile); } - @unlink((string)$itemLink); + @unlink((string) $itemLink); } } @@ -164,13 +135,13 @@ private function getFile($id, $mkdir = false) { // Use MD5 to favor speed over security, which is not an issue here $hash = str_replace('/', '-', base64_encode(hash('md5', static::class . $id, true))); - $dir = $this->directory.strtoupper($hash[0] . \DIRECTORY_SEPARATOR . $hash[1] . \DIRECTORY_SEPARATOR); + $dir = $this->directory.strtoupper($hash[0].\DIRECTORY_SEPARATOR.$hash[1].\DIRECTORY_SEPARATOR); if ($mkdir && !file_exists($dir)) { @mkdir($dir, 0777, true); } - return $dir . substr($hash, 2, 20); + return $dir.substr($hash, 2, 20); } private function getFilesystem(): Filesystem @@ -180,22 +151,14 @@ private function getFilesystem(): Filesystem private function getTagFolder($tag): string { - return $this->directory . self::TAG_FOLDER . \DIRECTORY_SEPARATOR . str_replace('/', '-', $tag) . \DIRECTORY_SEPARATOR; + return $this->directory.self::TAG_FOLDER.\DIRECTORY_SEPARATOR.str_replace('/', '-', $tag).\DIRECTORY_SEPARATOR; } private function getTagIdFile($id): string { // Use MD5 to favor speed over security, which is not an issue here - $hash = str_replace('/', '-', base64_encode(hash('md5', static::class . $id, true))); + $hash = str_replace('/', '-', base64_encode(hash('md5', static::class.$id, true))); return substr($hash, 0, 20); } - - /** - * @internal for unit tests only - */ - public function setFilesystem(Filesystem $fs) - { - $this->fs = $fs; - } } diff --git a/src/lib/Incubator/Cache/TagAware/RedisTagAwareAdapter.php b/src/lib/Symfony/Components/Cache/Adapter/TagAware/RedisTagAwareAdapter.php similarity index 59% rename from src/lib/Incubator/Cache/TagAware/RedisTagAwareAdapter.php rename to src/lib/Symfony/Components/Cache/Adapter/TagAware/RedisTagAwareAdapter.php index d5f60ff..76a7767 100644 --- a/src/lib/Incubator/Cache/TagAware/RedisTagAwareAdapter.php +++ b/src/lib/Symfony/Components/Cache/Adapter/TagAware/RedisTagAwareAdapter.php @@ -1,13 +1,17 @@ + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. */ declare(strict_types=1); -namespace EzSystems\SymfonyTools\Incubator\Cache\TagAware; +namespace Symfony\Component\Cache\Adapter\TagAware; use Predis; use Predis\Response\Status; @@ -18,21 +22,26 @@ * Class RedisTagAwareAdapter, stores tag <> id relationship as a Set so we don't need to fetch tags on get* operations. * * Requirements/Limitations: + * - Redis 3.2+ (sPOP) + * - PHP Redis 3.1.3+ (sPOP) or Predis * - Redis configured with any `volatile-*` eviction policy, or `noeviction` if you are sure to never fill up memory * - This is to guarantee that tags ("relations") survives cache items so we can reliably invalidate on them, * which is archived by always storing cache with a expiry, while Set is without expiry (non-volatile). - * - As we use Redis "SPOP" command with count argument for invalidation: - * - Redis 3.2 or higher - * - PHP Redis 3.1.3 or higher, or Predis + * - Max 2 billion keys per tag, so if you use a "all" items tag for expiry, that limits you to 2 billion items */ final class RedisTagAwareAdapter extends AbstractTagAwareAdapter implements TagAwareAdapterInterface { use RedisTrait; /** - * Limit for how many items are popped from tags per iteration to not run out of memory pipelining deletes. + * Redis "Set" can hold more than 4 billion members, here we limit ourselves to PHP's > 2 billion max int (32Bit). + */ + private const POP_MAX_LIMIT = 2147483647 - 1; + + /** + * Limits for how many keys are deleted in batch. */ - private const BULK_INVALIDATION_POP_LIMIT = 10000; + private const BULK_DELETE_LIMIT = 10000; /** * On cache items without a lifetime set, we force it to 10 days. @@ -44,6 +53,8 @@ final class RedisTagAwareAdapter extends AbstractTagAwareAdapter implements TagA * @param \Redis|\RedisArray|\RedisCluster|\Predis\Client $redisClient The redis client * @param string $namespace The default namespace * @param int $defaultLifetime The default lifetime + * + * @throws \Exception If phpredis is in use but with version lower then 3.1.3. */ public function __construct($redisClient, string $namespace = '', int $defaultLifetime = 0) { @@ -51,7 +62,7 @@ public function __construct($redisClient, string $namespace = '', int $defaultLi // Make sure php-redis is 3.1.3 or higher configured for Redis classes if (!$this->redis instanceof Predis\Client && version_compare(phpversion('redis'), '3.1.3', '<')) { - throw new \Exception('RedisTagAwareAdapter requries php-redis 3.1.3 or higher, alternativly use predis/predis'); + throw new \Exception('RedisTagAwareAdapter requries php-redis 3.1.3 or higher, alternatively use predis/predis'); } } @@ -66,7 +77,7 @@ public function __construct($redisClient, string $namespace = '', int $defaultLi */ protected function doSave(array $values, $lifetime) { - if (!$serialized = $this->marshaller->marshall($values, $failed)) { + if (!$serialized = self::$marshaller->marshall($values, $failed)) { return $failed; } @@ -74,13 +85,13 @@ protected function doSave(array $values, $lifetime) $tagSets = []; foreach ($values as $id => $value) { foreach ($value['tags'] as $tag) { - $tagSets[$this->getId(self::TAGS_PREFIX . $tag)][] = $id; + $tagSets[$this->getId(self::TAGS_PREFIX.$tag)][] = $id; } } // While pipeline isn't supported on RedisCluster, other setups will at least benefit from doing this in one op $results = $this->pipeline(function () use ($serialized, $lifetime, $tagSets) { - // 1: Store cache items, force a ttl if none is set, as there is no MSETEX we need to set each one + // Store cache items, force a ttl if none is set, as there is no MSETEX we need to set each one foreach ($serialized as $id => $value) { yield 'setEx' => [ $id, @@ -89,7 +100,7 @@ protected function doSave(array $values, $lifetime) ]; } - // 2: append tag sets, method to add with array values differs on PHP Redis clients + // Append tag sets, method to add with array values differs on PHP Redis clients $method = $this->redis instanceof Predis\Client ? 'sAdd' : 'sAddArray'; foreach ($tagSets as $tagId => $ids) { yield $method => [$tagId, $ids]; @@ -110,26 +121,6 @@ protected function doSave(array $values, $lifetime) return $failed; } - /** - * This method overrides @see \Symfony\Component\Cache\Traits\RedisTrait::doFetch in order to use mget & marshaller. - * - * {@inheritdoc} - */ - protected function doFetch(array $ids) - { - if (empty($ids)) { - return []; - } - - // Using MGET for speed on RedisCluster as pipeline is not supported there - foreach ($this->redis->mget($ids) as $key => $v) { - // Not found items will have value as false, key will be same as on $ids - if ($v) { - yield $ids[$key] => $this->marshaller->unmarshall($v); - } - } - } - /** * @param array $tags * @@ -141,25 +132,22 @@ public function invalidateTags(array $tags) return; } - // Retrieve and delete items in bulk of 10.000 at a time to not overflow buffers - // - // NOTE: Nicolas wants to look into rather finding a way to do invalidation with Lua on the Redis server. - // Reason is that the design here can risk ending up in a endless loop if a items are rapidly added - // with the tag(s) we try to invalidate. On RedisCluster a Lua approach would need to run on all servers. - do { - $tagIdSets = $this->pipeline(function () use ($tags) { - foreach (array_unique($tags) as $tag) { - // Requires Predis or PHP Redis 3.1.3+ (https://github.com/phpredis/phpredis/commit/d2e203a6) - yield 'sPop' => [$this->getId(self::TAGS_PREFIX . $tag), self::BULK_INVALIDATION_POP_LIMIT]; - } - }); - - // flatten generator result from pipleline - $ids = array_merge(...iterator_to_array($tagIdSets, false)); - - // delete ids - $this->doDelete(array_unique($ids)); - } while (count($ids) >= self::BULK_INVALIDATION_POP_LIMIT); + // Pop all tag info at once to avoid race conditions + $tagIdSets = $this->pipeline(function () use ($tags) { + foreach (array_unique($tags) as $tag) { + // Requires Predis or PHP Redis 3.1.3+: https://github.com/phpredis/phpredis/commit/d2e203a6 + // And Redis 3.2: https://redis.io/commands/spop + yield 'sPop' => [$this->getId(self::TAGS_PREFIX.$tag), self::POP_MAX_LIMIT]; + } + }); + + // Flatten generator result from pipleline, ignore keys (tag ids) + $ids = array_unique(array_merge(...iterator_to_array($tagIdSets, false))); + + // Delete chunks of id's + while (!empty($ids)) { + $this->doDelete(\array_slice($ids, 0, self::BULK_DELETE_LIMIT)); + } return true; }