diff --git a/cache/stores/redis/addinstanceform.php b/cache/stores/redis/addinstanceform.php index ad9bc0053594b..1a3faa16c37bc 100644 --- a/cache/stores/redis/addinstanceform.php +++ b/cache/stores/redis/addinstanceform.php @@ -58,5 +58,11 @@ protected function configuration_definition() { $form->addHelpButton('serializer', 'useserializer', 'cachestore_redis'); $form->setDefault('serializer', Redis::SERIALIZER_PHP); $form->setType('serializer', PARAM_INT); + + $compressoroptions = cachestore_redis::config_get_compressor_options(); + $form->addElement('select', 'compressor', get_string('usecompressor', 'cachestore_redis'), $compressoroptions); + $form->addHelpButton('compressor', 'usecompressor', 'cachestore_redis'); + $form->setDefault('compressor', cachestore_redis::COMPRESSOR_NONE); + $form->setType('compressor', PARAM_INT); } } diff --git a/cache/stores/redis/lang/en/cachestore_redis.php b/cache/stores/redis/lang/en/cachestore_redis.php index 0d155be68ba77..c9afbbd84f7b9 100644 --- a/cache/stores/redis/lang/en/cachestore_redis.php +++ b/cache/stores/redis/lang/en/cachestore_redis.php @@ -24,6 +24,8 @@ defined('MOODLE_INTERNAL') || die(); +$string['compressor_none'] = 'No compression.'; +$string['compressor_php_gzip'] = 'Use gzip compression.'; $string['pluginname'] = 'Redis'; $string['prefix'] = 'Key prefix'; $string['prefix_help'] = 'This prefix is used for all key names on the Redis server. @@ -48,3 +50,5 @@ $string['useserializer_help'] = 'Specifies the serializer to use for serializing. The valid serializers are Redis::SERIALIZER_PHP or Redis::SERIALIZER_IGBINARY. The latter is supported only when phpredis is configured with --enable-redis-igbinary option and the igbinary extension is loaded.'; +$string['usecompressor'] = 'Use compressor'; +$string['usecompressor_help'] = 'Specifies the compressor to use after serializing. It is done at Moodle Cache API level, not at php-redis level.'; diff --git a/cache/stores/redis/lib.php b/cache/stores/redis/lib.php index 35962002b95ca..1b908dda171d1 100644 --- a/cache/stores/redis/lib.php +++ b/cache/stores/redis/lib.php @@ -38,6 +38,16 @@ */ class cachestore_redis extends cache_store implements cache_is_key_aware, cache_is_lockable, cache_is_configurable, cache_is_searchable { + /** + * Compressor: none. + */ + const COMPRESSOR_NONE = 0; + + /** + * Compressor: PHP GZip. + */ + const COMPRESSOR_PHP_GZIP = 1; + /** * Name of this store. * @@ -80,6 +90,13 @@ class cachestore_redis extends cache_store implements cache_is_key_aware, cache_ */ protected $serializer = Redis::SERIALIZER_PHP; + /** + * Compressor for this store. + * + * @var int + */ + protected $compressor = self::COMPRESSOR_NONE; + /** * Determines if the requirements for this type of store are met. * @@ -134,6 +151,9 @@ public function __construct($name, array $configuration = array()) { if (array_key_exists('serializer', $configuration)) { $this->serializer = (int)$configuration['serializer']; } + if (array_key_exists('compressor', $configuration)) { + $this->compressor = (int)$configuration['compressor']; + } $password = !empty($configuration['password']) ? $configuration['password'] : ''; $prefix = !empty($configuration['prefix']) ? $configuration['prefix'] : ''; $this->redis = $this->new_redis($configuration['server'], $prefix, $password); @@ -161,7 +181,10 @@ protected function new_redis($server, $prefix = '', $password = '') { if (!empty($password)) { $redis->auth($password); } - $redis->setOption(Redis::OPT_SERIALIZER, $this->serializer); + // If using compressor, serialisation will be done at cachestore level, not php-redis. + if ($this->compressor == self::COMPRESSOR_NONE) { + $redis->setOption(Redis::OPT_SERIALIZER, $this->serializer); + } if (!empty($prefix)) { $redis->setOption(Redis::OPT_PREFIX, $prefix); } @@ -236,7 +259,13 @@ public function is_ready() { * @return mixed The value of the key, or false if there is no value associated with the key. */ public function get($key) { - return $this->redis->hGet($this->hash, $key); + $value = $this->redis->hGet($this->hash, $key); + + if ($this->compressor == self::COMPRESSOR_NONE) { + return $value; + } + + return $this->uncompress($value); } /** @@ -246,7 +275,17 @@ public function get($key) { * @return array An array of the values of the given keys. */ public function get_many($keys) { - return $this->redis->hMGet($this->hash, $keys); + $values = $this->redis->hMGet($this->hash, $keys); + + if ($this->compressor == self::COMPRESSOR_NONE) { + return $values; + } + + foreach ($values as &$value) { + $value = $this->uncompress($value); + } + + return $values; } /** @@ -257,6 +296,8 @@ public function get_many($keys) { * @return bool True if the operation succeeded, false otherwise. */ public function set($key, $value) { + $value = $this->compress($value); + return ($this->redis->hSet($this->hash, $key, $value) !== false); } @@ -270,7 +311,8 @@ public function set($key, $value) { public function set_many(array $keyvaluearray) { $pairs = []; foreach ($keyvaluearray as $pair) { - $pairs[$pair['key']] = $pair['value']; + $key = $pair['key']; + $pairs[$key] = $this->compress($pairs[$key]); } if ($this->redis->hMSet($this->hash, $pairs)) { return count($pairs); @@ -446,7 +488,8 @@ public static function config_get_configuration_array($data) { 'server' => $data->server, 'prefix' => $data->prefix, 'password' => $data->password, - 'serializer' => $data->serializer + 'serializer' => $data->serializer, + 'compressor' => $data->compressor, ); } @@ -465,6 +508,9 @@ public static function config_set_edit_form_data(moodleform $editform, array $co if (!empty($config['serializer'])) { $data['serializer'] = $config['serializer']; } + if (!empty($config['compressor'])) { + $data['compressor'] = $config['compressor']; + } $editform->set_data($data); } @@ -538,4 +584,104 @@ public static function config_get_serializer_options() { } return $options; } + + /** + * Gets an array of options to use as the compressor. + * + * @return array + */ + public static function config_get_compressor_options() { + return [ + self::COMPRESSOR_NONE => get_string('compressor_none', 'cachestore_redis'), + self::COMPRESSOR_PHP_GZIP => get_string('compressor_php_gzip', 'cachestore_redis'), + ]; + } + + /** + * Compress the given value, serializing it first. + * + * @param mixed $value + * @return string + */ + private function compress($value) { + $value = $this->serialize($value); + + switch ($this->compressor) { + case self::COMPRESSOR_NONE: + return $value; + + case self::COMPRESSOR_PHP_GZIP: + return gzencode($value); + + default: + debugging("Invalid compressor: {$this->compressor}"); + return $value; + } + } + + /** + * Uncompresses (deflates) the data, unserialising it afterwards. + * + * @param string $value + * @return mixed + */ + private function uncompress($value) { + if ($value === false) { + return false; + } + + if ($this->compressor == self::COMPRESSOR_NONE) { + return $value; + } + + switch ($this->compressor) { + case self::COMPRESSOR_PHP_GZIP: + $value = gzdecode($value); + break; + default: + debugging("Invalid compressor: {$this->compressor}"); + } + + return $this->unserialize($value); + } + + /** + * Serializes the data according to the configured serializer. + * + * @param mixed $value + * @return string + */ + private function serialize($value) { + switch ($this->serializer) { + case Redis::SERIALIZER_NONE: + return $value; + case Redis::SERIALIZER_PHP: + return serialize($value); + case Redis::SERIALIZER_IGBINARY: + return igbinary_serialize($value); + default: + debugging("Invalid serializer: {$this->serializer}"); + return $value; + } + } + + /** + * Unserializes the data according to the configured serializer + * + * @param string $value + * @return mixed + */ + private function unserialize($value) { + switch ($this->serializer) { + case Redis::SERIALIZER_NONE: + return $value; + case Redis::SERIALIZER_PHP: + return unserialize($value); + case Redis::SERIALIZER_IGBINARY: + return igbinary_unserialize($value); + default: + debugging("Invalid serializer: {$this->serializer}"); + return $value; + } + } } diff --git a/cache/stores/redis/tests/compressor_test.php b/cache/stores/redis/tests/compressor_test.php new file mode 100644 index 0000000000000..900ffaf4ae54a --- /dev/null +++ b/cache/stores/redis/tests/compressor_test.php @@ -0,0 +1,404 @@ +. + +/** + * Redis cache test. + * + * If you wish to use these unit tests all you need to do is add the following definition to + * your config.php file. + * + * define('TEST_CACHESTORE_REDIS_TESTSERVERS', '127.0.0.1'); + * + * @package cachestore_redis + * @copyright 2018 Catalyst IT Australia {@link http://www.catalyst-au.net} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +require_once(__DIR__.'/../../../tests/fixtures/stores.php'); +require_once(__DIR__.'/../lib.php'); + +/** + * Redis cache test - compressor settings. + * + * @package cachestore_redis + * @author Daniel Thee Roperto + * @copyright 2018 Catalyst IT Australia {@link http://www.catalyst-au.net} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class cachestore_redis_compressor_test extends advanced_testcase { + /** + * @var cachestore_redis + */ + protected $store = null; + + /** + * Create a cachestore. + * + * @param int $compressor + * @param int|null $serializer + * @return cachestore_redis|null + */ + public function create_store($compressor, $serializer = null) { + if (!cachestore_redis::are_requirements_met() || !defined('TEST_CACHESTORE_REDIS_TESTSERVERS')) { + $this->markTestSkipped('Could not test cachestore_redis. Requirements are not met.'); + return null; + } + + if (is_null($serializer)) { + $serializer = Redis::SERIALIZER_PHP; + } + + /** @var cache_definition $definition */ + $definition = cache_definition::load_adhoc(cache_store::MODE_APPLICATION, 'cachestore_redis', 'phpunit_test'); + $config = cachestore_redis::unit_test_configuration(); + $config['compressor'] = $compressor; + $config['serializer'] = $serializer; + $store = new cachestore_redis('Test', $config); + $store->initialise($definition); + + return $store; + } + + /** + * Create a cache store. + */ + public function setUp() { + parent::setUp(); + + $this->store = $this->create_store(cachestore_redis::COMPRESSOR_PHP_GZIP); + } + + /** + * Destroy cache store. + */ + protected function tearDown() { + parent::tearDown(); + + if ($this->store instanceof cachestore_redis) { + $this->store->purge(); + } + } + + /** + * Set a value. + */ + public function test_it_can_set() { + if (is_null($this->store)) { + return; + } + + $this->store->set('the key', 'the value'); + $expected = gzencode(serialize('the value')); + + // Disable compressor to check stored value. + $rawstore = $this->create_store(cachestore_redis::COMPRESSOR_NONE, Redis::SERIALIZER_NONE); + $actual = $rawstore->get('the key'); // Compressor was disabled. + + self::assertSame($expected, $actual); + } + + /** + * Set many values. + */ + public function test_it_can_set_many() { + if (is_null($this->store)) { + return; + } + + // Create values. + $values = []; + for ($i = 0; $i < 10; $i++) { + $values[] = [ + 'key' => "key_{$i}", + 'value' => "value #{$i}", + ]; + } + + // Store it. + $this->store->set_many($values); + + // Disable compressor to check stored value. + $rawstore = $this->create_store(cachestore_redis::COMPRESSOR_NONE, Redis::SERIALIZER_NONE); + + foreach ($values as $value) { + $expected = gzencode(serialize($value['value'])); + $actual = $rawstore->get($value['key']); // Compressor was disabled. + self::assertSame($expected, $actual, "Invalid value for key={$value['key']}"); + } + } + + /** + * Gets a value. + */ + public function test_it_can_get() { + if (is_null($this->store)) { + return; + } + + $this->store->set('the key', 'the value'); + $actual = $this->store->get('the key'); + self::assertSame('the value', $actual); + } + + /** + * Gets many values. + */ + public function test_it_can_get_many() { + if (is_null($this->store)) { + return; + } + + // Create values. + $values = []; + $keys = []; + $expected = []; + for ($i = 0; $i < 10; $i++) { + $key = "getkey_{$i}"; + $value = "getvalue #{$i}"; + $keys[] = $key; + $values[] = [ + 'key' => $key, + 'value' => $value, + ]; + $expected[$key] = $value; + } + + $this->store->set_many($values); + $actual = $this->store->get_many($keys); + self::assertSame($expected, $actual); + } + + /** + * It misses a value. + */ + public function test_it_can_miss_one() { + if (is_null($this->store)) { + return; + } + $actual = $this->store->get('missme'); + self::assertFalse($actual); + } + + /** + * It misses many values. + */ + public function test_it_can_miss_many() { + if (is_null($this->store)) { + return; + } + $expected = ['missme' => false, 'missmetoo' => false]; + $actual = $this->store->get_many(array_keys($expected)); + self::assertSame($expected, $actual); + } + + /** + * It misses some values. + */ + public function test_it_can_miss_some() { + if (is_null($this->store)) { + return; + } + + $this->store->set('iamhere', 'youfoundme'); + + $expected = ['missme' => false, 'missmetoo' => false, 'iamhere' => 'youfoundme']; + $actual = $this->store->get_many(array_keys($expected)); + self::assertSame($expected, $actual); + } + + /** + * A provider for test_works_with_different_types + * + * @return array + */ + public function provider_for_test_it_works_with_different_types() { + $object = new stdClass(); + $object->field = 'value'; + + return [ + ['string', 'Abc Def'], + ['string_empty', ''], + ['string_binary', gzencode('some binary data')], + ['int', 123], + ['int_zero', 0], + ['int_negative', -100], + ['int_huge', PHP_INT_MAX], + ['float', 3.14], + ['boolean_true', true], + // Boolean 'false' is not tested as it is not allowed in Moodle. + ['array', [1, 'b', 3.4]], + ['array_map', ['a' => 'b', 'c' => 'd']], + ['object_stdClass', $object], + ['null', null], + ]; + } + + /** + * It works with different types. + * + * @dataProvider provider_for_test_it_works_with_different_types + * @param string $key + * @param mixed $value + */ + public function test_it_works_with_different_types($key, $value) { + if (is_null($this->store)) { + return; + } + + $this->store->set($key, $value); + $actual = $this->store->get($key); + self::assertEquals($value, $actual, "Failed set/get for: {$key}"); + } + + /** + * Test it works with different types for many. + */ + public function test_it_works_with_different_types_for_many() { + if (is_null($this->store)) { + return; + } + + $provider = $this->provider_for_test_it_works_with_different_types(); + $keys = []; + $values = []; + $expected = []; + foreach ($provider as $item) { + $keys[] = $item[0]; + $values[] = ['key' => $item[0], 'value' => $item[1]]; + $expected[$item[0]] = $item[1]; + } + + $this->store->set_many($values); + $actual = $this->store->get_many($keys); + self::assertEquals($expected, $actual); + } + + /** + * Test it does not use PHP Redis serialization if compression is on. + */ + public function test_it_does_not_use_phpredis_serialisation() { + if (is_null($this->store)) { + return; // Redis not enabled. + } + + $this->store->set('my key', 'my value'); + + // Create a connection without serialisation or compressor to fetch raw data. + $rawstore = $this->create_store(cachestore_redis::COMPRESSOR_NONE, Redis::SERIALIZER_NONE); + + $rawdata = $rawstore->get('my key'); + $expected = gzencode(serialize('my value')); // It should not have an extra serialisation. + self::assertSame($expected, $rawdata); + } + + /** + * Provider for serializer tests. + * + * @return array + */ + public function provider_for_test_it_can_use_serializers() { + if (!class_exists('Redis')) { + return []; + } + + $data = [ + ['none', Redis::SERIALIZER_NONE, gzencode('value1'), gzencode('value2')], + ['php', Redis::SERIALIZER_PHP, gzencode(serialize('value1')), gzencode(serialize('value2'))], + ]; + + if (defined('Redis::SERIALIZER_IGBINARY')) { + $data[] = [ + 'igbinary', + Redis::SERIALIZER_IGBINARY, + gzencode(igbinary_serialize('value1')), + gzencode(igbinary_serialize('value2')), + ]; + } + + return $data; + } + + /** + * Test it can use serializers with get and set. + * + * @dataProvider provider_for_test_it_can_use_serializers + * @param string $name + * @param int $serializer + * @param string $rawexpected1 + * @param string $rawexpected2 + */ + public function test_it_can_use_serializers_getset($name, $serializer, $rawexpected1, $rawexpected2) { + if (is_null($this->store)) { + return; // Redis not enabled. + } + + // Create a connection with the desired serialisation. + $store = $this->create_store(cachestore_redis::COMPRESSOR_PHP_GZIP, $serializer); + + // Create a connection without serialisation or compressor to fetch raw data. + $rawstore = $this->create_store(cachestore_redis::COMPRESSOR_NONE, Redis::SERIALIZER_NONE); + + $store->set('key', 'value1'); + $data = $store->get('key'); + $rawdata = $rawstore->get('key'); + self::assertSame('value1', $data, "Invalid serialisation/unserialisation for: {$name}"); + self::assertSame($rawexpected1, $rawdata, "Invalid rawdata for: {$name}"); + } + + /** + * Test it can use serializers with get and set many. + * + * @dataProvider provider_for_test_it_can_use_serializers + * @param string $name + * @param int $serializer + * @param string $rawexpected1 + * @param string $rawexpected2 + */ + public function test_it_can_use_serializers_getsetmany($name, $serializer, $rawexpected1, $rawexpected2) { + if (is_null($this->store)) { + return; // Redis not enabled. + } + + $many = [ + ['key' => 'key1', 'value' => 'value1'], + ['key' => 'key2', 'value' => 'value2'], + ]; + $keys = ['key1', 'key2']; + $expectations = ['key1' => 'value1', 'key2' => 'value2']; + $rawexpectations = ['key1' => $rawexpected1, 'key2' => $rawexpected2]; + + // Create a connection with the desired serialisation. + $store = $this->create_store(cachestore_redis::COMPRESSOR_PHP_GZIP, $serializer); + $store->set_many($many); + + // Create a connection without serialisation or compressor to fetch raw data. + $rawstore = $this->create_store(cachestore_redis::COMPRESSOR_NONE, Redis::SERIALIZER_NONE); + + $data = $store->get_many($keys); + $rawdata = $rawstore->get_many($keys); + + foreach ($keys as $key) { + self::assertSame($expectations[$key], + $data[$key], + "Invalid serialisation/unserialisation for {$key} with serializer {$name}"); + self::assertSame($rawexpectations[$key], + $rawdata[$key], + "Invalid rawdata for {$key} with serializer {$name}"); + } + } +}