Skip to content

Commit

Permalink
MDL-42071 caching Support for clustered memcached caching
Browse files Browse the repository at this point in the history 
For stores where there is a very high rate of sets compared to gets, it
is beneficial to retrieve from the local server, skipping the network
overhead, at the expense of having to set many severs when a key is set.

This patch adds a memcached option to enable clustering. When on, only
one "server" is allowed, which will be where fetches are from, while
sets/updates/deletes/purges will occur to the all the servers in the
"set server" list.

To run unit tests, define TEST_CACHESTORE_MEMCACHED_TESTSERVERS with
multiple (return delimited) servers.
  • Loading branch information
@ericmerrill
ericmerrill committed Jun 30, 2014
1 parent 7a4832e commit 1c0518a
Show file tree
Hide file tree
Showing 5 changed files with 410 additions and 7 deletions.
44 changes: 44 additions & 0 deletions cache/stores/memcached/addinstanceform.php
View file
Expand Up @@ -74,5 +74,49 @@ protected function configuration_definition() {
$form->addHelpButton('bufferwrites', 'bufferwrites', 'cachestore_memcached');
$form->setDefault('bufferwrites', 0);
$form->setType('bufferwrites', PARAM_BOOL);

$form->addElement('header', 'clusteredheader', get_string('clustered', 'cachestore_memcached'));

$form->addElement('checkbox', 'clustered', get_string('clustered', 'cachestore_memcached'));
$form->setDefault('checkbox', false);
$form->addHelpButton('clustered', 'clustered', 'cachestore_memcached');

$form->addElement('textarea', 'setservers', get_string('setservers', 'cachestore_memcached'),
array('cols' => 75, 'rows' => 5));
$form->addHelpButton('setservers', 'setservers', 'cachestore_memcached');
$form->disabledIf('setservers', 'clustered');
$form->setType('setservers', PARAM_RAW);
}

/**
* Perform minimal validation on the settings form.
*
* @param array $data
* @param array $files
*/
public function validation($data, $files) {
$errors = parent::validation($data, $files);

if (isset($data['clustered']) && ($data['clustered'] == 1)) {
// Set servers is required with in cluster mode.
if (!isset($data['setservers']) || empty(trim($data['setservers']))) {
$errors['setservers'] = get_string('required');
}

$validservers = false;
if (isset($data['servers'])) {
$servers = trim($data['servers']);
$servers = explode("\n", $servers);
if (count($servers) === 1) {
$validservers = true;
}
}

if (!$validservers) {
$errors['servers'] = get_string('serversclusterinvalid', 'cachestore_memcached');
}
}

return $errors;
}
}
21 changes: 21 additions & 0 deletions cache/stores/memcached/lang/en/cachestore_memcached.php
View file
Expand Up @@ -28,6 +28,13 @@

$string['bufferwrites'] = 'Buffer writes';
$string['bufferwrites_help'] = 'Enables or disables buffered I/O. Enabling buffered I/O causes storage commands to "buffer" instead of being sent. Any action that retrieves data causes this buffer to be sent to the remote connection. Quitting the connection or closing down the connection will also cause the buffered data to be pushed to the remote connection.';
$string['clustered'] = 'Enable clustered servers';
$string['clustered_help'] = 'This is used to allow read-one, set-multi functionality.
The intended use case is to create an improved store for load-balanced configurations. The store will fetch from one server (usually localhost), but set to many (all the servers in the load-balance pool). For caches with very high read to set ratios, this saves a significant amount of network overhead.
When this setting is enabled, the server listed above will be used for fetching.';
$string['clusteredheader'] = 'Split servers';
$string['hash'] = 'Hash method';
$string['hash_help'] = 'Specifies the hashing algorithm used for the item keys. Each hash algorithm has its advantages and its disadvantages. Go with the default if you don\'t know or don\'t care.';
$string['hash_default'] = 'Default (one-at-a-time)';
Expand Down Expand Up @@ -56,6 +63,20 @@
server.url.com
ipaddress:port
servername:port:weight
</pre>
If *Enable clustered servers* is enabled below, there must be only one server listed here. This would usually be a name that always resolves to the local manchine, like 127.0.0.1 or localhost.';
$string['serversclusterinvalid'] = 'Exactly one server is required when clustering is enabled.';
$string['setservers'] = 'Set Servers';
$string['setservers_help'] = 'This is the list of servers that will updated when data is modified in the cache. Generally the fully qualified name of each server in the pool.
It **must** include the server listed in *Servers* above, even if by a different hostname.
Servers should be defined one per line and consist of a server address and optionally a port.
If no port is provided then the default port (11211) is used.
For example:
<pre>
server.url.com
ipaddress:port
</pre>';
$string['testservers'] = 'Test servers';
$string['testservers_desc'] = 'The test servers get used for unit tests and for performance tests. It is entirely optional to set up test servers. Servers should be defined one per line and consist of a server address and optionally a port and weight.
Expand Down
164 changes: 158 additions & 6 deletions cache/stores/memcached/lib.php
View file
Expand Up @@ -86,6 +86,24 @@ class cachestore_memcached extends cache_store implements cache_is_configurable
*/
protected $definition;

/**
* Set to true when this store is clustered.
* @var bool
*/
protected $clustered = false;

/**
* Array of servers to set when in clustered mode.
* @var array
*/
protected $setservers = array();

/**
* The an array of memcache connections for the set servers, once established.
* @var array
*/
protected $setconnections = array();

/**
* Constructs the store instance.
*
Expand Down Expand Up @@ -127,6 +145,30 @@ public function __construct($name, array $configuration = array()) {
}
$this->servers[] = $server;
}

$this->clustered = array_key_exists('clustered', $configuration) ? (bool)$configuration['clustered'] : false;

if ($this->clustered) {
if (!array_key_exists('setservers', $configuration) || (count($configuration['setservers']) < 1)) {
// Can't setup clustering without set servers.
return;
}
if (count($this->servers) !== 1) {
// Can only setup cluster with exactly 1 get server.
return;
}
foreach ($configuration['setservers'] as $server) {
// We do not use weights (3rd part) on these servers.
if (!is_array($server)) {
$server = explode(':', $server, 3);
}
if (!array_key_exists(1, $server)) {
$server[1] = 11211;
}
$this->setservers[] = $server;
}
}

$this->options[Memcached::OPT_COMPRESSION] = $compression;
$this->options[Memcached::OPT_SERIALIZER] = $serialiser;
$this->options[Memcached::OPT_PREFIX_KEY] = $prefix;
Expand All @@ -141,7 +183,20 @@ public function __construct($name, array $configuration = array()) {
}
$this->connection->addServers($this->servers);
}
// Test the connection to the pool of servers.

if ($this->clustered) {
foreach ($this->setservers as $setserver) {
// Since we will have a number of them with the same name, append server and port.
$connection = new Memcached(crc32($this->name.$setserver[0].$setserver[1]));
foreach ($this->options as $key => $value) {
$connection->setOption($key, $value);
}
$connection->addServer($setserver[0], $setserver[1]);
$this->setconnections[] = $connection;
}
}

// Test the connection to the main connection.
$this->isready = @$this->connection->set("ping", 'ping', 1);
}

Expand Down Expand Up @@ -267,6 +322,14 @@ public function get_many($keys) {
* @return bool True if the operation was a success false otherwise.
*/
public function set($key, $data) {
if ($this->clustered) {
$status = true;
foreach ($this->setconnections as $connection) {
$status = $connection->set($key, $data, $this->definition->get_ttl()) && $status;
}
return $status;
}

return $this->connection->set($key, $data, $this->definition->get_ttl());
}

Expand All @@ -283,7 +346,17 @@ public function set_many(array $keyvaluearray) {
foreach ($keyvaluearray as $pair) {
$pairs[$pair['key']] = $pair['value'];
}
if ($this->connection->setMulti($pairs, $this->definition->get_ttl())) {

$status = true;
if ($this->clustered) {
foreach ($this->setconnections as $connection) {
$status = $connection->setMulti($pairs, $this->definition->get_ttl()) && $status;
}
} else {
$status = $this->connection->setMulti($pairs, $this->definition->get_ttl());
}

if ($status) {
return count($keyvaluearray);
}
return 0;
Expand All @@ -296,6 +369,14 @@ public function set_many(array $keyvaluearray) {
* @return bool Returns true if the operation was a success, false otherwise.
*/
public function delete($key) {
if ($this->clustered) {
$status = true;
foreach ($this->setconnections as $connection) {
$status = $connection->delete($key) && $status;
}
return $status;
}

return $this->connection->delete($key);
}

Expand All @@ -306,9 +387,29 @@ public function delete($key) {
* @return int The number of items successfully deleted.
*/
public function delete_many(array $keys) {
if ($this->clustered) {
// Get the minimum deleted from any of the connections.
$count = count($keys);
foreach ($this->setconnections as $connection) {
$count = min($this->delete_many_connection($connection, $keys), $count);
}
return $count;
}

return $this->delete_many_connection($this->connection, $keys);
}

/**
* Deletes several keys from the cache in a single action for a specific connection.
*
* @param Memcached $connection The connection to work on.
* @param array $keys The keys to delete
* @return int The number of items successfully deleted.
*/
protected function delete_many_connection(Memcached $connection, array $keys) {
$count = 0;
foreach ($keys as $key) {
if ($this->connection->delete($key)) {
if ($connection->delete($key)) {
$count++;
}
}
Expand All @@ -322,7 +423,13 @@ public function delete_many(array $keys) {
*/
public function purge() {
if ($this->isready) {
$this->connection->flush();
if ($this->clustered) {
foreach ($this->setconnections as $connection) {
$connection->flush();
}
} else {
$this->connection->flush();
}
}

return true;
Expand Down Expand Up @@ -382,13 +489,37 @@ public static function config_get_configuration_array($data) {
}
$servers[] = explode(':', $line, 3);
}

$clustered = false;
if (isset($data->clustered)) {
$clustered = true;
}

$lines = explode("\n", $data->setservers);
$setservers = array();
foreach ($lines as $line) {
// Trim surrounding colons and default whitespace.
$line = trim(trim($line), ":");
if ($line === '') {
continue;
}
$setserver = explode(':', $line, 3);
// We don't use weights, so display a debug message.
if (count($setserver) > 2) {
debugging('Memcached Set Server '.$setserver[0].' has too many parameters.');
}
$setservers[] = $setserver;
}

return array(
'servers' => $servers,
'compression' => $data->compression,
'serialiser' => $data->serialiser,
'prefix' => $data->prefix,
'hash' => $data->hash,
'bufferwrites' => $data->bufferwrites,
'clustered' => $clustered,
'setservers' => $setservers
);
}

Expand Down Expand Up @@ -422,6 +553,16 @@ public static function config_set_edit_form_data(moodleform $editform, array $co
if (isset($config['bufferwrites'])) {
$data['bufferwrites'] = (bool)$config['bufferwrites'];
}
if (isset($config['clustered'])) {
$data['clustered'] = (bool)$config['clustered'];
}
if (!empty($config['setservers'])) {
$servers = array();
foreach ($config['setservers'] as $server) {
$servers[] = join(":", $server);
}
$data['setservers'] = join("\n", $servers);
}
$editform->set_data($data);
}

Expand Down Expand Up @@ -464,7 +605,7 @@ public static function initialise_test_instance(cache_definition $definition) {
}

$configuration = array();
$configuration['servers'] = $config->testservers;
$configuration['servers'] = explode("\n", $config->testservers);
if (!empty($config->testcompression)) {
$configuration['compression'] = $config->testcompression;
}
Expand All @@ -480,8 +621,19 @@ public static function initialise_test_instance(cache_definition $definition) {
if (!empty($config->testbufferwrites)) {
$configuration['bufferwrites'] = $config->testbufferwrites;
}
if (!empty($config->testclustered)) {
$configuration['clustered'] = $config->testclustered;
}
if (!empty($config->testsetservers)) {
$configuration['setservers'] = explode("\n", $config->testsetservers);
}
if (!empty($config->testname)) {
$name = $config->testname;
} else {
$name = 'Test memcached';
}

$store = new cachestore_memcached('Test memcached', $configuration);
$store = new cachestore_memcached($name, $configuration);
$store->initialise($definition);

return $store;
Expand Down
2 changes: 1 addition & 1 deletion cache/stores/memcached/settings.php
View file
Expand Up @@ -30,4 +30,4 @@
'cachestore_memcached/testservers',
new lang_string('testservers', 'cachestore_memcached'),
new lang_string('testservers_desc', 'cachestore_memcached'),
'', PARAM_RAW, 60, 3));
'', PARAM_RAW, 60, 3));

0 comments on commit 1c0518a

Please sign in to comment.