Skip to content

Commit

Permalink
Issue #2851625 by amontero: New feature: option to use SCAN command i…
Browse files Browse the repository at this point in the history 
…nstead of KEYS on cache wildcard deletions
  • Loading branch information
@pataquets @omega8cc
pataquets authored and omega8cc committed Feb 14, 2017
1 parent 921f951 commit fceaef5
Show file tree
Hide file tree
Showing 4 changed files with 114 additions and 10 deletions.
98 changes: 95 additions & 3 deletions lib/Redis/Cache/Base.php
View file
Expand Up @@ -2,7 +2,6 @@

/**
* @todo
* - Improve lua scripts by using SCAN family commands
* - Deambiguate why we need the namespace only for flush*() operations
* - Implement the isEmpty() method by using SCAN or KEYS
*/
Expand All @@ -17,23 +16,116 @@ abstract class Redis_Cache_Base extends Redis_AbstractBackend
* Delete by prefix lua script
*/
const EVAL_DELETE_PREFIX = <<<EOT
redis.log(redis.LOG_NOTICE, "Delete by prefix requested for '" .. ARGV[1] .. "'.")
local deleted_keys_count = 0
local scan_delete = tonumber(ARGV[2])
if scan_delete == nil then
scan_delete = 0
end
if scan_delete ~= 0 then
local server_info = redis.call("INFO", "SERVER")
local version_string = string.match(server_info, "redis_version:([%d.]*)")
local version_major, version_minor = 0, 0
version_major, version_minor = string.match(version_string, "(%d+).(%d+)")
version_major = tonumber(version_major)
version_minor = tonumber(version_minor)
redis.log(redis.LOG_DEBUG, "Non-blocking delete requested. " ..
"Server version string: " .. version_string)
if (version_major >= 3 and version_minor >= 2) then
local scan_count = (ARGV[3] or 100)
local result, keys, cursor = {}, {}, "0"
redis.log(redis.LOG_DEBUG, "Using SCAN command to delete. COUNT arg: " ..
tostring(scan_count) .. ".")
redis.replicate_commands()
repeat
result = redis.call("SCAN", cursor, "MATCH", ARGV[1], "COUNT", scan_count)
cursor = result[1]
keys = result[2]
for _, k in pairs(keys) do
redis.call("DEL", k)
deleted_keys_count = deleted_keys_count + 1
end
until cursor == "0"
redis.log(redis.LOG_NOTICE, "Delete by prefix (SCAN) finished for '" ..
ARGV[1] .. "' (" .. tostring(deleted_keys_count) .. " keys).")
return 1
end
redis.log(redis.LOG_WARNING, "Non-blocking delete requested but not supported by server.")
end
redis.log(redis.LOG_NOTICE, "Using KEYS command to delete.")
local keys = redis.call("KEYS", ARGV[1])
for i, k in ipairs(keys) do
redis.log(redis.LOG_NOTICE, "KEYS command finished.")
for _, k in ipairs(keys) do
redis.call("DEL", k)
deleted_keys_count = deleted_keys_count + 1
end
redis.log(redis.LOG_NOTICE, "Delete by prefix (KEYS) finished for '" ..
ARGV[1] .. "' (" .. tostring(deleted_keys_count) .. " keys).")
return 1
EOT;

/**
* Delete volatile by prefix lua script
*/
const EVAL_DELETE_VOLATILE = <<<EOT
redis.log(redis.LOG_NOTICE, "Delete volatile by prefix requested for '" .. ARGV[1] .. "'.")
local deleted_keys_count = 0
local scan_delete = tonumber(ARGV[2])
if scan_delete == nil then
scan_delete = 0
end
if scan_delete ~= 0 then
local server_info = redis.call("INFO", "SERVER")
local version_string = string.match(server_info, "redis_version:([%d.]*)")
local version_major, version_minor = 0, 0
version_major, version_minor = string.match(version_string, "(%d+).(%d+)")
version_major = tonumber(version_major)
version_minor = tonumber(version_minor)
redis.log(redis.LOG_DEBUG, "Non-blocking delete requested. " ..
"Server version string: " .. version_string)
if (version_major >= 3 and version_minor >= 2) then
local scan_count = (ARGV[3] or 100)
local result, keys, cursor = {}, {}, "0"
redis.log(redis.LOG_DEBUG, "Using SCAN command to delete. COUNT arg: " ..
tostring(scan_count) .. ".")
redis.replicate_commands()
repeat
result = redis.call("SCAN", cursor, "MATCH", ARGV[1], "COUNT", scan_count)
cursor = result[1]
keys = result[2]
for _, k in pairs(keys) do
if "1" == redis.call("HGET", k, "volatile") then
redis.call("DEL", k)
deleted_keys_count = deleted_keys_count + 1
end
end
until cursor == "0"
redis.log(redis.LOG_NOTICE, "Delete volatile by prefix (SCAN) finished for '" ..
ARGV[1] .. "' (" .. tostring(deleted_keys_count) .. " keys).")
return 1
end
redis.log(redis.LOG_WARNING, "Non-blocking delete requested but not supported by server.")
end
local keys = redis.call('KEYS', ARGV[1])
for i, k in ipairs(keys) do
redis.log(redis.LOG_NOTICE, "KEYS command finished.")
for _, k in ipairs(keys) do
if "1" == redis.call("HGET", k, "volatile") then
redis.call("DEL", k)
deleted_keys_count = deleted_keys_count + 1
end
end
redis.log(redis.LOG_NOTICE, "Delete volatile by prefix (KEYS) finished for '" ..
ARGV[1] .. "' (" .. tostring(deleted_keys_count) .. " keys).")
return 1
EOT;
}
9 changes: 6 additions & 3 deletions lib/Redis/Cache/PhpRedis.php
View file
Expand Up @@ -123,7 +123,8 @@ public function deleteMultiple(array $idList)
public function deleteByPrefix($prefix)
{
$client = $this->getClient();
$ret = $client->eval(self::EVAL_DELETE_PREFIX, array($this->getKey($prefix . '*')));
$scan_delete = variable_get('redis_scan_delete', FALSE);
$ret = $client->eval(self::EVAL_DELETE_PREFIX, array($this->getKey($prefix . '*'), $scan_delete));
if (1 != $ret) {
trigger_error(sprintf("EVAL failed: %s", $client->getLastError()), E_USER_ERROR);
}
Expand All @@ -132,7 +133,8 @@ public function deleteByPrefix($prefix)
public function flush()
{
$client = $this->getClient();
$ret = $client->eval(self::EVAL_DELETE_PREFIX, array($this->getKey('*')));
$scan_delete = variable_get('redis_scan_delete', FALSE);
$ret = $client->eval(self::EVAL_DELETE_PREFIX, array($this->getKey('*'), $scan_delete));
if (1 != $ret) {
trigger_error(sprintf("EVAL failed: %s", $client->getLastError()), E_USER_ERROR);
}
Expand All @@ -141,7 +143,8 @@ public function flush()
public function flushVolatile()
{
$client = $this->getClient();
$ret = $client->eval(self::EVAL_DELETE_VOLATILE, array($this->getKey('*')));
$scan_delete = variable_get('redis_scan_delete', FALSE);
$ret = $client->eval(self::EVAL_DELETE_VOLATILE, array($this->getKey('*'), $scan_delete));
if (1 != $ret) {
trigger_error(sprintf("EVAL failed: %s", $client->getLastError()), E_USER_ERROR);
}
Expand Down
11 changes: 7 additions & 4 deletions lib/Redis/Cache/Predis.php
View file
Expand Up @@ -119,16 +119,18 @@ public function deleteMultiple(array $idList)
public function deleteByPrefix($prefix)
{
$client = $this->getClient();
$ret = $client->eval(self::EVAL_DELETE_PREFIX, 0, $this->getKey($prefix . '*'));
$scan_delete = variable_get('redis_scan_delete', FALSE);
$ret = $client->eval(self::EVAL_DELETE_PREFIX, 0, $this->getKey($prefix . '*'), $scan_delete);
if (1 != $ret) {
trigger_error(sprintf("EVAL failed: %s", $client->getLastError()), E_USER_ERROR);
}
}

public function flush()
{
$client = $this->getClient();
$ret = $client->eval(self::EVAL_DELETE_PREFIX, 0, $this->getKey('*'));
$client = $this->getClient();
$scan_delete = variable_get('redis_scan_delete', FALSE);
$ret = $client->eval(self::EVAL_DELETE_PREFIX, 0, $this->getKey('*'), $scan_delete);
if (1 != $ret) {
trigger_error(sprintf("EVAL failed: %s", $client->getLastError()), E_USER_ERROR);
}
Expand All @@ -137,7 +139,8 @@ public function flush()
public function flushVolatile()
{
$client = $this->getClient();
$ret = $client->eval(self::EVAL_DELETE_VOLATILE, 0, $this->getKey('*'));
$scan_delete = variable_get('redis_scan_delete', FALSE);
$ret = $client->eval(self::EVAL_DELETE_VOLATILE, 0, $this->getKey('*'), $scan_delete);
if (1 != $ret) {
trigger_error(sprintf("EVAL failed: %s", $client->getLastError()), E_USER_ERROR);
}
Expand Down
6 changes: 6 additions & 0 deletions redis.admin.inc
View file
Expand Up @@ -58,6 +58,12 @@ function redis_settings_form($form, &$form_state) {
'#default_value' => variable_get('redis_client_interface', NULL),
'#description' => t("Redis low level backend."),
);
$form['connection']['redis_scan_delete'] = array(
'#type' => 'checkbox',
'#title' => t('Use SCAN command instead of KEYS for cache wildcard key deletions'),
'#default_value' => variable_get('redis_scan_delete', FALSE),
'#description' => t('Requires Redis 3.2 or later. Uses non-atomic, non-blocking and concurrency friendler SCAN command instead of KEYS to perform cache wildcard key deletions.'),
);

$form = system_settings_form($form);

Expand Down

0 comments on commit fceaef5

Please sign in to comment.