Permalink
Browse files

add support for mget/mset hash tags

  • Loading branch information...
1 parent 1b5e624 commit e5d5318a92cff1b33023a75dccdb74c8f0e6a827 @salimane committed Nov 16, 2012
Showing with 95 additions and 40 deletions.
  1. +5 −1 README.rst
  2. +49 −16 rediscluster/cluster_client.py
  3. +39 −21 tests/cluster_commands.py
  4. +2 −2 tests/config.py
View
@@ -117,13 +117,17 @@ Hash Tags
-----------
In order to specify your own hash key (so that related keys can all land
-on a given node), rediscluster allows you to pass a list where you’d normally pass a scalar.
+on a given node), ``rediscluster`` allows you to pass a string in the form "a{b}" where you’d normally pass a scalar.
The first element of the list is the key to use for the hash and the
second is the real key that should be fetched/modify:
::
>>> r.get("bar{foo}")
+ >>>
+ >>> r.mset({"bar{foo}": "bar", "foo": "foo"})
+ >>>
+ >>> r.mget(["bar{foo}", "foo"])
In that case “foo” is the hash key but “bar” is still the name of
the key that is fetched from the redis node that “foo” hashes to.
@@ -2,7 +2,8 @@
import binascii
import redis
-from redis._compat import (b, iteritems, dictvalues)
+from redis._compat import (
+ b, iteritems, dictkeys, dictvalues, basestring, bytes)
from redis.client import list_or_args
@@ -81,7 +82,7 @@ class StrictRedisCluster:
}
def __init__(self, cluster={}, db=0):
- #raise exception when wrong server hash
+ # raise exception when wrong server hash
if 'nodes' not in cluster or 'master_of' not in cluster:
raise Exception(
"rediscluster: Please set a correct array of redis cluster.")
@@ -90,7 +91,7 @@ def __init__(self, cluster={}, db=0):
self.no_servers = len(cluster['master_of'])
slaves = dictvalues(cluster['master_of'])
self.redises = {}
- #connect to all servers
+ # connect to all servers
for alias, server in iteritems(cluster['nodes']):
info = {}
try:
@@ -100,7 +101,7 @@ def __init__(self, cluster={}, db=0):
raise redis.DataError(
"rediscluster: server %s is not a slave." % (server,))
except Exception as e:
- #if node is slave and is down, replace its connection with its master's
+ # if node is slave and is down, replace its connection with its master's
try:
ms = [k for k, v in iteritems(cluster['master_of'])
if v == alias and (('role' in info and info['role'] == 'slave') or cluster['nodes'][k] == cluster['nodes'][v])][0]
@@ -129,28 +130,60 @@ def __getattr__(self, name, *args, **kwargs):
"""
def function(*args, **kwargs):
if name not in StrictRedisCluster._loop_keys:
- try:
- tag_start = args[0].index('{')
- except Exception as e:
- tag_start = None
+ # take care of hash tags
+ tag_start = None
+ key_type = hash_tag = ''
+ # since we don't have "first item" in dict,
+ # this list is needed in order to check hash_tag in mset({"a{a}": "a", "b":"b"})
+ list_ht = []
+ if isinstance(args[0], basestring) or isinstance(args[0], bytes):
+ key_type = 'string'
+ list_ht.append(args[0])
+ else:
+ if isinstance(args[0], list):
+ key_type = 'list'
+ list_ht.append(args[0][0])
+ else:
+ key_type = 'dict'
+ list_ht = dictkeys(args[0])
+
+ # check for hash tags
+ for k in list_ht:
+ try:
+ tag_start = k.index('{')
+ hash_tag = k
+ break
+ except Exception as e:
+ tag_start = None
- # trigger error msg on banned keys unless u're using it with tagged keys e.g. "bar{zap}"
+ # trigger error msg on tag keys unless we have hash tags e.g. "bar{zap}"
if name in StrictRedisCluster._tag_keys and not tag_start:
try:
return getattr(self, '_rc_' + name)(*args, **kwargs)
except AttributeError:
raise redis.DataError("rediscluster: Command %s Not Supported (each key name has its own node)" % name)
- #get the hash key depending on tags or not
+ # get the hash key
hkey = args[0]
- # take care of hash tags names for forcing multiple keys on the same node, e.g. $redis -> set("bar{zap}", "bar")
- if tag_start:
+ # take care of hash tags names for forcing multiple keys on the same node,
+ # e.g. r.set("bar{zap}", "bar"), r.mget(["foo{foo}","bar"])
+ if tag_start is not None:
L = list(args)
- hkey = L[0][tag_start + 1:-1]
- L[0] = L[0][0:tag_start]
+ if key_type != 'string':
+ if key_type == 'list':
+ hkey = L[0][0][tag_start + 1:-1]
+ L[0][0] = L[0][0][0:tag_start]
+ else:
+ hkey = hash_tag[tag_start + 1:-1]
+ L[0][hash_tag[0:tag_start]] = L[0][hash_tag]
+ del L[0][hash_tag]
+ else:
+ hkey = L[0][tag_start + 1:-1]
+ L[0] = L[0][0:tag_start]
+
args = tuple(L)
- #get the node number
+ # get the node number
node = self._getnodenamefor(hkey)
redisent = self.redises[self.cluster['default_node']]
if name in StrictRedisCluster._write_keys:
@@ -159,7 +192,7 @@ def function(*args, **kwargs):
redisent = self.redises[
self.cluster['master_of'][node]]
- #Execute the command on the server
+ # Execute the command on the server
return getattr(redisent, name)(*args, **kwargs)
else:
View
@@ -21,7 +21,7 @@ def setUp(self):
def tearDown(self):
self.client.flushdb()
- #self.client.connection_pool.disconnect()
+ # self.client.connection_pool.disconnect()
def test_response_callbacks(self):
try:
@@ -459,12 +459,30 @@ def test_mget(self):
self.client.mget(['a', 'other', 'b', 'c']),
[b('1'), None, b('2'), b('3')])
+ def test_mget_hash_tag(self):
+ self.assertEquals(self.client.mget(['foo{foo}', 'bar']), [None, None])
+ self.client['foo'] = '1'
+ self.client['bar{foo}'] = '2'
+ self.client['other{foo}'] = '3'
+ self.assertEquals(
+ self.client.mget(['foo{foo}', 'c', 'bar', 'other']),
+ [b('1'), None, b('2'), b('3')])
+
def test_mset(self):
d = {'a': '1', 'b': '2', 'c': '3'}
self.assert_(self.client.mset(d))
for k, v in iteritems(d):
self.assertEquals(self.client[k], b(v))
+ def test_mset_mget_hash_tag(self):
+ self.assert_(
+ self.client.mset({'foo{foo}': '1', 'bar': '2', 'other': '3'}))
+ self.assertEquals(self.client.mget(
+ ['foo{foo}', 'bar', 'other']), [b('1'), b('2'), b('3')])
+ self.assertEquals(self.client['foo'], b('1'))
+ self.assertEquals(self.client['bar{foo}'], b('2'))
+ self.assertEquals(self.client['other{foo}'], b('3'))
+
def test_msetnx(self):
d = {'a': '1', 'b': '2', 'c': '3'}
self.assert_(self.client.msetnx(d))
@@ -475,7 +493,7 @@ def test_msetnx(self):
self.assertEquals(self.client.get('d'), None)
def test_randomkey(self):
- #CLUSTER
+ # CLUSTER
try:
raise unittest.SkipTest()
except AttributeError:
@@ -487,7 +505,7 @@ def test_randomkey(self):
self.assert_(self.client.randomkey() in (b('a'), b('b'), b('c')))
def test_rename(self):
- #CLUSTER
+ # CLUSTER
try:
raise unittest.SkipTest()
except AttributeError:
@@ -498,7 +516,7 @@ def test_rename(self):
self.assertEquals(self.client['b'], b('1'))
def test_renamenx(self):
- #CLUSTER
+ # CLUSTER
try:
raise unittest.SkipTest()
except AttributeError:
@@ -566,7 +584,7 @@ def make_list(self, name, l):
self.client.rpush(name, i)
def test_blpop(self):
- #CLUSTER
+ # CLUSTER
self.make_list('a', 'ab')
self.make_list('b', 'cd')
self.assertEquals(
@@ -962,7 +980,7 @@ def test_smembers(self):
set([b('a'), b('b'), b('c')]))
def test_smove(self):
- #CLUSTER
+ # CLUSTER
try:
raise unittest.SkipTest()
except AttributeError:
@@ -1117,7 +1135,7 @@ def test_zincrby(self):
self.assertEquals(self.client.zscore('a', 'a3'), 8.0)
def test_zinterstore(self):
- #CLUSTER
+ # CLUSTER
try:
raise unittest.SkipTest()
except AttributeError:
@@ -1322,7 +1340,7 @@ def test_zscore(self):
self.assertEquals(self.client.zscore('a', 'a4'), None)
def test_zunionstore(self):
- #CLUSTER
+ # CLUSTER
try:
raise unittest.SkipTest()
except AttributeError:
@@ -1489,7 +1507,7 @@ def test_hincrby(self):
self.assertEquals(self.client.hincrby('a', 'a1'), 2)
self.assertEquals(self.client.hincrby('a', 'a1', amount=2), 4)
# negative values decrement
- self.assertEquals(self.client.hincrby('a', 'a1', amount= -3), 1)
+ self.assertEquals(self.client.hincrby('a', 'a1', amount=-3), 1)
# hash that exists, but key that doesn't
self.assertEquals(self.client.hincrby('a', 'a2', amount=3), 3)
# finally a key that's not an int
@@ -1563,7 +1581,7 @@ def test_hvals(self):
# SORT
def test_sort_bad_key(self):
- #CLUSTER
+ # CLUSTER
try:
raise unittest.SkipTest()
except AttributeError:
@@ -1576,7 +1594,7 @@ def test_sort_bad_key(self):
del self.client['a']
def test_sort_basic(self):
- #CLUSTER
+ # CLUSTER
try:
raise unittest.SkipTest()
except AttributeError:
@@ -1587,7 +1605,7 @@ def test_sort_basic(self):
[b('1'), b('2'), b('3'), b('4')])
def test_sort_limited(self):
- #CLUSTER
+ # CLUSTER
try:
raise unittest.SkipTest()
except AttributeError:
@@ -1598,7 +1616,7 @@ def test_sort_limited(self):
[b('2'), b('3')])
def test_sort_by(self):
- #CLUSTER
+ # CLUSTER
try:
raise unittest.SkipTest()
except AttributeError:
@@ -1612,7 +1630,7 @@ def test_sort_by(self):
[b('2'), b('3'), b('1')])
def test_sort_get(self):
- #CLUSTER
+ # CLUSTER
try:
raise unittest.SkipTest()
except AttributeError:
@@ -1626,7 +1644,7 @@ def test_sort_get(self):
[b('u1'), b('u2'), b('u3')])
def test_sort_get_multi(self):
- #CLUSTER
+ # CLUSTER
try:
raise unittest.SkipTest()
except AttributeError:
@@ -1640,7 +1658,7 @@ def test_sort_get_multi(self):
[b('u1'), b('1'), b('u2'), b('2'), b('u3'), b('3')])
def test_sort_desc(self):
- #CLUSTER
+ # CLUSTER
try:
raise unittest.SkipTest()
except AttributeError:
@@ -1651,7 +1669,7 @@ def test_sort_desc(self):
[b('3'), b('2'), b('1')])
def test_sort_alpha(self):
- #CLUSTER
+ # CLUSTER
try:
raise unittest.SkipTest()
except AttributeError:
@@ -1662,7 +1680,7 @@ def test_sort_alpha(self):
[b('a'), b('b'), b('c'), b('d'), b('e')])
def test_sort_store(self):
- #CLUSTER
+ # CLUSTER
try:
raise unittest.SkipTest()
except AttributeError:
@@ -1674,7 +1692,7 @@ def test_sort_store(self):
[b('1'), b('2'), b('3')])
def test_sort_all_options(self):
- #CLUSTER
+ # CLUSTER
try:
raise unittest.SkipTest()
except AttributeError:
@@ -1755,7 +1773,7 @@ def test_strict_pexpire(self):
self.assertEquals(client.persist('a'), True)
self.assertEquals(client.pttl('a'), -1)
- ## BINARY SAFE
+ # # BINARY SAFE
# TODO add more tests
def test_binary_get_set(self):
self.assertTrue(self.client.set(' foo bar ', '123'))
@@ -1783,7 +1801,7 @@ def test_binary_lists(self):
self.assertTrue(self.client.rpush(key, c))
# check that KEYS returns all the keys as they are
- #self.assertEqual(sorted(self.client.keys('*')), sorted(dictkeys(mapping)))
+ # self.assertEqual(sorted(self.client.keys('*')), sorted(dictkeys(mapping)))
# check that it is possible to get list content by key name
for key in dictkeys(mapping):
View
@@ -10,8 +10,8 @@
},
# replication information
'master_of': {
- 'node_1': 'node_6', # node_6 slaveof node_1 in redis6.conf
- 'node_2': 'node_5', # node_5 slaveof node_2 in redis5.conf
+ 'node_1': 'node_6', # node_6 slaveof node_1 in redis6.conf
+ 'node_2': 'node_5', # node_5 slaveof node_2 in redis5.conf
},
'default_node': 'node_1'

0 comments on commit e5d5318

Please sign in to comment.