Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Fix documentation, version and default behavior

  • Loading branch information...
commit d764ff850ed1623a3b9fd934fdcb1c3cd7b0127c 1 parent 825f52e
John Watson authored
View
8 README.rst
@@ -9,7 +9,7 @@ The following example creates a Redis connection cluster which will distribute r
>>>
>>> redis = create_cluster({
>>> 'engine': 'nydus.db.backends.redis.Redis',
- >>> 'router': 'nydus.db.routers.redis.PartitionRouter',
+ >>> 'router': 'nydus.db.routers.keyvalue.PartitionRouter',
>>> 'hosts': {
>>> 0: {'db': 0},
>>> 1: {'db': 1},
@@ -111,7 +111,7 @@ hash on the key::
>>> redis = create_cluster({
>>> 'engine': 'nydus.db.backends.redis.Redis',
- >>> 'router': 'nydus.db.routers.redis.PartitionRouter',
+ >>> 'router': 'nydus.db.routers.keyvalue.PartitionRouter',
>>> 'hosts': {
>>> 0: {'db': 0},
>>> },
@@ -124,7 +124,7 @@ An improvement upon hashing, Nydus provides a Ketama-based consistent hashing ro
>>> redis = create_cluster({
>>> 'engine': 'nydus.db.backends.redis.Redis',
- >>> 'router': 'nydus.db.routers.redis.ConsistentHashingRouter',
+ >>> 'router': 'nydus.db.routers.keyvalue.ConsistentHashingRouter',
>>> 'hosts': {
>>> 0: {'db': 0},
>>> },
@@ -137,7 +137,7 @@ An additional option for distributing queries is the round robin router::
>>> redis = create_cluster({
>>> 'engine': 'nydus.db.backends.redis.Redis',
- >>> 'router': 'nydus.db.routers.redis.RoundRobinRouter',
+ >>> 'router': 'nydus.db.routers.RoundRobinRouter',
>>> 'hosts': {
>>> 0: {'db': 0},
>>> },
View
3  nydus/db/routers/__init__.py
@@ -6,4 +6,5 @@
:license: Apache License 2.0, see LICENSE for more details.
"""
-from .base import BaseRouter, RoundRobinRouter, PartitionRouter
+from .base import BaseRouter, RoundRobinRouter
+
View
11 nydus/db/routers/base.py
@@ -7,11 +7,9 @@
"""
import time
-from binascii import crc32
-from collections import defaultdict
from itertools import cycle
-__all__ = ('BaseRouter', 'RoundRobinRouter', 'PartitionRouter')
+__all__ = ('BaseRouter', 'RoundRobinRouter')
class BaseRouter(object):
@@ -77,7 +75,7 @@ def _route(self, cluster, attr, key, *args, **kwargs):
"""
Perform routing and return db_nums
"""
- return [cluster.hosts.keys()[0]]
+ return cluster.hosts.keys()
def _post_routing(self, cluster, attr, key, db_nums, *args, **kwargs):
"""
@@ -175,8 +173,3 @@ def _post_routing(self, cluster, attr, key, db_nums, *args, **kwargs):
return db_nums
-
-class PartitionRouter(BaseRouter):
- def _route(self, cluster, attr, key, *args, **kwargs):
- return [crc32(str(key)) % len(cluster)]
-
View
65 nydus/db/routers/keyvalue.py
@@ -0,0 +1,65 @@
+"""
+nydus.db.routers.keyvalue
+~~~~~~~~~~~~~~~~~~~~~~
+
+:copyright: (c) 2011 DISQUS.
+:license: Apache License 2.0, see LICENSE for more details.
+"""
+
+from binascii import crc32
+
+from nydus.contrib.ketama import Ketama
+from nydus.db.routers import BaseRouter, RoundRobinRouter
+
+__all__ = ('ConsistentHashingRouter', 'PartitionRouter')
+
+
+class ConsistentHashingRouter(RoundRobinRouter):
+ '''
+ Router that returns host number based on a consistent hashing algorithm.
+ The consistent hashing algorithm only works if a key argument is provided.
+ If a key is not provided, then all hosts are returned.
+ '''
+
+ def __init__(self, *args, **kwargs):
+ self._db_num_id_map = {}
+ super(ConsistentHashingRouter, self).__init__(*args, **kwargs)
+
+ def flush_down_connections(self):
+ for db_num in self._down_connections:
+ self._hash.add_node(self._db_num_id_map[db_num])
+
+ super(ConsistentHashingRouter, self).flush_down_connections()
+
+ def mark_connection_down(self, db_num):
+ db_num = self.ensure_db_num(db_num)
+ self._hash.remove_node(self._db_num_id_map[db_num])
+
+ super(ConsistentHashingRouter, self).mark_connection_down(db_num)
+
+ def mark_conenction_up(self, db_num):
+ db_num = self.ensure_db_num(db_num)
+ self._hash.add_node(self._db_num_id_map[db_num])
+
+ super(ConsistentHashingRouter, self).mark_connection_up(db_num)
+
+ def _setup_router(self, cluster, *args, **kwargs):
+ self._db_num_id_map = dict([(db_num, host.identifier) for db_num, host in cluster.hosts.iteritems()])
+ self._hash = Ketama(self._db_num_id_map.values())
+
+ return True
+
+ def _route(self, cluster, attr, key, *args, **kwargs):
+ found = self._hash.get_node(key)
+
+ if not found and len(self._down_connections) > 0:
+ raise self.HostListExhausted()
+
+ return [i for i, h in cluster.hosts.iteritems()
+ if h.identifier == found]
+
+
+class PartitionRouter(BaseRouter):
+ def _route(self, cluster, attr, key, *args, **kwargs):
+ return [crc32(str(key)) % len(cluster)]
+
View
46 nydus/db/routers/redis.py
@@ -7,49 +7,7 @@
"""
from nydus.db.routers import RoundRobinRouter
-from nydus.contrib.ketama import Ketama
+from nydus.db.routers.keyvalue import ConsistentHashingRouter, PartitionRouter
+__all__ = ('ConsistentHashingRouter', 'PartitionRouter', 'RoundRobinRouter')
-class ConsistentHashingRouter(RoundRobinRouter):
- '''
- Router that returns host number based on a consistent hashing algorithm.
- The consistent hashing algorithm only works if a key argument is provided.
- If a key is not provided, then all hosts are returned.
- '''
-
- def __init__(self, *args, **kwargs):
- self._db_num_id_map = {}
- super(ConsistentHashingRouter, self).__init__(*args, **kwargs)
-
- def flush_down_connections(self):
- for db_num in self._down_connections:
- self._hash.add_node(self._db_num_id_map[db_num])
-
- super(ConsistentHashingRouter, self).flush_down_connections()
-
- def mark_connection_down(self, db_num):
- db_num = self.ensure_db_num(db_num)
- self._hash.remove_node(self._db_num_id_map[db_num])
-
- super(ConsistentHashingRouter, self).mark_connection_down(db_num)
-
- def mark_conenction_up(self, db_num):
- db_num = self.ensure_db_num(db_num)
- self._hash.add_node(self._db_num_id_map[db_num])
-
- super(ConsistentHashingRouter, self).mark_connection_up(db_num)
-
- def _setup_router(self, cluster, *args, **kwargs):
- self._db_num_id_map = dict([(db_num, host.identifier) for db_num, host in cluster.hosts.iteritems()])
- self._hash = Ketama(self._db_num_id_map.values())
-
- return True
-
- def _route(self, cluster, attr, key, *args, **kwargs):
- found = self._hash.get_node(key)
-
- if not found and len(self._down_connections) > 0:
- raise self.HostListExhausted()
-
- return [i for i, h in cluster.hosts.iteritems()
- if h.identifier == found]
View
2  setup.py
@@ -22,7 +22,7 @@
setup(
name='nydus',
- version='0.7.2',
+ version='0.8.0',
author='David Cramer',
author_email='dcramer@gmail.com',
url='http://github.com/disqus/nydus',
View
6 tests/nydus/db/backends/redis/tests.py
@@ -32,7 +32,7 @@ def test_provides_identifier(self):
def test_pipelined_map(self):
redis = create_cluster({
'engine': 'nydus.db.backends.redis.Redis',
- 'router': 'nydus.db.routers.PartitionRouter',
+ 'router': 'nydus.db.routers.keyvalue.PartitionRouter',
'hosts': {
0: {'db': 5},
1: {'db': 6},
@@ -59,7 +59,7 @@ def test_client_instantiates_with_kwargs(self, RedisClient):
def test_map_does_pipeline(self, RedisClient):
redis = create_cluster({
'engine': 'nydus.db.backends.redis.Redis',
- 'router': 'nydus.db.routers.PartitionRouter',
+ 'router': 'nydus.db.routers.keyvalue.PartitionRouter',
'hosts': {
0: {'db': 0},
1: {'db': 1},
@@ -87,7 +87,7 @@ def test_map_does_pipeline(self, RedisClient):
def test_map_only_runs_on_required_nodes(self, RedisClient):
redis = create_cluster({
'engine': 'nydus.db.backends.redis.Redis',
- 'router': 'nydus.db.routers.PartitionRouter',
+ 'router': 'nydus.db.routers.keyvalue.PartitionRouter',
'hosts': {
0: {'db': 0},
1: {'db': 1},
View
4 tests/nydus/db/backends/thoonk/tests.py
@@ -43,11 +43,11 @@ def tearDown(self):
pass
def test_flush_db(self):
- pubsub = self.get_cluster('nydus.db.routers.redis.ConsistentHashingRouter')
+ pubsub = self.get_cluster('nydus.db.routers.keyvalue.ConsistentHashingRouter')
pubsub.flushdb()
def test_job_with_ConsistentHashingRouter(self):
- pubsub = self.get_cluster('nydus.db.routers.redis.ConsistentHashingRouter')
+ pubsub = self.get_cluster('nydus.db.routers.keyvalue.ConsistentHashingRouter')
job = pubsub.job("test1")
jid = job.put("10")
View
6 tests/nydus/db/connections/tests.py
@@ -82,7 +82,7 @@ def test_with_router(self):
hosts={0: c, 1: c2},
)
self.assertEquals(p.foo(), ['foo', 'bar'])
- self.assertEquals(p.foo('foo'), 'foo')
+ self.assertEquals(p.foo('foo'), ['foo', 'bar'])
def test_get_conn(self):
c = DummyConnection(alias='foo', num=0, resp='foo')
@@ -102,7 +102,7 @@ def test_get_conn(self):
hosts={0: c, 1: c2},
)
self.assertEquals(p.get_conn(), [c, c2])
- self.assertEquals(p.get_conn('foo'), c)
+ self.assertEquals(p.get_conn('foo'), [c, c2])
def test_map(self):
c = DummyConnection(num=0, resp='foo')
@@ -134,7 +134,7 @@ def test_map(self):
self.assertEquals(bar, None)
self.assertEquals(foo, ['foo', 'bar'])
- self.assertEquals(bar, 'foo')
+ self.assertEquals(bar, ['foo', 'bar'])
class FlakeyConnection(DummyConnection):
View
10 tests/nydus/db/routers/tests.py
@@ -10,8 +10,8 @@
from tests import BaseTest
from nydus.db.base import Cluster
from nydus.db.backends import BaseConnection
-from nydus.db.routers import BaseRouter, RoundRobinRouter, PartitionRouter
-from nydus.db.routers.redis import ConsistentHashingRouter
+from nydus.db.routers import BaseRouter, RoundRobinRouter
+from nydus.db.routers.keyvalue import ConsistentHashingRouter, PartitionRouter
class DummyConnection(BaseConnection):
@@ -67,9 +67,6 @@ def test_offers_router_interface(self):
def test_returns_whole_cluster_without_key(self):
self.assertEquals(self.hosts.keys(), self.get_dbs(attr='test'))
- def test_returns_sequence_with_one_item_when_given_key(self):
- self.assertEqual(len(self.get_dbs(attr='test', key='foo')), 1)
-
def test_get_dbs_handles_exception(self):
with patch.object(self.router, '_route') as _route:
with patch.object(self.router, '_handle_exception') as _handle_exception:
@@ -103,6 +100,9 @@ def test__handle_exception_raises_same_exception(self):
with self.assertRaises(self.TestException):
self.router._handle_exception(e)
+ def test_returns_sequence_with_one_item_when_given_key(self):
+ self.assertEqual(len(self.get_dbs(attr='test', key='foo')), len(self.hosts))
+
class BaseRoundRobinRouterTest(BaseRouterTest):
Router = RoundRobinRouter
Please sign in to comment.
Something went wrong with that request. Please try again.