Permalink
Browse files

Working on TemporalStorage unit tests.

  • Loading branch information...
1 parent 37c211e commit 01796a60e95e41ed506c48a55c12ad1c0ca33d02 @BasicWolf committed Dec 30, 2012
Showing with 147 additions and 45 deletions.
  1. +6 −0 TODO.rst
  2. +5 −18 kaylee/contrib/storages.py
  3. +4 −4 kaylee/node.py
  4. +12 −4 kaylee/storage.py
  5. +120 −19 kaylee/testsuite/storage_tests.py
View
@@ -30,6 +30,12 @@ Feature requests
* Client: operations timeout (e.g. project import timeout)
* User management and rankings
+
+Refactoring requests
+--------------------
+* NodeID consructor
+* NodeID from_object (do we need it?)
+
Also
----
* PDB support in Emacs
View
@@ -1,6 +1,7 @@
# -*- coding: utf-8 -*-
#pylint: disable-msg=W0231
from kaylee.storage import TemporalStorage, PermanentStorage
+from kaylee.node import NodeID
class MemoryTemporalStorage(TemporalStorage):
"""A simple Python dict-based temporal results storage."""
@@ -11,23 +12,21 @@ def __init__(self):
def add(self, task_id, node_id, result):
d = self._d.get(task_id, {})
- d[node_id] = result
+ d[node_id.binary] = result
self._d[task_id] = d
def remove(self, task_id, node_id=None):
if node_id is None:
- del self._[task_id]
+ del self._d[task_id]
else:
del self._d[task_id][node_id]
def clear(self):
self._d = {}
def __getitem__(self, task_id):
- try:
- return self._d[task_id]
- except KeyError:
- return []
+ nr_dict = self._d[task_id]
+ return ((NodeID(n), r) for n, r in nr_dict.iteritems())
def contains(self, task_id, node_id=None, result=None):
if result is None and node_id is None:
@@ -56,18 +55,6 @@ def node_result_tuple_generator():
yield (node_id, result)
return node_result_tupe_generator()
- def __contains__(self, task_id):
- return task_id in self._d
-
- def __delitem__(self, task_id):
- del self._d[task_id]
-
- def __iter__(self):
- return iter(self._d)
-
- def __len__(self):
- return sum(len(res) for res in self._d)
-
class MemoryPermanentStorage(PermanentStorage):
"""A simple Python dict-based permanent results storage."""
View
@@ -178,7 +178,7 @@ class NodeID(object):
_inc = 0
_inc_lock = threading.Lock()
- def __init__(self, node_id = None, remote_host = '127.0.0.1'):
+ def __init__(self, node_id=None, remote_host='127.0.0.1'):
if node_id is None and not isinstance(remote_host, basestring):
raise TypeError('remote_host must be an instance of {}, not {}'
.format(basestring.__name__,
@@ -198,7 +198,7 @@ def for_host(host):
:type host: string
:returns: NodeID object
"""
- return NodeID(remote_host = host)
+ return NodeID(remote_host=host)
@staticmethod
def from_object(node):
@@ -210,11 +210,11 @@ def from_object(node):
:returns: NodeID object
"""
if isinstance(node, basestring):
- return NodeID(node_id = node)
+ return NodeID(node_id=node)
elif isinstance(node, NodeID):
return node
elif isinstance(node, Node):
- return NodeID(node_id = node.id)
+ return NodeID(node_id=node.id)
else:
raise TypeError('node must be an instance of {}, {}, or {} not'
' {}'.format(basestring.__name__,
View
@@ -22,7 +22,14 @@ class TemporalStorage(object):
@abstractmethod
def add(self, task_id, node_id, result):
- """Stores the task result returned by a node."""
+ """Stores the task result returned by a node.
+
+ :param task_id: task id
+ :param node_id: the id of the node which solved the task
+ :param result: task result
+ :type task_id: :class:`str`
+ :type node_id: :class:`NodeID`
+ """
@abstractmethod
def remove(self, task_id, node_id=None):
@@ -34,10 +41,11 @@ def clear(self):
"""Removes all results from the storage."""
@abstractmethod
- def __getitem_(_self, task_id):
+ def __getitem__(self, task_id):
"""Returns the task results.
- :rtype: (node_id : result) ... iterator
+ :rtype: ``(node_id : result)`` ... iterator, where
+ ``node_id`` is an instance of :class:`NodeID`.
"""
@abstractmethod
@@ -88,7 +96,7 @@ def __iter__(self):
def __len__(self):
"""The same as :meth:`TemporalStorage.count`."""
-
+ return self.count
class PermanentStorage(object):
"""The interface for applications' permanent results storage.
@@ -3,17 +3,115 @@
from kaylee.contrib.storages import (MemoryTemporalStorage,
MemoryPermanentStorage)
from copy import deepcopy
+from kaylee.node import NodeID
SOME = 10
MANY = 100
+def _tgen(i, prefix='t'):
+ return "{}{}".format(prefix, i)
+
+def _rgen(i, prefix='r'):
+ return "{}{}".format(prefix, i)
+
+
class TemporalStorageTestsBase(KayleeTest):
cls = TemporalStorage
def setUp(self):
pass
+ def test_init(self):
+ ts = self.cls()
+ self.assertIsInstance(ts, TemporalStorage)
+ self.assertEqual(len(ts), 0)
+
+ def test_add(self):
+ # add one
+ ts = self.cls()
+ node_id = NodeID()
+ ts.add('t1', node_id, 'r1')
+ self.assertEqual(list(ts['t1']), [(node_id, 'r1'), ])
+
+ # add many
+ ts = self.cls()
+ node_id = NodeID()
+ self._fill_storage(ts, MANY, node_id=node_id)
+ for i in range(0, MANY):
+ tid = _tgen(i)
+ for nid, res in ts[tid]:
+ self.assertEqual(nid, node_id)
+ self.assertEqual(res, _rgen(i))
+
+ def test_overwrite(self):
+ # overwrite one
+ ts = self.cls()
+ node_id = NodeID()
+ ts.add('t1', node_id, 'r1')
+ ts.add('t1', node_id, 'r2')
+ self.assertEqual(list(ts['t1']), [(node_id, 'r2'), ])
+
+ # overwrite many
+ ts = self.cls()
+ rgen = lambda i : "XYZ{}".format(i)
+ self._fill_storage(ts, SOME, node_id=node_id)
+ self._fill_storage(ts, SOME, rgen_func=rgen, node_id=node_id)
+
+ for i in range(0, SOME):
+ tid = _tgen(i)
+ for nid, res in ts[tid]:
+ self.assertEqual(nid, node_id)
+ self.assertEqual(res, rgen(i))
+
+ def test_remove_and_del(self):
+ # remove one
+ ts = self.cls()
+ node_id = NodeID()
+ ts.add('t1', node_id, 'r1')
+ ts.remove('t1')
+ self.assertRaises(KeyError, ts.__getitem__, 't1')
+ self.assertEqual(len(ts), 0)
+
+ ts.add('t1', node_id, 'r1')
+ self.assertRaises(KeyError, ts.remove, 't1', NodeID())
+ self.assertRaises(KeyError, ts.remove, 't2')
+ ts.remove('t1', node_id)
+ self.assertEqual(list(ts['t1']), [])
+ self.assertEqual(len(ts), 1)
+ ts.remove('t1')
+ self.assertEqual(len(ts), 0)
+ self.assertRaises(KeyError, ts.__getitem__, 't1')
+
+ # test del
+ ts.add('t3', node_id, 'r1')
+ del ts['t3']
+ self.assertRaises(KeyError, ts.__getitem__, 't3')
+ self.assertEqual(len(ts), 0)
+
+ # remove many
+ ts = self.cls()
+ self._fill_storage(ts, SOME, node_id=node_id)
+ self.assertEqual(len(ts), SOME)
+ for i in range(0, SOME):
+ tid = ts[_tget(i)]
+ del tid
+ self.assertRaises(KeyError, ts.__getitem__, tid)
+ self.assertEqual(len(ts), 0)
+
+
+ @staticmethod
+ def _fill_storage(ts, count, tgen_func=_tgen, rgen_func=_rgen, node_id=None):
+ if node_id is None:
+ # if node_id has not been defined by a user
+ # define callable _nodeid == NodeID
+ _nodeid = NodeID
+ else:
+ # a callable which returns the `node_id` argument
+ _nodeid = lambda: node_id
+ for i in range(0, count):
+ ts.add(tgen_func(i), _nodeid(), rgen_func(i))
+
class PermanentStorageTestsBase(KayleeTest):
cls = PermanentStorage # redefine in sub-classes
@@ -23,24 +121,25 @@ def setUp(self):
def test_init(self):
ps = self.cls()
+ self.assertIsInstance(ps, PermanentStorage)
self.assertEqual(len(ps), 0)
- def test_add_one(self):
+ def test_add(self):
ps = self.cls()
ps.add('1', 'r1')
self.assertEqual(ps['1'], ['r1'])
self.assertEqual(len(ps), 1)
self.assertEqual(list(iter(ps)), ['1'])
- def test_add_many(self):
+ # add many
ps = self.cls()
self._fill_storage(ps, MANY)
self.assertEqual(len(ps), MANY)
for i in range(0, MANY):
- task_id = self._tid(i)
- result = [self._rid(i)]
+ task_id = _tgen(i)
+ result = [_rgen(i)]
self.assertEqual(ps[task_id], result)
def test_add_same_task_results(self):
@@ -50,8 +149,8 @@ def test_add_same_task_results(self):
self._fill_storage(ps, SOME)
for i in range(0, SOME):
- task_id = self._tid(i)
- result = [self._rid(i)] * 3
+ task_id = _tgen(i)
+ result = [_rgen(i)] * 3
self.assertEqual(ps[task_id], result)
def test_add_complex_result(self):
@@ -73,13 +172,13 @@ def test_add_complex_result(self):
self.assertEqual(ps['1'], [res, res])
for i in range(0, SOME):
- ps.add(self._tid(i), deepcopy(res))
- self.assertEqual(ps[self._tid(i)], [res])
+ ps.add(_tgen(i), deepcopy(res))
+ self.assertEqual(ps[_tgen(i)], [res])
def test_keys_iter(self):
ps = self.cls()
self._fill_storage(ps, SOME)
- test_keys_set = set(self._tid(i) for i in range(0, SOME))
+ test_keys_set = set(_tgen(i) for i in range(0, SOME))
ps_iter_keys_set = set(iter(ps))
self.assertEqual(len(list(iter(ps))), SOME)
self.assertEqual(test_keys_set, ps_iter_keys_set)
@@ -88,7 +187,7 @@ def test_keys_iter(self):
def test_values_iter(self):
ps = self.cls()
self._fill_storage(ps, SOME)
- test_values_list = [[self._rid(i)] for i in range(0, SOME)]
+ test_values_list = [[_rgen(i)] for i in range(0, SOME)]
self.assertEqual(test_values_list, sorted(list(ps.values())) )
def test_count(self):
@@ -128,19 +227,21 @@ def test_contains(self):
self.assertTrue(ps.contains('t0', res))
- def _fill_storage(self, ps, count, tid_prefix='t', res_prefix='r'):
+ @staticmethod
+ def _fill_storage(ps, count, tgen_func=_tgen, rgen_func=_rgen):
for i in range(0, count):
- ps.add("{}{}".format(tid_prefix, i), "{}{}".format(res_prefix, i))
-
- def _tid(self, i, prefix='t'):
- return "{}{}".format(prefix, i)
-
- def _rid(self, i, prefix='r'):
- return "{}{}".format(prefix, i)
+ ps.add(tgen_func(i), _rgen(i))
class MemoryPermanentStorageTests(PermanentStorageTestsBase):
cls = MemoryPermanentStorage
-kaylee_suite = load_tests([MemoryPermanentStorageTests])
+class MemoryTemporalStorageTests(TemporalStorageTestsBase):
+ cls = MemoryTemporalStorage
+
+
+kaylee_suite = load_tests([
+ MemoryTemporalStorageTests,
+ MemoryPermanentStorageTests
+])

0 comments on commit 01796a6

Please sign in to comment.