Skip to content

Loading…

Added native data type support via cPickle serialization; PEP 8 cleanup #3

Open
wants to merge 7 commits into from

2 participants

@wayoutmind

Feel free to disregard the fork specific README commits, but I added support for native data types by serializing elements in cPickle. Let me know what you think.

For example:
from qr import Queue
queue = Queue('Test')
queue.push(1)
type(queue.pop())
>>>

"""Original implementation simply returns strings,
regardless of data type pushed into data structure"""

@fictorial

@wayoutmind: In c611ac, doesn't this create a new connection for each instance of Queue et al.?

@wayoutmind

@fictorial If you pass connection_pool when instantiating then it will not create a new connection for each instance

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Showing with 91 additions and 52 deletions.
  1. +3 −1 README.md
  2. +80 −43 qr.py
  3. +8 −8 test/tests.py
View
4 README.md
@@ -1,9 +1,11 @@
QR
=====
-**QR** helps you create and work with **queue, capped collection (bounded queue), deque, and stack** data structures for **Redis**.
+**QR** helps you create and work with **queue, capped collection (bounded queue), deque, and stack** data structures for **Redis**.
Redis is well-suited for implementations of these abstract data structures, and QR makes it even easier to work with the structures in Python.
+*Note: [stackd/qr](https://github.com/stackd/qr "stackd/qr") builds upon the work of [tnm/qr](https://github.com/tnm/qr "tnm/qr") by serializing data structure elements with [cPickle](http://docs.python.org/library/pickle.html#module-cPickle) out of the box. This conveniently allows elements to be returned in their native types, as opposed to <'str'>, without extra boilerplate.*
+
Quick Setup
------------
You'll need [Redis](http://github.com/antirez/redis/ "Redis") itself -- QR makes use of MULTI/EXEC, so you'll need Redis 2.0 or
View
123 qr.py
@@ -1,6 +1,7 @@
"""
QR | Redis-Based Data Structures in Python
+ 1 Apr 2011 | Native data types; added ability to pass redis config arguments
16 Dec 2010 | Add more logging (0.2.1)
26 Apr 2010 | Major API change; capped collections, remove autopop (0.2.0)
7 Mar 2010 | Auto popping for bounded queues is optional (0.1.4)
@@ -13,7 +14,9 @@
__author__ = 'Ted Nyman'
__version__ = '0.2.1'
__license__ = 'MIT'
+__maintainer__ = 'Fredrick Galoso'
+import cPickle as pickle
import redis
import logging
@@ -22,157 +25,191 @@
except ImportError:
import simplejson as json
-# redis interface
-redis = redis.Redis()
+
class NullHandler(logging.Handler):
"""A logging handler that discards all logging records"""
def emit(self, record):
pass
+
# Clients can add handlers if they are interested.
log = logging.getLogger('qr')
log.addHandler(NullHandler())
-
-class Deque(object):
+
+
+class Redis(object):
+ """Base class for passing Redis params"""
+
+ def __init__(self, host='localhost', port=6379, db=0,
+ password=None, socket_timeout=None, connection_pool=None,
+ charset='utf-8', errors='strict'):
+ self.redis = redis.Redis(host, port, db, password,
+ socket_timeout, connection_pool,
+ charset, errors)
+
+
+class Deque(Redis):
"""Implements a double-ended queue"""
- def __init__(self, key):
+ def __init__(self, key, **kwargs):
self.key = key
+ super(Deque, self).__init__(**kwargs)
def pushback(self, element):
"""Push an element to the back"""
key = self.key
- push_it = redis.lpush(key, element)
+ push_it = self.redis.lpush(key, pickle.dumps(element))
log.debug('Pushed ** %s ** for key ** %s **' % (element, key))
-
+
def pushfront(self, element):
"""Push an element to the front"""
key = self.key
- push_it = redis.rpush(key, element)
+ push_it = self.redis.rpush(key, pickle.dumps(element))
log.debug('Pushed ** %s ** for key ** %s **' % (element, key))
def popfront(self):
"""Pop an element from the front"""
key = self.key
- popped = redis.rpop(key)
+ popped = self.redis.rpop(key)
log.debug('Popped ** %s ** from key ** %s **' % (popped, key))
- return popped
+ try:
+ return pickle.loads(popped)
+ except TypeError:
+ return None
def popback(self):
"""Pop an element from the back"""
key = self.key
- popped = redis.lpop(key)
+ popped = self.redis.lpop(key)
log.debug('Popped ** %s ** from key ** %s **' % (popped, key))
- return popped
+ try:
+ return pickle.loads(popped)
+ except TypeError:
+ return None
def elements(self):
"""Return all elements as a Python list"""
key = self.key
- all_elements = redis.lrange(key, 0, -1)
+ all_elements = self.redis.lrange(key, 0, -1)
return all_elements
-
+
def elements_as_json(self):
"""Return all elements as a JSON object"""
key = self.key
- all_elements = redis.lrange(key, 0, -1)
+ all_elements = [pickle.loads(item) for item in self.redis.lrange(key, 0, -1)]
all_elements_as_json = json.dumps(all_elements)
return all_elements_as_json
-class Queue(object):
+
+class Queue(Redis):
"""Implements a FIFO queue"""
- def __init__(self, key):
+ def __init__(self, key, **kwargs):
self.key = key
-
+ super(Queue, self).__init__(**kwargs)
+
def push(self, element):
"""Push an element"""
key = self.key
- push_it = redis.lpush(key, element)
+ push_it = self.redis.lpush(key, pickle.dumps(element))
log.debug('Pushed ** %s ** for key ** %s **' % (element, key))
def pop(self):
"""Pop an element"""
key = self.key
- popped = redis.rpop(key)
+ popped = self.redis.rpop(key)
log.debug('Popped ** %s ** from key ** %s **' % (popped, key))
- return popped
+ try:
+ return pickle.loads(popped)
+ except TypeError:
+ return None
def elements(self):
"""Return all elements as a Python list"""
key = self.key
- all_elements = redis.lrange(key, 0, -1) or [ ]
+ all_elements = self.redis.lrange(key, 0, -1) or []
return all_elements
-
+
def elements_as_json(self):
- """Return all elements as a JSON object"""
+ """Return all elements as a JSON object"""
key = self.key
- all_elements = redis.lrange(key, 0, -1) or [ ]
+ all_elements = [pickle.loads(item) for item in self.redis.lrange(key, 0, -1)] or []
all_elements_as_json = json.dumps(all_elements)
return all_elements_as_json
-class CappedCollection(object):
+
+class CappedCollection(Redis):
"""Implements a capped collection (the collection never
gets larger than the specified size)."""
- def __init__(self, key, size):
+ def __init__(self, key, size, **kwargs):
self.key = key
self.size = size
+ super(CappedCollection, self).__init__(**kwargs)
def push(self, element):
key = self.key
size = self.size
- pipe = redis.pipeline() # Use multi-exec command via redis-py pipelining
- pipe = pipe.lpush(key, element).ltrim(key, 0, size-1) # ltrim is zero-indexed
+ pipe = self.redis.pipeline() # Use multi-exec command via self.redis-py pipelining
+ pipe = pipe.lpush(key, pickle.dumps(element)).ltrim(key, 0, size - 1) # ltrim is zero-indexed
pipe.execute()
def pop(self):
key = self.key
- popped = redis.rpop(key)
+ popped = self.redis.rpop(key)
log.debug('Popped ** %s ** from key ** %s **' % (popped, key))
- return popped
+ try:
+ return pickle.loads(popped)
+ except TypeError:
+ return None
def elements(self):
"""Return all elements as Python list"""
key = self.key
- all_elements = redis.lrange(key, 0, -1)
+ all_elements = self.redis.lrange(key, 0, -1)
return all_elements
def elements_as_json(self):
"""Return all elements as JSON object"""
key = self.key
- all_elements = redis.lrange(key, 0, -1)
+ all_elements = [pickle.loads(item) for item in self.redis.lrange(key, 0, -1)]
all_elements_as_json = json.dumps(all_elements)
return all_elements_as_json
-class Stack(object):
- """Implements a LIFO stack"""
- def __init__(self, key):
+class Stack(Redis):
+ """Implements a LIFO stack"""
+
+ def __init__(self, key, **kwargs):
self.key = key
+ super(Stack, self).__init__(**kwargs)
def push(self, element):
"""Push an element"""
key = self.key
- push_it = redis.lpush(key, element)
+ push_it = self.redis.lpush(key, pickle.dumps(element))
log.debug('Pushed ** %s ** for key ** %s **' % (element, key))
-
+
def pop(self):
"""Pop an element"""
key = self.key
- popped = redis.lpop(key)
+ popped = self.redis.lpop(key)
log.debug('Popped ** %s ** from key ** %s **' % (popped, key))
- return popped
-
+ try:
+ return pickle.loads(popped)
+ except TypeError:
+ return None
+
def elements(self):
"""Return all elements as Python list"""
key = self.key
- all_elements = redis.lrange(key, 0, -1)
+ all_elements = self.redis.lrange(key, 0, -1)
return all_elements
def elements_as_json(self):
"""Return all elements as JSON object"""
key = self.key
- all_elements = redis.lrange(key, 0, -1)
+ all_elements = [pickle.loads(item) for item in self.redis.lrange(key, 0, -1)]
all_elements_as_json = json.dumps(all_elements)
return all_elements_as_json
View
16 test/tests.py
@@ -4,10 +4,11 @@
r = redis.Redis()
+
class Queue(unittest.TestCase):
def setUp(self):
- r.delete('qrtestqueue')
- self.q = qr.Queue(key='qrtestqueue')
+ r.delete('qrtestqueue')
+ self.q = qr.Queue(key='qrtestqueue', host='localhost', port=6379)
self.assertEquals(len(self.q.elements()), 0)
def test_roundtrip(self):
@@ -31,10 +32,11 @@ def test_order_mixed(self):
q.push('bar')
self.assertEquals(q.pop(), 'bar')
+
class CappedCollection(unittest.TestCase):
def setUp(self):
- r.delete('qrtestcc')
- self.aq = qr.CappedCollection(key='qrtestcc', size=3)
+ r.delete('qrtestcc')
+ self.aq = qr.CappedCollection(key='qrtestcc', size=3)
self.assertEquals(len(self.aq.elements()), 0)
def test_roundtrip(self):
@@ -72,9 +74,10 @@ def test_limit(self):
self.assertEquals(aq.pop(), 'e')
self.assertEquals(len(aq.elements()), 0)
+
class Stack(unittest.TestCase):
def setUp(self):
- r.delete('qrteststack')
+ r.delete('qrteststack')
self.stack = qr.Stack(key='qrteststack')
def test_roundtrip(self):
@@ -100,6 +103,3 @@ def test_order_mixed(self):
if __name__ == '__main__':
unittest.main()
-
-
-
Something went wrong with that request. Please try again.