Merge pull request #4 from pior/master

Add a caching extension for redis

.travis.yml

@@ -7,6 +7,7 @@ install: "pip install -r requirements-test.txt && pip install -e ."
 
 services:
   - memcached
+  - redis-server
 
 script:  "python setup.py nosetests"
 

CHANGES.rst

@@ -17,7 +17,7 @@ Development
   method.
 * Add a cache manager which plug all components.
 * Add serializers to adapt objects to store on cache
-* A quick wrapper around ``umemcache.Client``
+* Add Redis client for caching and versioning
 * TODO:
   * Ability to activate / deactivate cache
   * introspectables

example/__init__.py

@@ -2,6 +2,6 @@
 
 def includeme(config):  # pragma no cover
     config.include('pyramid_caching')
-    config.include('pyramid_caching.ext.umemcache')
+    config.include('pyramid_caching.ext.redis')
     config.include('.model')
     config.include('.views')

pyramid_caching/__init__.py

@@ -1,6 +1,7 @@
 
 
 def includeme(config):
-    config.include('.versioners')
+    config.include('.versioner')
+    config.include('.key_versioner')
     config.include('.serializers')
     config.include('.cache')

pyramid_caching/cache.py

@@ -29,7 +29,7 @@ def register():
         config.registry.registerUtility(manager)
         log.debug('registering cache manager %r', manager)
 
-    config.action((__name__, 'cache_manager'), register, order=1)
+    config.action((__name__, 'cache_manager'), register, order=2)
 
 
 def get_cache_client(config_or_request):

pyramid_caching/exc.py

@@ -0,0 +1,10 @@
+
+
+class Base(Exception):
+    """Base exception for pyramid_caching"""
+
+class CacheError(Base):
+    """Base exception for cache client"""
+
+class CacheKeyAlreadyExists(CacheError):
+    """Trying to create an existing key in cache"""

pyramid_caching/ext/redis.py

@@ -0,0 +1,78 @@
+from __future__ import absolute_import
+
+import os
+
+from redis import StrictRedis
+
+from pyramid_caching.exc import (CacheError, CacheKeyAlreadyExists)
+
+def includeme(config):
+    include_cache_store(config)
+    include_version_store(config)
+
+def include_cache_store(config):
+    uri = os.environ['CACHE_STORE_REDIS_URI']
+    client = StrictRedis.from_url(uri)
+    config.add_cache_client(RedisCacheWrapper(client))
+
+def include_version_store(config):
+    uri = os.environ['VERSION_STORE_REDIS_URI']
+    client = StrictRedis.from_url(uri)
+    config.add_key_version_client(RedisVersionWrapper(client))
+
+
+class RedisCacheWrapper(object):
+
+    def __init__(self, client):
+        self.default_expiration = 3600 * 24 * 7  # 7 days
+        self.client = client
+
+    def add(self, key, value, expiration=None):
+        """
+        Note: Redis will only LRU evince volatile keys.
+
+        Default expiration: 7 days
+        """
+        if expiration is None:
+            expiration = self.default_expiration
+
+        rvalue = self.client.set(key, value, ex=expiration, nx=True)
+        if rvalue is None:
+            raise CacheKeyAlreadyExists(key)
+
+    def get(self, key):
+        return self.client.get(key)
+
+    def flush_all(self):
+        self.client.flushall()
+
+
+class RedisVersionWrapper(object):
+    """Redis implementation of the IKeyVersioner interface.
+
+    This KeyVersioner use Redis ability to namespace content with seperate
+    db, thus not prefixing the keys with anything like 'version:'.
+
+    The default value for this implementation is 0 since the INCR operation
+    enforce a value of 1 after executing on a non-existing key.
+    See `Redis documentation for INCR <http://redis.io/commands/incr>`_
+
+    Note: the return value are always string type.
+    """
+
+    def __init__(self, client):
+        self.client = client
+
+    def get(self, key):
+        value = self.client.get(key)
+        return value if value is not None else '0'
+
+    def get_multi(self, keys):
+        return [v if v is not None else '0'
+                for v in self.client.mget(keys)]
+
+    def incr(self, key):
+        self.client.incr(key)
+
+    def flush_all(self):
+        self.client.flushall()

pyramid_caching/ext/sqlalchemy.py

@@ -42,26 +42,31 @@ def identify(model):
 
 
 def register_sqla_session_caching_hook(config, session_cls):
-    versioner = config.get_versioner()
+    def register():
+        versioner = config.get_versioner()
 
-    def on_before_commit(session):
-        dirty = session.dirty
-        deleted = session.deleted
+        def on_before_commit(session):
+            dirty = session.dirty
+            deleted = session.deleted
 
-        def incr_models(session):
-            # XXX: should increment only cacheable models
-            log.debug('incrementing dirty=%s deleted=%s', dirty, deleted)
+            def incr_models(session):
+                # XXX: should increment only cacheable models
+                log.debug('incrementing dirty=%s deleted=%s', dirty, deleted)
 
-            for model in dirty:
-                versioner.incr(model)
+                for model in dirty:
+                    versioner.incr(model)
 
-            for model in deleted:
-                versioner.incr(model)
+                for model in deleted:
+                    versioner.incr(model)
+
+            if dirty or deleted:
+                event.listen(session, 'after_commit', incr_models)
+
+        event.listen(session_cls, 'before_commit', on_before_commit)
+
+    config.action((__name__, 'session_caching_hook'), register, order=3)
 
-        if dirty or deleted:
-            event.listen(session, 'after_commit', incr_models)
 
-    event.listen(session_cls, 'before_commit', on_before_commit)
 
 
 @implementer(IIdentityInspector)

pyramid_caching/interfaces.py

@@ -2,27 +2,35 @@
 
 
 class IKeyVersioner(Interface):
+    """
+    Notes about the default values:
 
-    def get(key, default=0):
+    There is two default values :
+    - the return value of get()/get_multi() when the version doesn't exist
+    - the stored version after an incr() operation
+
+    Those default values are implementation specific and MUST be equivalent.
+
+    See Redis implementation.
+    """
+
+    def get(key):
         """Get a key's version.
 
-        If found, key's version is returned, else ``default``.
+        If found, key's version is returned, else a default value. Defaults is
+        implementation specific.
         """
 
-    def get_multi(keys, default=0):
+    def get_multi(keys):
         """Get versions for a list of keys.
 
-        If found, a dict containing keys with their corresponding versions
-        is returned, else the keys versions are initialized to ``default``
-        i.e., {'key': 0, ...}.
-
-        Prefix 'ver_' is added to each input key for Memcache
-        query, however prefix is removed before returning results.
+        Return a list of corresponding versions. Defaults is implementation
+        specific.
         """
 
-    def incr(key, start=0):
-        """Increment an existing key version or add a new version starting at
-        ``start``.
+    def incr(key):
+        """Increment an existing key version. Defaults is implementation
+        specific.
         """
 
 
@@ -37,7 +45,7 @@ def get_multi_keys(objects_or_classes):
 
         .. code-block:: ipython
 
-            >>> from pyramid_caching.versioners import Versioner
+            >>> from pyramid_caching.versioner import Versioner
             >>> from myapp.model import MyModel
             >>> versioner = Versioner()
             >>> res = versioner.get_class_keys([MyModel])
@@ -65,12 +73,6 @@ def add(key, obj):
     def get(key):
         pass
 
-    def get_multi(keys):
-        pass
-
-    def incr(key):
-        pass
-
 
 class ICacheManager(Interface):
     pass

pyramid_caching/key_versioner.py

@@ -0,0 +1,29 @@
+import logging
+
+from zope.interface import classImplements
+
+from pyramid_caching.interfaces import IKeyVersioner
+
+log = logging.getLogger(__name__)
+
+
+def includeme(config):
+    config.add_directive('get_key_version_client', get_key_version_client,
+                         action_wrap=False)
+    config.add_directive('add_key_version_client', add_key_version_client)
+
+
+def get_key_version_client(config_or_request):
+    return config_or_request.registry.getUtility(IKeyVersioner)
+
+
+def add_key_version_client(config, key_versioner):
+    if not IKeyVersioner.implementedBy(key_versioner.__class__):
+        log.debug('assuming %r implements %r', key_versioner.__class__, IKeyVersioner)
+        classImplements(key_versioner.__class__, IKeyVersioner)
+
+    def register():
+        log.debug('registering KeyVersioner %r', key_versioner)
+        config.registry.registerUtility(key_versioner, IKeyVersioner)
+
+    config.action((__name__, 'key_version_client'), register, order=0)

pyramid_caching/tests/functional/__init__.py

@@ -8,7 +8,8 @@
 
 
 def setupPackage():
-    os.environ['MEMCACHE_URI'] = '127.0.0.1:11211'
+    os.environ['CACHE_STORE_REDIS_URI'] = 'redis://127.0.0.1:6379/5'
+    os.environ['VERSION_STORE_REDIS_URI'] = 'redis://127.0.0.1:6379/8'
 
 
 class Base(unittest.TestCase):

pyramid_caching/tests/unittests/test_key_versioners.py

@@ -0,0 +1,73 @@
+import unittest
+
+from nose_parameterized import parameterized
+
+from pyramid_caching.ext.redis import RedisVersionWrapper
+
+from redis import StrictRedis
+
+def get_redis():
+    return RedisVersionWrapper(StrictRedis())
+
+KEY_VERSIONERS = [
+    ("redis", get_redis, '0'),
+]
+
+class TestKeyVersioner(unittest.TestCase):
+
+    TEST_VALUE = '4'  # Best creative testing value
+
+    def setUp(self):
+        get_redis().flush_all()
+
+    @parameterized.expand(KEY_VERSIONERS)
+    def test_interface(self, name, get_key_versioner, default_value):
+        from zope.interface.verify import verifyObject
+        from pyramid_caching.interfaces import IKeyVersioner
+
+        key_versioner = get_key_versioner()
+
+        self.assertTrue(verifyObject(IKeyVersioner, key_versioner))
+
+    @parameterized.expand(KEY_VERSIONERS)
+    def test_get(self, name, get_key_versioner, default_value):
+        key_versioner = get_key_versioner()
+
+
+        self.assertEqual(key_versioner.get('FOO'), default_value)
+
+        self.assertEqual(key_versioner.get('FOO'), default_value)
+
+        for _ in range(int(self.TEST_VALUE)):
+            key_versioner.incr('FOO')
+
+        self.assertEqual(key_versioner.get('FOO'), self.TEST_VALUE)
+
+        self.assertEqual(key_versioner.get('FOO'), self.TEST_VALUE)
+
+    @parameterized.expand(KEY_VERSIONERS)
+    def test_get_multi(self, name, get_key_versioner, default_value):
+        key_versioner = get_key_versioner()
+
+        KEYS = ['FOO', 'BAR', '2000']
+        VERSIONS = ['0', '0', '0']
+
+        self.assertEqual(key_versioner.get_multi(KEYS), VERSIONS)
+
+        self.assertEqual(key_versioner.get_multi(KEYS), VERSIONS)
+
+        for _ in range(int(self.TEST_VALUE)):
+            key_versioner.incr('BAR')
+        VERSIONS = ['0', str(self.TEST_VALUE), '0']
+
+        self.assertEqual(key_versioner.get_multi(KEYS), VERSIONS)
+
+        self.assertEqual(key_versioner.get_multi(KEYS), VERSIONS)
+
+        for _ in range(42):
+            key_versioner.incr('2000')
+        VERSIONS = ['0', str(self.TEST_VALUE), '42']
+
+        self.assertEqual(key_versioner.get_multi(KEYS), VERSIONS)
+
+        self.assertEqual(key_versioner.get_multi(KEYS), VERSIONS)