Skip to content
Browse files

initial implementation of a umemcache based connector

  • Loading branch information...
1 parent cb7a32d commit a40a49df4e1aa8f6c12269f55c5c85fd8926a584 @tarekziade tarekziade committed Aug 1, 2012
View
1 CHANGES.rst
@@ -0,0 +1 @@
+
View
2 MANIFEST.in
@@ -0,0 +1,2 @@
+include README.rst
+include CHANGES.rst
View
0 memcachepool/__init__.py
No changes.
View
186 memcachepool/cache.py
@@ -0,0 +1,186 @@
+import time
+import Queue
+import sys
+import contextlib
+
+from django.core.cache.backends.memcached import MemcachedCache
+
+# Sentinel used to mark an empty slot in the MCClientPool queue.
+# Using sys.maxint as the timestamp ensures that empty slots will always
+# sort *after* live connection objects in the queue.
+EMPTY_SLOT = (sys.maxint, None)
+
+
+class ClientPool(object):
+
+ def __init__(self, factory, maxsize=None, timeout=60):
+ self.factory = factory
+ self.maxsize = maxsize
+ self.timeout = timeout
+ self.clients = Queue.PriorityQueue(maxsize)
+ # If there is a maxsize, prime the queue with empty slots.
+ if maxsize is not None:
+ for _ in xrange(maxsize):
+ self.clients.put(EMPTY_SLOT)
+
+ @contextlib.contextmanager
+ def reserve(self):
+ """Context-manager to obtain a Client object from the pool."""
+ ts, client = self._checkout_connection()
+ try:
+ yield client
+ finally:
+ self._checkin_connection(ts, client)
+
+ def _checkout_connection(self):
+ # If there's no maxsize, no need to block waiting for a connection.
+ blocking = self.maxsize is not None
+ # Loop until we get a non-stale connection, or we create a new one.
+ while True:
+ try:
+ ts, client = self.clients.get(blocking)
+ except Queue.Empty:
+ # No maxsize and no free connections, create a new one.
+ # XXX TODO: we should be using a monotonic clock here.
+ now = int(time.time())
+ return now, self.factory()
+ else:
+ now = int(time.time())
+ # If we got an empty slot placeholder, create a new connection.
+ if client is None:
+ return now, self.factory()
+ # If the connection is not stale, go ahead and use it.
+ if ts + self.timeout > now:
+ return ts, client
+ # Otherwise, the connection is stale.
+ # Close it, push an empty slot onto the queue, and retry.
+ client.disconnect_all()
+ self.clients.put(EMPTY_SLOT)
+ continue
+
+ def _checkin_connection(self, ts, client):
+ """Return a connection to the pool."""
+ # If the connection is now stale, don't return it to the pool.
+ # Push an empty slot instead so that it will be refreshed when needed.
+ now = int(time.time())
+ if ts + self.timeout > now:
+ self.clients.put((ts, client))
+ else:
+ if self.maxsize is not None:
+ self.clients.put(EMPTY_SLOT)
+
+
+# XXX not sure if keeping the base BaseMemcachedCache class has anymore value
+class UMemcacheCache(MemcachedCache):
+ "An implementation of a cache binding using python-memcached"
+ def __init__(self, server, params):
+ import umemcache
+ super(MemcachedCache, self).__init__(server, params,
+ library=umemcache,
+ value_not_found_exception=ValueError)
+ # see how to pass the pool value
+ self._pool = ClientPool(self._get_client)
+
+ def _get_client(self):
+ if len(self._servers) != 1:
+ raise ValueError('umemcached does not support several servers')
+
+ cli = self._lib.Client(self._servers[0])
+ cli.connect()
+ return cli
+
+ def add(self, key, value, timeout=0, version=None):
+ key = self.make_key(key, version=version)
+
+ with self._pool.reserve() as conn:
+ return conn.add(key, value, self._get_memcache_timeout(timeout))
+
+ def get(self, key, default=None, version=None):
+ key = self.make_key(key, version=version)
+ with self._pool.reserve() as conn:
+ val = conn.get(key)
+
+ if val is None:
+ return default
+ return val[0]
+
+ def set(self, key, value, timeout=0, version=None):
+ if not isinstance(value, str):
+ raise ValueError('Only string supported - you should serialize '
+ 'your data')
+
+ key = self.make_key(key, version=version)
+ with self._pool.reserve() as conn:
+ conn.set(key, value, self._get_memcache_timeout(timeout))
+
+ def delete(self, key, version=None):
+ key = self.make_key(key, version=version)
+ with self._pool.reserve() as conn:
+ conn.delete(key)
+
+ def get_many(self, keys, version=None):
+ new_keys = map(lambda x: self.make_key(x, version=version), keys)
+ with self._pool.reserve() as conn:
+ ret = conn.get_multi(new_keys)
+
+ if ret:
+ _ = {}
+ m = dict(zip(new_keys, keys))
+ for k, v in ret.items():
+ _[m[k]] = v
+ ret = _
+ return ret
+
+ def close(self, **kwargs):
+ # XXX none of your business Django
+ pass
+
+ def incr(self, key, delta=1, version=None):
+ key = self.make_key(key, version=version)
+ try:
+ with self._pool.reserve() as conn:
+ val = conn.incr(key, delta)
+
+ # python-memcache responds to incr on non-existent keys by
+ # raising a ValueError, pylibmc by raising a pylibmc.NotFound
+ # and Cmemcache returns None. In all cases,
+ # we should raise a ValueError though.
+ except self.LibraryValueNotFoundException:
+ val = None
+ if val is None:
+ raise ValueError("Key '%s' not found" % key)
+ return val
+
+ def decr(self, key, delta=1, version=None):
+ key = self.make_key(key, version=version)
+ try:
+ with self._pool.reserve() as conn:
+ val = conn.decr(key, delta)
+
+ # python-memcache responds to incr on non-existent keys by
+ # raising a ValueError, pylibmc by raising a pylibmc.NotFound
+ # and Cmemcache returns None. In all cases,
+ # we should raise a ValueError though.
+ except self.LibraryValueNotFoundException:
+ val = None
+ if val is None:
+ raise ValueError("Key '%s' not found" % key)
+ return val
+
+ def set_many(self, data, timeout=0, version=None):
+ safe_data = {}
+ for key, value in data.items():
+ key = self.make_key(key, version=version)
+ safe_data[key] = value
+
+ with self._pool.reserve() as conn:
+ conn.set_multi(safe_data, self._get_memcache_timeout(timeout))
+
+ def delete_many(self, keys, version=None):
+ l = lambda x: self.make_key(x, version=version)
+ with self._pool.reserve() as conn:
+ conn.delete_multi(map(l, keys))
+
+ def clear(self):
+ with self._pool.reserve() as conn:
+ conn._cache.flush_all()
View
5 memcachepool/tests/__init__.py
@@ -0,0 +1,5 @@
+import os
+
+
+def setUp():
+ os.environ['DJANGO_SETTINGS_MODULE'] = 'memcachepool.tests.settings'
View
1 memcachepool/tests/settings.py
@@ -0,0 +1 @@
+ok = 1
View
14 memcachepool/tests/test_cache.py
@@ -0,0 +1,14 @@
+from unittest import TestCase
+
+
+class TestCache(TestCase):
+
+ def test_pool(self):
+ from memcachepool.cache import UMemcacheCache
+
+ # creating the cache class
+ cache = UMemcacheCache('127.0.0.1:11211', {})
+
+ # simple calls
+ cache.set('a', '1')
+ self.assertEqual(cache.get('a'), '1')
View
36 setup.py
@@ -0,0 +1,36 @@
+import os
+from setuptools import setup, find_packages
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+with open(os.path.join(here, 'README.rst')) as f:
+ README = f.read()
+
+with open(os.path.join(here, 'CHANGES.rst')) as f:
+ CHANGES = f.read()
+
+
+requires = ['Django', 'umemcache']
+test_requires = ['nose']
+
+
+setup(name='django-memcached-pool',
+ version='0.1',
+ description='A Memcached Pool for Django',
+ long_description=README + '\n\n' + CHANGES,
+ classifiers=[
+ "Programming Language :: Python",
+ "Framework :: Pylons",
+ "Topic :: Internet :: WWW/HTTP",
+ "Topic :: Internet :: WWW/HTTP :: WSGI :: Application",
+ ],
+ author='Mozilla Services',
+ author_email='services-dev@mozilla.org',
+ url='https://github.com/mozilla/django-memcached-pool',
+ keywords='django memcached pool',
+ packages=find_packages(),
+ zip_safe=False,
+ install_requires=requires,
+ tests_require=test_requires,
+ test_suite="cornice")
+

0 comments on commit a40a49d

Please sign in to comment.
Something went wrong with that request. Please try again.