Permalink
Browse files

Fixed #11675 -- Added support for the PyLibMC cache library. In order…

… to support this, and clean up some other 1.3 caching additions, this patch also includes some changes to the way caches are defined. This means you can now have multiple caches, in the same way you have multiple databases. A huge thanks to Jacob Burch for the work on the PyLibMC backend, and to Jannis for his work on the cache definition changes.

git-svn-id: http://code.djangoproject.com/svn/django/trunk@15005 bcc190cf-cafb-0310-a4f2-bffc1f526a37
  • Loading branch information...
1 parent 3cf8502 commit 673e6fc7fb243ed44841b9969d26a161c25733b3 @freakboy3742 freakboy3742 committed Dec 21, 2010
@@ -431,14 +431,15 @@
# CACHE #
#########
+# New format
+CACHES = {
+}
# The cache backend to use. See the docstring in django.core.cache for the
# possible values.
CACHE_BACKEND = 'locmem://'
-CACHE_VERSION = 1
-CACHE_KEY_PREFIX = ''
-CACHE_KEY_FUNCTION = None
CACHE_MIDDLEWARE_KEY_PREFIX = ''
CACHE_MIDDLEWARE_SECONDS = 600
+CACHE_MIDDLEWARE_ALIAS = 'default'
####################
# COMMENTS #
@@ -1,5 +1,7 @@
import os
from django.conf import settings
+from django.core.cache import get_cache
+from django.core.cache.backends.db import BaseDatabaseCache
from django.core.exceptions import ImproperlyConfigured
from django.core.management import call_command
from django.db.backends.sqlite3.creation import DatabaseCreation
@@ -28,11 +30,12 @@ def create_test_db(self, verbosity=1, autoclobber=False):
self.load_spatialite_sql()
call_command('syncdb', verbosity=verbosity, interactive=False, database=self.connection.alias)
- if settings.CACHE_BACKEND.startswith('db://'):
- from django.core.cache import parse_backend_uri
- _, cache_name, _ = parse_backend_uri(settings.CACHE_BACKEND)
- call_command('createcachetable', cache_name)
-
+ for cache_alias in settings.CACHES:
+ cache = get_cache(cache_alias)
+ if isinstance(cache, BaseDatabaseCache):
+ from django.db import router
+ if router.allow_syncdb(self.connection.alias, cache.cache_model_class):
+ call_command('createcachetable', cache._table, database=self.connection.alias)
# Get a cursor (even though we don't need one yet). This has
# the side effect of initializing the test database.
cursor = self.connection.cursor()
@@ -12,8 +12,13 @@
(e.g. "memcached://127.0.0.1:11211/") and returns an instance of a backend
cache class.
-See docs/cache.txt for information on the public API.
+See docs/topics/cache.txt for information on the public API.
"""
+from django.conf import settings
+from django.core import signals
+from django.core.cache.backends.base import (
+ InvalidCacheBackendError, CacheKeyWarning, BaseCache)
+from django.utils import importlib
try:
# The mod_python version is more efficient, so try importing it first.
@@ -27,10 +32,9 @@
# PendingDeprecationWarning
from cgi import parse_qsl
-from django.conf import settings
-from django.core import signals
-from django.core.cache.backends.base import InvalidCacheBackendError, CacheKeyWarning
-from django.utils import importlib
+__all__ = [
+ 'get_cache', 'cache', 'DEFAULT_CACHE_ALIAS'
+]
# Name for use in settings file --> name of module in "backends" directory.
# Any backend scheme that is not in this dictionary is treated as a Python
@@ -43,6 +47,8 @@
'dummy': 'dummy',
}
+DEFAULT_CACHE_ALIAS = 'default'
+
def parse_backend_uri(backend_uri):
"""
Converts the "backend_uri" into a cache scheme ('db', 'memcached', etc), a
@@ -67,32 +73,102 @@ def parse_backend_uri(backend_uri):
return scheme, host, params
-def get_cache(backend_uri, key_prefix=None, version=None, key_func=None):
- if key_prefix is None:
- key_prefix = settings.CACHE_KEY_PREFIX
- if version is None:
- version = settings.CACHE_VERSION
- if key_func is None:
- key_func = settings.CACHE_KEY_FUNCTION
-
- if key_func is not None and not callable(key_func):
- key_func_module_path, key_func_name = key_func.rsplit('.', 1)
- key_func_module = importlib.import_module(key_func_module_path)
- key_func = getattr(key_func_module, key_func_name)
-
- scheme, host, params = parse_backend_uri(backend_uri)
- if scheme in BACKENDS:
- name = 'django.core.cache.backends.%s' % BACKENDS[scheme]
+if not settings.CACHES:
+ import warnings
+ warnings.warn(
+ "settings.CACHE_* is deprecated; use settings.CACHES instead.",
+ PendingDeprecationWarning
+ )
+ # Mapping for new-style cache backend api
+ backend_classes = {
+ 'memcached': 'memcached.CacheClass',
+ 'locmem': 'locmem.LocMemCache',
+ 'file': 'filebased.FileBasedCache',
+ 'db': 'db.DatabaseCache',
+ 'dummy': 'dummy.DummyCache',
+ }
+ engine, host, params = parse_backend_uri(settings.CACHE_BACKEND)
+ if engine in backend_classes:
+ engine = 'django.core.cache.backends.%s' % backend_classes[engine]
+ defaults = {
+ 'BACKEND': engine,
+ 'LOCATION': host,
+ }
+ defaults.update(params)
+ settings.CACHES[DEFAULT_CACHE_ALIAS] = defaults
+
+if DEFAULT_CACHE_ALIAS not in settings.CACHES:
+ raise ImproperlyConfigured("You must define a '%s' cache" % DEFAULT_CACHE_ALIAS)
+
+def parse_backend_conf(backend, **kwargs):
+ """
+ Helper function to parse the backend configuration
+ that doesn't use the URI notation.
+ """
+ # Try to get the CACHES entry for the given backend name first
+ conf = settings.CACHES.get(backend, None)
+ if conf is not None:
+ args = conf.copy()
+ backend = args.pop('BACKEND')
+ location = args.pop('LOCATION', '')
+ return backend, location, args
else:
- name = scheme
- module = importlib.import_module(name)
- return module.CacheClass(host, params, key_prefix=key_prefix, version=version, key_func=key_func)
+ # Trying to import the given backend, in case it's a dotted path
+ mod_path, cls_name = backend.rsplit('.', 1)
+ try:
+ mod = importlib.import_module(mod_path)
+ backend_cls = getattr(mod, cls_name)
+ except (AttributeError, ImportError):
+ raise InvalidCacheBackendError("Could not find backend '%s'" % backend)
+ location = kwargs.pop('LOCATION', '')
+ return backend, location, kwargs
+ raise InvalidCacheBackendError(
+ "Couldn't find a cache backend named '%s'" % backend)
-cache = get_cache(settings.CACHE_BACKEND)
+def get_cache(backend, **kwargs):
+ """
+ Function to load a cache backend dynamically. This is flexible by design
+ to allow different use cases:
+
+ To load a backend with the old URI-based notation::
+
+ cache = get_cache('locmem://')
+
+ To load a backend that is pre-defined in the settings::
+
+ cache = get_cache('default')
+
+ To load a backend with its dotted import path,
+ including arbitrary options::
+
+ cache = get_cache('django.core.cache.backends.memcached.MemcachedCache', **{
+ 'LOCATION': '127.0.0.1:11211', 'TIMEOUT': 30,
+ })
+
+ """
+ try:
+ if '://' in backend:
+ # for backwards compatibility
+ backend, location, params = parse_backend_uri(backend)
+ if backend in BACKENDS:
+ backend = 'django.core.cache.backends.%s' % BACKENDS[backend]
+ params.update(kwargs)
+ mod = importlib.import_module(backend)
+ backend_cls = mod.CacheClass
+ else:
+ backend, location, params = parse_backend_conf(backend, **kwargs)
+ mod_path, cls_name = backend.rsplit('.', 1)
+ mod = importlib.import_module(mod_path)
+ backend_cls = getattr(mod, cls_name)
+ except (AttributeError, ImportError), e:
+ raise InvalidCacheBackendError(
+ "Could not find backend '%s': %s" % (backend, e))
+ return backend_cls(location, params)
+
+cache = get_cache(DEFAULT_CACHE_ALIAS)
# Some caches -- python-memcached in particular -- need to do a cleanup at the
# end of a request cycle. If the cache provides a close() method, wire it up
# here.
if hasattr(cache, 'close'):
signals.request_finished.connect(cache.close)
-
@@ -2,8 +2,10 @@
import warnings
+from django.conf import settings
from django.core.exceptions import ImproperlyConfigured, DjangoRuntimeWarning
from django.utils.encoding import smart_str
+from django.utils.importlib import import_module
class InvalidCacheBackendError(ImproperlyConfigured):
pass
@@ -15,38 +17,55 @@ class CacheKeyWarning(DjangoRuntimeWarning):
MEMCACHE_MAX_KEY_LENGTH = 250
def default_key_func(key, key_prefix, version):
- """Default function to generate keys.
+ """
+ Default function to generate keys.
Constructs the key used by all other methods. By default it prepends
- the `key_prefix'. CACHE_KEY_FUNCTION can be used to specify an alternate
+ the `key_prefix'. KEY_FUNCTION can be used to specify an alternate
function with custom key making behavior.
"""
return ':'.join([key_prefix, str(version), smart_str(key)])
+def get_key_func(key_func):
+ """
+ Function to decide which key function to use.
+
+ Defaults to ``default_key_func``.
+ """
+ if key_func is not None:
+ if callable(key_func):
+ return key_func
+ else:
+ key_func_module_path, key_func_name = key_func.rsplit('.', 1)
+ key_func_module = import_module(key_func_module_path)
+ return getattr(key_func_module, key_func_name)
+ return default_key_func
+
class BaseCache(object):
- def __init__(self, params, key_prefix='', version=1, key_func=None):
- timeout = params.get('timeout', 300)
+ def __init__(self, params):
+ timeout = params.get('timeout', params.get('TIMEOUT', 300))
try:
timeout = int(timeout)
except (ValueError, TypeError):
timeout = 300
self.default_timeout = timeout
- max_entries = params.get('max_entries', 300)
+ options = params.get('OPTIONS', {})
+ max_entries = params.get('max_entries', options.get('MAX_ENTRIES', 300))
try:
self._max_entries = int(max_entries)
except (ValueError, TypeError):
self._max_entries = 300
- cull_frequency = params.get('cull_frequency', 3)
+ cull_frequency = params.get('cull_frequency', options.get('CULL_FREQUENCY', 3))
try:
self._cull_frequency = int(cull_frequency)
except (ValueError, TypeError):
self._cull_frequency = 3
- self.key_prefix = smart_str(key_prefix)
- self.version = version
- self.key_func = key_func or default_key_func
+ self.key_prefix = smart_str(params.get('KEY_PREFIX', ''))
+ self.version = params.get('VERSION', 1)
+ self.key_func = get_key_func(params.get('KEY_FUNCTION', None))
def make_key(self, key, version=None):
"""Constructs the key used by all other methods. By default it
@@ -25,16 +25,16 @@ def __init__(self, table):
self.managed = True
self.proxy = False
-class BaseDatabaseCacheClass(BaseCache):
- def __init__(self, table, params, key_prefix='', version=1, key_func=None):
- BaseCache.__init__(self, params, key_prefix, version, key_func)
+class BaseDatabaseCache(BaseCache):
+ def __init__(self, table, params):
+ BaseCache.__init__(self, params)
self._table = table
class CacheEntry(object):
_meta = Options(table)
self.cache_model_class = CacheEntry
-class CacheClass(BaseDatabaseCacheClass):
+class DatabaseCache(BaseDatabaseCache):
def get(self, key, default=None, version=None):
key = self.make_key(key, version=version)
self.validate_key(key)
@@ -140,3 +140,7 @@ def clear(self):
table = connections[db].ops.quote_name(self._table)
cursor = connections[db].cursor()
cursor.execute('DELETE FROM %s' % table)
+
+# For backwards compatibility
+class CacheClass(DatabaseCache):
+ pass
@@ -2,7 +2,7 @@
from django.core.cache.backends.base import BaseCache
-class CacheClass(BaseCache):
+class DummyCache(BaseCache):
def __init__(self, host, *args, **kwargs):
BaseCache.__init__(self, *args, **kwargs)
@@ -40,3 +40,7 @@ def delete_many(self, keys, version=None):
def clear(self):
pass
+
+# For backwards compatibility
+class CacheClass(DummyCache):
+ pass
@@ -11,9 +11,9 @@
from django.core.cache.backends.base import BaseCache
from django.utils.hashcompat import md5_constructor
-class CacheClass(BaseCache):
- def __init__(self, dir, params, key_prefix='', version=1, key_func=None):
- BaseCache.__init__(self, params, key_prefix, version, key_func)
+class FileBasedCache(BaseCache):
+ def __init__(self, dir, params):
+ BaseCache.__init__(self, params)
self._dir = dir
if not os.path.exists(self._dir):
self._createdir()
@@ -161,3 +161,7 @@ def clear(self):
shutil.rmtree(self._dir)
except (IOError, OSError):
pass
+
+# For backwards compatibility
+class CacheClass(FileBasedCache):
+ pass
@@ -9,12 +9,19 @@
from django.core.cache.backends.base import BaseCache
from django.utils.synch import RWLock
-class CacheClass(BaseCache):
- def __init__(self, _, params, key_prefix='', version=1, key_func=None):
- BaseCache.__init__(self, params, key_prefix, version, key_func)
- self._cache = {}
- self._expire_info = {}
- self._lock = RWLock()
+# Global in-memory store of cache data. Keyed by name, to provide
+# multiple named local memory caches.
+_caches = {}
+_expire_info = {}
+_locks = {}
+
+class LocMemCache(BaseCache):
+ def __init__(self, name, params):
+ BaseCache.__init__(self, params)
+ global _caches, _expire_info, _locks
+ self._cache = _caches.setdefault(name, {})
+ self._expire_info = _expire_info.setdefault(name, {})
+ self._lock = _locks.setdefault(name, RWLock())
def add(self, key, value, timeout=None, version=None):
key = self.make_key(key, version=version)
@@ -133,3 +140,7 @@ def delete(self, key, version=None):
def clear(self):
self._cache.clear()
self._expire_info.clear()
+
+# For backwards compatibility
+class CacheClass(LocMemCache):
+ pass
Oops, something went wrong.

0 comments on commit 673e6fc

Please sign in to comment.