Fetching contributors…
Cannot retrieve contributors at this time
335 lines (263 sloc) 10.6 KB
import functools
import logging
from django.conf import settings
from django.core.cache import cache
from django.db import models
from django.db.models import signals
from django.db.models.sql import query
from django.utils import encoding
from .invalidation import invalidator, flush_key, make_key, byid
class NullHandler(logging.Handler):
def emit(self, record):
log = logging.getLogger('caching')
CACHE_PREFIX = getattr(settings, 'CACHE_PREFIX', '')
FETCH_BY_ID = getattr(settings, 'FETCH_BY_ID', False)
class CachingManager(models.Manager):
# Tell Django to use this manager when resolving foreign keys.
use_for_related_fields = True
def get_query_set(self):
return CachingQuerySet(self.model)
def contribute_to_class(self, cls, name):
signals.post_save.connect(self.post_save, sender=cls)
signals.post_delete.connect(self.post_delete, sender=cls)
return super(CachingManager, self).contribute_to_class(cls, name)
def post_save(self, instance, **kwargs):
def post_delete(self, instance, **kwargs):
def invalidate(self, *objects):
"""Invalidate all the flush lists associated with ``objects``."""
keys = [k for o in objects for k in o._cache_keys()]
def raw(self, raw_query, params=None, *args, **kwargs):
return CachingRawQuerySet(raw_query, self.model, params=params,
using=self._db, *args, **kwargs)
def cache(self, timeout=None):
return self.get_query_set().cache(timeout)
def no_cache(self):
return self.cache(NO_CACHE)
class CacheMachine(object):
Handles all the cache management for a QuerySet.
Takes the string representation of a query and a function that can be
called to get an iterator over some database results.
def __init__(self, query_string, iter_function, timeout=None):
self.query_string = query_string
self.iter_function = iter_function
self.timeout = timeout
def query_key(self):
"""Generate the cache key for this query."""
return make_key('qs:%s' % self.query_string, with_locale=False)
def __iter__(self):
query_key = self.query_key()
except query.EmptyResultSet:
raise StopIteration
# Try to fetch from the cache.
cached = cache.get(query_key)
if cached is not None:
log.debug('cache hit: %s' % self.query_string)
for obj in cached:
obj.from_cache = True
yield obj
# Do the database query, cache it once we have all the objects.
iterator = self.iter_function()
to_cache = []
while True:
obj =
obj.from_cache = False
yield obj
except StopIteration:
if to_cache:
def cache_objects(self, objects):
"""Cache query_key => objects, then update the flush lists."""
query_key = self.query_key()
query_flush = flush_key(self.query_string)
cache.add(query_key, objects, timeout=self.timeout)
invalidator.cache_objects(objects, query_key, query_flush)
class CachingQuerySet(models.query.QuerySet):
def __init__(self, *args, **kw):
super(CachingQuerySet, self).__init__(*args, **kw)
self.timeout = None
def flush_key(self):
return flush_key(self.query_key())
def query_key(self):
sql, params = self.query.get_compiler(using=self.db).as_sql()
return sql % params
def iterator(self):
iterator = super(CachingQuerySet, self).iterator
if self.timeout == NO_CACHE:
return iter(iterator())
# Work-around for Django #12717.
query_string = self.query_key()
except query.EmptyResultSet:
return iterator()
iterator = self.fetch_by_id
return iter(CacheMachine(query_string, iterator, self.timeout))
def fetch_by_id(self):
Run two queries to get objects: one for the ids, one for id__in=ids.
After getting ids from the first query we can try cache.get_many to
reuse objects we've already seen. Then we fetch the remaining items
from the db, and put those in the cache. This prevents cache
# Include columns from extra since they could be used in the query's
# order_by.
vals = self.values_list('pk', *self.query.extra.keys())
pks = [val[0] for val in vals]
keys = dict((byid(self.model._cache_key(pk)), pk) for pk in pks)
cached = dict((k, v) for k, v in cache.get_many(keys).items()
if v is not None)
# Pick up the objects we missed.
missed = [pk for key, pk in keys.items() if key not in cached]
if missed:
others = self.fetch_missed(missed)
# Put the fetched objects back in cache.
new = dict((byid(o), o) for o in others)
new = {}
# Use pks to return the objects in the correct order.
objects = dict((, o) for o in cached.values() + new.values())
for pk in pks:
yield objects[pk]
def fetch_missed(self, pks):
# Reuse the queryset but get a clean query.
others = self.all()
# Clear out the default ordering since we order based on the query.
others = others.order_by().filter(pk__in=pks)
if hasattr(others, 'no_cache'):
others = others.no_cache()
if self.query.select_related:
return others
def count(self):
timeout = getattr(settings, 'CACHE_COUNT_TIMEOUT', None)
super_count = super(CachingQuerySet, self).count
query_string = 'count:%s' % self.query_key()
if self.timeout == NO_CACHE or timeout is None:
return super_count()
return cached_with(self, super_count, query_string, timeout)
def cache(self, timeout=None):
qs = self._clone()
qs.timeout = timeout
return qs
def no_cache(self):
return self.cache(NO_CACHE)
def _clone(self, *args, **kw):
qs = super(CachingQuerySet, self)._clone(*args, **kw)
qs.timeout = self.timeout
return qs
class CachingMixin:
"""Inherit from this class to get caching and invalidation helpers."""
def flush_key(self):
return flush_key(self)
def cache_key(self):
"""Return a cache key based on the object's primary key."""
return self._cache_key(
def _cache_key(cls, pk):
Return a string that uniquely identifies the object.
For the Addon class, with a pk of 2, we get "o:addons.addon:2".
key_parts = ('o', cls._meta, pk)
return ':'.join(map(encoding.smart_unicode, key_parts))
def _cache_keys(self):
"""Return the cache key for self plus all related foreign keys."""
fks = dict((f, getattr(self, f.attname)) for f in self._meta.fields
if isinstance(f, models.ForeignKey))
keys = [ for fk, val in fks.items()
if val is not None and hasattr(, '_cache_key')]
return (self.cache_key,) + tuple(keys)
class CachingRawQuerySet(models.query.RawQuerySet):
def __iter__(self):
iterator = super(CachingRawQuerySet, self).__iter__
sql = self.raw_query % tuple(self.params)
for obj in CacheMachine(sql, iterator):
yield obj
raise StopIteration
def _function_cache_key(key):
return make_key('f:%s' % key, with_locale=True)
def cached(function, key_, duration=None):
"""Only calls the function if ``key`` is not already in the cache."""
key = _function_cache_key(key_)
val = cache.get(key)
if val is None:
log.debug('cache miss for %s' % key)
val = function()
cache.set(key, val, duration)
log.debug('cache hit for %s' % key)
return val
def cached_with(obj, f, f_key, timeout=None):
"""Helper for caching a function call within an object's flush list."""
obj_key = (obj.query_key() if hasattr(obj, 'query_key')
else obj.cache_key)
except AttributeError:
log.warning(u'%r cannot be cached.' % encoding.smart_str(obj))
return f()
key = '%s:%s' % tuple(map(encoding.smart_str, (f_key, obj_key)))
# Put the key generated in cached() into this object's flush list.
{obj.flush_key(): [_function_cache_key(key)]})
return cached(f, key, timeout)
class cached_method(object):
Decorator to cache a method call in this object's flush list.
The external cache will only be used once per (instance, args). After that
a local cache on the object will be used.
Lifted from werkzeug.
def __init__(self, func):
self.func = func
functools.update_wrapper(self, func)
def __get__(self, obj, type=None):
if obj is None:
return self
_missing = object()
value = obj.__dict__.get(self.__name__, _missing)
if value is _missing:
w = MethodWrapper(obj, self.func)
obj.__dict__[self.__name__] = w
return w
return value
class MethodWrapper(object):
Wraps around an object's method for two-level caching.
The first call for a set of (args, kwargs) will use an external cache.
After that, an object-local dict cache will be used.
def __init__(self, obj, func):
self.obj = obj
self.func = func
functools.update_wrapper(self, func)
self.cache = {}
def __call__(self, *args, **kwargs):
k = lambda o: o.cache_key if hasattr(o, 'cache_key') else o
arg_keys = map(k, args)
kwarg_keys = [(key, k(val)) for key, val in kwargs.items()]
key = 'm:%s:%s:%s:%s' % (self.obj.cache_key, self.func.__name__,
arg_keys, kwarg_keys)
if key not in self.cache:
f = functools.partial(self.func, self.obj, *args, **kwargs)
self.cache[key] = cached_with(self.obj, f, key)
return self.cache[key]