Add support for invalidation after adding an object

caching/__init__.py

@@ -1,4 +1,4 @@
 from __future__ import unicode_literals
 
-VERSION = ('0', '9', '0', 'dev1')
+VERSION = ('0', '9')
 __version__ = '.'.join(VERSION)

caching/base.py

@@ -4,12 +4,12 @@
 import logging
 
 import django
-from django.conf import settings
 from django.db import models
 from django.db.models import signals
 from django.db.models.sql import query, EmptyResultSet
 from django.utils import encoding
 
+from caching import config
 from .compat import DEFAULT_TIMEOUT
 from .invalidation import invalidator, flush_key, make_key, byid, cache
 
@@ -23,12 +23,6 @@ def emit(self, record):
 log = logging.getLogger('caching')
 log.addHandler(NullHandler())
 
-NO_CACHE = -1
-CACHE_PREFIX = getattr(settings, 'CACHE_PREFIX', '')
-FETCH_BY_ID = getattr(settings, 'FETCH_BY_ID', False)
-CACHE_EMPTY_QUERYSETS = getattr(settings, 'CACHE_EMPTY_QUERYSETS', False)
-TIMEOUT = getattr(settings, 'CACHE_COUNT_TIMEOUT', NO_CACHE)
-
 
 class CachingManager(models.Manager):
 
@@ -47,14 +41,23 @@ def contribute_to_class(self, cls, name):
         return super(CachingManager, self).contribute_to_class(cls, name)
 
     def post_save(self, instance, **kwargs):
-        self.invalidate(instance)
+        self.invalidate(instance, is_new_instance=kwargs['created'],
+                        model_cls=kwargs['sender'])
 
     def post_delete(self, instance, **kwargs):
         self.invalidate(instance)
 
-    def invalidate(self, *objects):
+    def invalidate(self, *objects, **kwargs):
         """Invalidate all the flush lists associated with ``objects``."""
         keys = [k for o in objects for k in o._cache_keys()]
+        # If whole-model invalidation on create is enabled, include this model's
+        # key in the list to be invalidated. Note that the key itself won't
+        # contain anything in the cache, but its corresponding flush key will.
+        is_new_instance = kwargs.pop('is_new_instance', False)
+        model_cls = kwargs.pop('model_cls', None)
+        if (config.CACHE_INVALIDATE_ON_CREATE == config.WHOLE_MODEL and
+           is_new_instance and model_cls and hasattr(model_cls, 'model_key')):
+            keys.append(model_cls.model_key())
         invalidator.invalidate_keys(keys)
 
     def raw(self, raw_query, params=None, *args, **kwargs):
@@ -65,7 +68,7 @@ def cache(self, timeout=DEFAULT_TIMEOUT):
         return self.get_queryset().cache(timeout)
 
     def no_cache(self):
-        return self.cache(NO_CACHE)
+        return self.cache(config.NO_CACHE)
 
 
 class CacheMachine(object):
@@ -76,7 +79,8 @@ class CacheMachine(object):
     called to get an iterator over some database results.
     """
 
-    def __init__(self, query_string, iter_function, timeout=DEFAULT_TIMEOUT, db='default'):
+    def __init__(self, model, query_string, iter_function, timeout=DEFAULT_TIMEOUT, db='default'):
+        self.model = model
         self.query_string = query_string
         self.iter_function = iter_function
         self.timeout = timeout
@@ -120,7 +124,7 @@ def __iter__(self):
                 to_cache.append(obj)
                 yield obj
         except StopIteration:
-            if to_cache or CACHE_EMPTY_QUERYSETS:
+            if to_cache or config.CACHE_EMPTY_QUERYSETS:
                 self.cache_objects(to_cache)
             raise
 
@@ -129,7 +133,7 @@ def cache_objects(self, objects):
         query_key = self.query_key()
         query_flush = flush_key(self.query_string)
         cache.add(query_key, objects, timeout=self.timeout)
-        invalidator.cache_objects(objects, query_key, query_flush)
+        invalidator.cache_objects(self.model, objects, query_key, query_flush)
 
 
 class CachingQuerySet(models.query.QuerySet):
@@ -148,17 +152,17 @@ def query_key(self):
 
     def iterator(self):
         iterator = super(CachingQuerySet, self).iterator
-        if self.timeout == NO_CACHE:
+        if self.timeout == config.NO_CACHE:
             return iter(iterator())
         else:
             try:
                 # Work-around for Django #12717.
                 query_string = self.query_key()
             except query.EmptyResultSet:
                 return iterator()
-            if FETCH_BY_ID:
+            if config.FETCH_BY_ID:
                 iterator = self.fetch_by_id
-            return iter(CacheMachine(query_string, iterator, self.timeout, db=self.db))
+            return iter(CacheMachine(self.model, query_string, iterator, self.timeout, db=self.db))
 
     def fetch_by_id(self):
         """
@@ -210,18 +214,18 @@ def count(self):
             query_string = 'count:%s' % self.query_key()
         except query.EmptyResultSet:
             return 0
-        if self.timeout == NO_CACHE or TIMEOUT == NO_CACHE:
+        if self.timeout == config.NO_CACHE or config.TIMEOUT == config.NO_CACHE:
             return super_count()
         else:
-            return cached_with(self, super_count, query_string, TIMEOUT)
+            return cached_with(self, super_count, query_string, config.TIMEOUT)
 
     def cache(self, timeout=DEFAULT_TIMEOUT):
         qs = self._clone()
         qs.timeout = timeout
         return qs
 
     def no_cache(self):
-        return self.cache(NO_CACHE)
+        return self.cache(config.NO_CACHE)
 
     def _clone(self, *args, **kw):
         qs = super(CachingQuerySet, self)._clone(*args, **kw)
@@ -240,6 +244,15 @@ def cache_key(self):
         """Return a cache key based on the object's primary key."""
         return self._cache_key(self.pk, self._state.db)
 
+    @classmethod
+    def model_key(cls):
+        """
+        Return a cache key for the entire model (used by invalidation).
+        """
+        # use dummy PK and DB reference that will never resolve to an actual
+        # cache key for an objection
+        return cls._cache_key('all-pks', 'all-dbs')
+
     @classmethod
     def _cache_key(cls, pk, db):
         """
@@ -269,13 +282,13 @@ def __init__(self, *args, **kw):
 
     def __iter__(self):
         iterator = super(CachingRawQuerySet, self).__iter__
-        if self.timeout == NO_CACHE:
+        if self.timeout == config.NO_CACHE:
             iterator = iterator()
             while True:
                 yield next(iterator)
         else:
             sql = self.raw_query % tuple(self.params)
-            for obj in CacheMachine(sql, iterator, timeout=self.timeout):
+            for obj in CacheMachine(self.model, sql, iterator, timeout=self.timeout):
                 yield obj
             raise StopIteration
 

caching/config.py

@@ -0,0 +1,18 @@
+from django.conf import settings
+
+NO_CACHE = -1
+WHOLE_MODEL = 'whole-model'
+
+CACHE_PREFIX = getattr(settings, 'CACHE_PREFIX', '')
+FETCH_BY_ID = getattr(settings, 'FETCH_BY_ID', False)
+FLUSH = CACHE_PREFIX + ':flush:'
+CACHE_EMPTY_QUERYSETS = getattr(settings, 'CACHE_EMPTY_QUERYSETS', False)
+TIMEOUT = getattr(settings, 'CACHE_COUNT_TIMEOUT', NO_CACHE)
+CACHE_INVALIDATE_ON_CREATE = getattr(settings, 'CACHE_INVALIDATE_ON_CREATE', None)
+CACHE_MACHINE_NO_INVALIDATION = getattr(settings, 'CACHE_MACHINE_NO_INVALIDATION', False)
+CACHE_MACHINE_USE_REDIS = getattr(settings, 'CACHE_MACHINE_USE_REDIS', False)
+
+_invalidate_on_create_values = (None, WHOLE_MODEL)
+if CACHE_INVALIDATE_ON_CREATE not in _invalidate_on_create_values:
+    raise ValueError('CACHE_INVALIDATE_ON_CREATE must be one of: '
+                     '%s' % _invalidate_on_create_values)

caching/invalidation.py

@@ -29,17 +29,14 @@
 except (InvalidCacheBackendError, ValueError):
     cache = default_cache
 
-
-CACHE_PREFIX = getattr(settings, 'CACHE_PREFIX', '')
-FETCH_BY_ID = getattr(settings, 'FETCH_BY_ID', False)
-FLUSH = CACHE_PREFIX + ':flush:'
+from caching import config
 
 log = logging.getLogger('caching.invalidation')
 
 
 def make_key(k, with_locale=True):
     """Generate the full key for ``k``, with a prefix."""
-    key = encoding.smart_bytes('%s:%s' % (CACHE_PREFIX, k))
+    key = encoding.smart_bytes('%s:%s' % (config.CACHE_PREFIX, k))
     if with_locale:
         key += encoding.smart_bytes(translation.get_language())
     # memcached keys must be < 250 bytes and w/o whitespace, but it's nice
@@ -50,7 +47,7 @@ def make_key(k, with_locale=True):
 def flush_key(obj):
     """We put flush lists in the flush: namespace."""
     key = obj if isinstance(obj, six.string_types) else obj.cache_key
-    return FLUSH + make_key(key, with_locale=False)
+    return config.FLUSH + make_key(key, with_locale=False)
 
 
 def byid(obj):
@@ -86,31 +83,37 @@ def invalidate_keys(self, keys):
         """Invalidate all the flush lists named by the list of ``keys``."""
         if not keys:
             return
-        flush, flush_keys = self.find_flush_lists(keys)
-
-        if flush:
-            cache.delete_many(flush)
+        obj_keys, flush_keys = self.find_flush_lists(keys)
+        if obj_keys:
+            log.debug('obj_keys: %s' % obj_keys)
+            cache.delete_many(obj_keys)
         if flush_keys:
+            log.debug('flush_keys: %s' % flush_keys)
             self.clear_flush_lists(flush_keys)
 
-    def cache_objects(self, objects, query_key, query_flush):
+    def cache_objects(self, model, objects, query_key, query_flush):
         # Add this query to the flush list of each object.  We include
         # query_flush so that other things can be cached against the queryset
         # and still participate in invalidation.
         flush_keys = [o.flush_key() for o in objects]
 
         flush_lists = collections.defaultdict(set)
         for key in flush_keys:
+            log.debug('adding %s to %s' % (query_flush, key))
             flush_lists[key].add(query_flush)
         flush_lists[query_flush].add(query_key)
-
+        # Add this query to the flush key for the entire model, if enabled
+        model_flush = flush_key(model.model_key())
+        if config.CACHE_INVALIDATE_ON_CREATE == config.WHOLE_MODEL:
+            flush_lists[model_flush].add(query_key)
         # Add each object to the flush lists of its foreign keys.
         for obj in objects:
             obj_flush = obj.flush_key()
             for key in map(flush_key, obj._cache_keys()):
-                if key != obj_flush:
+                if key not in (obj_flush, model_flush):
+                    log.debug('related: adding %s to %s' % (obj_flush, key))
                     flush_lists[key].add(obj_flush)
-                if FETCH_BY_ID:
+                if config.FETCH_BY_ID:
                     flush_lists[key].add(byid(obj))
         self.add_to_flush_list(flush_lists)
 
@@ -121,20 +124,24 @@ def find_flush_lists(self, keys):
         The search starts with the lists in `keys` and expands to any flush
         lists found therein.  Returns ({objects to flush}, {flush keys found}).
         """
-        new_keys = keys = set(map(flush_key, keys))
-        flush = set(keys)
+        objs = set(keys)
+        search_keys = keys = set(map(flush_key, keys))
 
         # Add other flush keys from the lists, which happens when a parent
         # object includes a foreign key.
         while 1:
-            to_flush = self.get_flush_lists(new_keys)
-            flush.update(to_flush)
-            new_keys = set(k for k in to_flush if k.startswith(FLUSH))
-            diff = new_keys.difference(keys)
-            if diff:
+            new_keys = set()
+            for key in self.get_flush_lists(search_keys):
+                if key.startswith(config.FLUSH):
+                    new_keys.add(key)
+                else:
+                    objs.add(key)
+            if new_keys:
+                log.debug('search for %s found keys %s' % (search_keys, new_keys))
                 keys.update(new_keys)
+                search_keys = new_keys
             else:
-                return flush, keys
+                return objs, keys
 
     def add_to_flush_list(self, mapping):
         """Update flush lists with the {flush_key: [query_key,...]} map."""
@@ -247,9 +254,9 @@ def get_redis_backend():
                           socket_timeout=socket_timeout)
 
 
-if getattr(settings, 'CACHE_MACHINE_NO_INVALIDATION', False):
+if config.CACHE_MACHINE_NO_INVALIDATION:
     invalidator = NullInvalidator()
-elif getattr(settings, 'CACHE_MACHINE_USE_REDIS', False):
+elif config.CACHE_MACHINE_USE_REDIS:
     redis = get_redis_backend()
     invalidator = RedisInvalidator()
 else:

docs/index.rst

@@ -90,6 +90,21 @@ By default cache machine will not cache empty querysets. To cache them::
 
     CACHE_EMPTY_QUERYSETS = True
 
+.. _object-creation:
+
+Object creation
+^^^^^^^^^^^^^^^
+
+By default Cache Machine does not invalidate queries when a new object is
+created, because it can be expensive to maintain a flush list of all the
+queries associated with a given table and cause significant disruption on
+high-volume sites when *all* the queries for a particular model are
+invalidated at once. If these are not issues for your site and immediate
+inclusion of created objects in previously cached queries is desired, you
+can enable this feature as follows::
+
+    CACHE_INVALIDATE_ON_CREATE = 'whole-model'
+
 Cache Manager
 -------------
 

docs/releases.rst

@@ -3,10 +3,13 @@
 Release Notes
 ==================
 
-v0.9 (release date TBD)
------------------------
+v0.9 (2015-07-29)
+-----------------
 
 - Support for Python 3
+- A new setting, ``CACHE_INVALIDATE_ON_CREATE``, which facilitates invalidation
+  when a new model object is created. For more information, see
+  :ref:`object-creation`.
 
 v0.8.1 (2015-07-03)
 -----------------------