version 0.3 and ability to record all cached URLs

README.md

@@ -30,8 +30,8 @@ In your Django views:
     from fancy_cache import cache_page
 
     @cache_page(60 * 60)
-        def myview(request):
-            return render(request, 'page1.html')
+    def myview(request):
+        return render(request, 'page1.html')
 
     def prefixer(request):
         if request.method != 'GET':
@@ -55,6 +55,100 @@ In your Django views:
         return render(request, 'page3.html')
 
 
+Optional uses
+-------------
+
+If you want to you can have `django-fancy-cache` record every URL it
+caches. This can be useful for things like invalidation or curious
+statistical inspection.
+
+You can either switch this on on the decorator itself. Like this:
+
+
+    from fancy_cache import cache_page
+
+    @cache_page(60 * 60, remember_all_urls=True)
+    def myview(request):
+        return render(request, 'page1.html')
+
+Or, more conveniently to apply it to all uses of the `cache_page`
+decorator you can set the default in your settings with:
+
+    FANCY_REMEMBER_ALL_URLS = True
+
+Now, suppose you have the this option enabled. Now you can do things
+like this:
+
+    >>> from fancy_cache.memory import find_urls
+    >>> list(find_urls(['/some/searchpath', '/or/like/*/this.*']))
+    >>> # or, to get all:
+    >>> list(find_urls([]))
+
+There is also another option to this and that is to purge (aka.
+invalidate) the remembered URLs. You simply all the `purge=True`
+option like this:
+
+    >>> from fancy_cache.memory import find_urls
+    >>> list(find_urls([], purge=True))
+
+Note: Since `find_urls()` returns a generator, the purging won't
+happen unless you exhaust the generator. E.g. looping over it or
+turning it into a list.
+
+The second way to inspect all recorded URLs is to use the
+`fancy-cache` management command. This is only available if you have
+added `fancy_cache` to your `INSTALLED_APPS` setting. Now you can do
+this:
+
+    $ ./manage.py fancy-cache --help
+    $ ./manage.py fancy-cache
+    $ ./manage.py fancy-cache /some/searchpath /or/like/*/this.*
+    $ ./manage.py fancy-cache /some/place/* --purge
+    $ # or to purge them all!
+    $ ./manage.py fancy-cache --purge
+
+Note, it will only print out URLs that if found (and purged, if
+applicable).
+
+The third way to inspect the recorded URLs is to add this to your root
+`urls.py`:
+
+    url(r'fancy-cache', include('fancy_cache.urls')),
+
+Now, if you visit `http://localhost:8000/fancy-cache` you get a table
+listing every URL that `django-fancy-cache` has recorded.
+
+
+Optional uses (for the exceptionally curious)
+---------------------------------------------
+
+If you have enabled `FANCY_REMEMBER_ALL_URLS` you can also enable
+`FANCY_REMEMBER_STATS_ALL_URLS` in your settings. What this does is
+that it attempts to count the number of cache hits and cache misses
+you have for each URL.
+
+This counting of hits and misses is configured to last "a long time".
+Possibly longer than you cache your view. So, over time you can expect
+to have more than one miss because your view cache expires and it
+starts over.
+
+You can see the stats whenever you use any of the ways described in
+the section above. For example like this:
+
+    >>> from fancy_cache.memory import find_urls
+    >>> found = list(find_urls([]))[0]
+    >>> found[0]
+    '/some/page.html'
+    >>> found[2]
+    {'hits': 1235, 'misses': 12}
+
+There is obviously a small additional performance cost of using the
+`FANCY_REMEMBER_ALL_URLS` and/or `FANCY_REMEMBER_STATS_ALL_URLS` in
+your project so only use it if you don't have any smarter way to
+invalidate, for debugging or if you really want make it possible to
+purge all cached responses when you run an upgrade of your site or
+something.
+
 Running the test suite
 ----------------------
 

example/example/app/templates/home.html

@@ -19,5 +19,11 @@
       <li><a href="{% url 'page4' %}">Page 4</a></li>
       <li><a href="{% url 'page5' %}">Page 5</a></li>
     </ul>
+    <hr>
+    <p>This is just for the uber-curious about what's going on. These are the current
+    remembered URLs:</p>
+    <pre>
+{{ remembered_urls }}
+    </pre>
     </body>
 </html>

example/example/app/views.py

@@ -1,10 +1,24 @@
 import time
+from cStringIO import StringIO
+from pprint import pprint
+
 from django.shortcuts import render, redirect
+from django.core.cache import cache
+
 from fancy_cache import cache_page
+from fancy_cache.middleware import REMEMBERED_URLS_KEY
 
 
 def home(request):
-    return render(request, 'home.html')
+    remembered_urls = cache.get(REMEMBERED_URLS_KEY, {})
+    out = StringIO()
+    pprint(remembered_urls, out)
+    remembered_urls = out.getvalue()
+    return render(
+        request,
+        'home.html',
+        {'remembered_urls': remembered_urls}
+    )
 
 
 def commafy(s):
@@ -16,7 +30,7 @@ def commafy(s):
     return ''.join(r)
 
 
-@cache_page(10)
+@cache_page(60)
 def page1(request):
     print "CACHE MISS", request.build_absolute_uri()
     t0 = time.time()
@@ -35,7 +49,7 @@ def key_prefixer(request):
     return request.GET.get('number')
 
 
-@cache_page(10, key_prefix=key_prefixer)
+@cache_page(60, key_prefix=key_prefixer)
 def page2(request):
     if not request.GET.get('number'):
         return redirect(request.build_absolute_uri() + '?number=25000000')
@@ -59,7 +73,7 @@ def post_processor(response, request):
     return response
 
 
-@cache_page(10, post_process_response=post_processor)
+@cache_page(60, post_process_response=post_processor)
 def page3(request):
     print "CACHE MISS", request.build_absolute_uri()
     t0 = time.time()
@@ -84,7 +98,7 @@ def post_processor_always(response, request):
     return response
 
 
-@cache_page(10, post_process_response_always=post_processor_always)
+@cache_page(60, post_process_response_always=post_processor_always)
 def page4(request):
     print "CACHE MISS", request.build_absolute_uri()
     t0 = time.time()
@@ -98,7 +112,7 @@ def page4(request):
     )
 
 
-@cache_page(20, only_get_keys=['foo', 'bar'])
+@cache_page(60, only_get_keys=['foo', 'bar'])
 def page5(request):
     print "CACHE MISS", request.build_absolute_uri()
     t0 = time.time()

example/example/settings.py

@@ -1,5 +1,8 @@
 # Django settings for example project.
 
+FANCY_REMEMBER_ALL_URLS = True
+FANCY_REMEMBER_STATS_ALL_URLS = True
+
 DEBUG = True
 TEMPLATE_DEBUG = DEBUG
 
@@ -11,16 +14,28 @@
 
 DATABASES = {
     'default': {
-        'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
-        'NAME': 'database.db',                      # Or path to database file if using sqlite3.
-        # The following settings are not used with sqlite3:
-        'USER': '',
-        'PASSWORD': '',
-        'HOST': '',                      # Empty for localhost through domain sockets or '127.0.0.1' for localhost through TCP.
-        'PORT': '',                      # Set to empty string for default.
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': 'database.db',
+    }
+}
+
+CACHES = {
+    'default': {
+        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
+        'LOCATION': 'unique-snowflake'
     }
 }
 
+CACHES = {
+    'default': {
+        'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
+        'LOCATION': 'localhost:11211',
+        'TIMEOUT': 500,
+        'KEY_PREFIX': 'example',
+    }
+}
+
+
 # Local time zone for this installation. Choices can be found here:
 # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
 # although not all choices may be available on all operating systems.

example/example/urls.py

@@ -4,4 +4,5 @@
 urlpatterns = patterns(
     '',
     url(r'', include('example.app.urls')),
+    url(r'fancy-cache', include('fancy_cache.urls')),
 )

fabfile.py

@@ -21,4 +21,4 @@ def test():
     local('django-admin.py syncdb --noinput')
     local('django-admin.py flush --noinput')
 
-    local('django-admin.py test')
+    local('django-admin.py test -s')

fancy_cache/__init__.py

@@ -1,2 +1,2 @@
-__version__ = '0.2'
+__version__ = '0.3'
 from cache_page import cache_page

fancy_cache/management/commands/fancy-urls.py

@@ -0,0 +1,57 @@
+__doc__ = """
+If you enable `FANCY_REMEMBER_ALL_URLS` then every URL take is turned
+into a cache key for cache_page() to remember is recorded.
+You can use this to do statistics or to do invalidation by URL.
+
+To use: simply add the URL patterns after like this::
+
+    $ ./manage.py %(this_file)s /path1.html /path3/*/*.json
+
+To show all cached URLs simply run it with no pattern like this::
+
+    $ ./manage.py %(this_file)s
+
+Equally the ``--purge`` switch can always be added. For example,
+running this will purge all cached URLs::
+
+    $ ./manage.py %(this_file)s --purge
+
+If you enable `FANCY_REMEMBER_STATS_ALL_URLS` you can get a tally for each
+URL how many cache HITS and MISSES it has had.
+
+""" % dict(this_file=__file__)
+
+from optparse import make_option
+
+from django.core.management.base import BaseCommand
+
+from fancy_cache.memory import find_urls
+
+
+class Command(BaseCommand):
+    help = __doc__.strip()
+
+    option_list = BaseCommand.option_list + (
+        make_option(
+            '-p', '--purge', dest='purge', action='store_true',
+            help='Purge found URLs'
+        ),
+    )
+    args = 'urls'
+
+    def handle(self, *urls, **options):
+        verbose = int(options['verbosity']) > 1
+
+        _count = 0
+        for url, cache_key, stats in find_urls(urls, purge=options['purge']):
+            _count += 1
+            if stats:
+                print url[:70].ljust(65),
+                print "HITS", str(stats['hits']).ljust(5),
+                print "MISSES", str(stats['misses']).ljust(5)
+
+            else:
+                print url
+
+        if verbose:
+            print "-- %s URLs cached --" % _count

fancy_cache/memory.py

@@ -0,0 +1,70 @@
+import re
+
+from django.core.cache import cache
+
+from fancy_cache.middleware import REMEMBERED_URLS_KEY, LONG_TIME
+
+__all__ = ('find_urls',)
+
+
+def _match(url, regexes):
+    if not regexes:
+        return url
+    for regex in regexes:
+        if regex.match(url):
+            return True
+    return False
+
+
+def _urls_to_regexes(urls):
+    regexes = []
+    for each in urls:
+        parts = each.split('*')
+        if len(parts) == 1:
+            regexes.append(re.compile(re.escape(parts[0])))
+        else:
+            _re = '.*'.join(re.escape(x) for x in parts)
+            regexes.append(re.compile(_re))
+    return regexes
+
+
+def find_urls(urls, purge=False):
+    remembered_urls = cache.get(REMEMBERED_URLS_KEY, {})
+    _del_keys = []
+    regexes = _urls_to_regexes(urls)
+
+    for url in remembered_urls:
+        if _match(url, regexes):
+            cache_key = remembered_urls[url]
+            if not cache.get(cache_key):
+                continue
+            if purge:
+                cache.delete(cache_key)
+                _del_keys.append(url)
+            misses_cache_key = '%s__misses' % url
+            hits_cache_key = '%s__hits' % url
+            misses = cache.get(misses_cache_key)
+            hits = cache.get(hits_cache_key)
+            if misses is None and hits is None:
+                stats = None
+            else:
+                stats = {
+                    'hits': hits or 0,
+                    'misses': misses or 0
+                }
+            yield (url, cache_key, stats)
+
+    if _del_keys:
+        # means something was changed
+        for url in _del_keys:
+            remembered_urls.pop(url)
+            misses_cache_key = '%s__misses' % url
+            hits_cache_key = '%s__hits' % url
+            cache.delete(misses_cache_key)
+            cache.delete(hits_cache_key)
+
+        cache.set(
+            REMEMBERED_URLS_KEY,
+            remembered_urls,
+            LONG_TIME
+        )