Permalink
Browse files

more testing and comments

  • Loading branch information...
SeanHayes committed Dec 31, 2011
1 parent a49313e commit d2b093dfc855939ec936c5be51cfee56515ab215
View
5 README
@@ -21,3 +21,8 @@ http://www.opensource.org/licenses/bsd-license.php
https://github.com/SeanHayes/django-query-caching
http://pypi.python.org/pypi/django-query-caching
http://djangopackages.com/packages/p/django-query-caching/
+
+== Alternatives ==
+http://packages.python.org/johnny-cache/
+https://github.com/jbalogh/django-cache-machine
+
@@ -16,7 +16,7 @@
logger = logging.getLogger(__name__)
-VERSION = (0, 1, 2)
+VERSION = (0, 2, 0, 'dev')
__version__ = "".join([".".join(map(str, VERSION[0:3])), "".join(VERSION[3:])])
@@ -70,7 +70,7 @@ def get_query_key(compiler):
#NOTE: keys must be 250 characters or fewer
sql, params = compiler.as_nested_sql()
#profiling shows that using string.replace and regex to shorten sql (removing [ `.]) adds more
- #time than is saved during sha256-ing. [6:] has some slight benefit though.
+ #time than is saved by hashing the full string via sha256. [6:] has some slight benefit though.
key = '%s:%s' % (compiler.using, sql[6:])
#logger.debug(key)
#logger.debug(params)
@@ -82,7 +82,7 @@ def get_query_key(compiler):
def get_table_keys(query):
"Returns a set of cache keys based on table names. These keys are used to store timestamps of when the last time said tables were last updated."
logger.debug('get_table_keys()')
- table_keys = set([])
+ table_keys = set()
tables = query.tables
#OPTIMIZE: if not tables:
if len(tables) == 0:
@@ -101,7 +101,7 @@ def get_current_timestamp():
#FIXME: before doing INSERT, UPDATE, or DELETE, Django first does a SELECT to see if the rows exists, which means the cache will needlessly be updated right before the updated parts are invalidated. Any way around this? Maybe these SELECT queries have some sort of flag we can use to ignore them.
def try_cache(self, result_type=MULTI):
- "This function overwrites the default behavior of SQLCompiler.execute_sql(), attempting to retreive data from the cache first before trying the database."
+ "This function patches the default behavior of SQLCompiler.execute_sql(), attempting to retrieve data from the cache first before trying the database."
#logger.debug('try_cache()')
#logger.debug(self)
#logger.debug('Result type: %s' % result_type)
@@ -198,7 +198,6 @@ def try_cache(self, result_type=MULTI):
#if the query result was obtained from the cache, then it's actually a tuple of the form (timestamp, query_result)
ret = ret[1]
- return ret
#INSERT, UPDATE, DELETE statements (and SELECT with caching disabled)
else:
#perform operation
@@ -222,12 +221,12 @@ def try_cache(self, result_type=MULTI):
#update key lists
#TODO: only do this for tables not in EXCLUDE_TABLES and EXCLUDE_MODELS
#FIXME: only update if existing timestamp is older than new one (help avoid race conditions)
- table_key_map = dict((key, now) for key in get_table_keys(self.query))
+ table_key_map = dict((key, now,) for key in get_table_keys(self.query))
logger.debug(table_key_map)
cache.set_many(table_key_map, timeout=TIMEOUT)
#pdb.set_trace()
-
- return ret
+ logger.debug('------------------------------------------------------------')
+ return ret
if not patched:
logger.debug('Patching')
@@ -1,14 +0,0 @@
-#!/usr/bin/env python
-
-#NOTE: you will have to checkout Django from SVN and symlink /django/trunk/tests/* here
-#ln -s path/to/django/trunk/tests/*
-import runtests
-
-
-
-#ControlDjangoTestCase
-
-
-#ExpDjangoTestCase
-
-#TODO: compare execution times of tests both with and without this project enabled
No changes.
@@ -2,6 +2,22 @@
# Create your models here.
-class TestModel(models.Model):
+class TestModelBase(models.Model):
+ name = models.CharField(max_length=25)
added = models.DateTimeField(auto_now_add=True)
last_edited = models.DateTimeField(auto_now=True)
+
+ def __unicode__(self):
+ return u'%s: %s (Created on %s, Edited on %s)' % (self.__class__, self.name, self.added, self.last_edited,)
+
+ class Meta:
+ abstract = True
+
+class TestModelA(TestModelBase):
+ pass
+
+class TestModelB(TestModelBase):
+ related = models.ForeignKey(TestModelA)
+
+class TestModelC(TestModelBase):
+ related = models.ManyToManyField(TestModelA)
@@ -1,23 +1,139 @@
+from datetime import datetime
import logging
import pdb
from django import db
+from django.core.cache import cache
from django.test import TestCase
-from models import *
+from models import TestModelA, TestModelB, TestModelC
logger = logging.getLogger(__name__)
class SimpleTest(TestCase):
- def test_select_caching(self):
- "There should be one DB query to get the data, then any futerther attempts to get the same data should bypass the DB."
+ "There should be one DB query to get the data, then any further attempts to get the same data should bypass the DB, unless invalidated."
+
+ def setUp(self):
+ with self.assertNumQueries(2):
+ self.ta1 = TestModelA(name="foo")
+ self.ta1.save()
+ self.ta2 = TestModelA(name="bar")
+ self.ta2.save()
+
+ def test_basic_select_caching(self):
+ "Test that a whole queryset is retrieved from the cache the second time it's needed."
+ with self.assertNumQueries(1):
+ tas1 = TestModelA.objects.all()
+ self.assertEqual(len(tas1), 2)
+
+ with self.assertNumQueries(0):
+ tas2 = TestModelA.objects.all()
+ self.assertEqual(len(tas2), 2)
+
+ self.assertEqual(len(tas1), len(tas2))
+ for i in xrange(0, len(tas1)):
+ self.assertEqual(tas1[i], tas2[i])
+
+ def test_caches_with_filter(self):
+ "Test that a filtered queryset is retrieved from the cache the second time it's needed."
+ with self.assertNumQueries(1):
+ tas1 = TestModelA.objects.filter(name__contains='f')
+ self.assertEqual(len(tas1), 1)
+
+ with self.assertNumQueries(0):
+ tas2 = TestModelA.objects.filter(name__contains='f')
+ self.assertEqual(len(tas2), 1)
+
+ self.assertEqual(tas1[0], tas2[0])
+
+ def test_filtering_caches_two_seperate_entries(self):
+ "Test that a filtered queryset is retrieved from the cache the second time it's needed, and that the cached values are different."
+ with self.assertNumQueries(1):
+ len(TestModelA.objects.filter(name__contains='f'))
+
+ with self.assertNumQueries(1):
+ len(TestModelA.objects.filter(name__contains='b'))
+
+ with self.assertNumQueries(0):
+ tas1 = TestModelA.objects.filter(name__contains='f')
+ self.assertEqual(len(tas1), 1)
+
+ with self.assertNumQueries(0):
+ tas2 = TestModelA.objects.filter(name__contains='b')
+ self.assertEqual(len(tas2), 1)
+
+ self.assertNotEqual(tas1[0], tas2[0])
+
+ def test_cache_invalidation_on_single_table_when_saving_after_get(self):
+ "Test that a call to save() invalidates the cache and results in a new DB query."
+ with self.assertNumQueries(1):
+ tas1a = TestModelA.objects.get(id=self.ta1.id)
+
+ with self.assertNumQueries(0):
+ tas1a = TestModelA.objects.get(id=self.ta1.id)
+
+ with self.assertNumQueries(2):
+ #the save() method triggers a SELECT using the ID. Even though we already did a get(), the query generated by save() is slightly different.
+ tas1a.save()
+
with self.assertNumQueries(1):
- tms = TestModel.objects.all()
- len(tms)
+ tas1a = TestModelA.objects.get(id=self.ta1.id)
with self.assertNumQueries(0):
- tms = TestModel.objects.all()
- len(tms)
+ tas1a = TestModelA.objects.get(id=self.ta1.id)
+ def test_cache_invalidation_on_single_table_when_saving_after_filter(self):
+ "Test that a call to save() invalidates the cache and results in a new DB query."
+ with self.assertNumQueries(1):
+ tas1 = TestModelA.objects.filter(name__contains='f')
+ self.assertEqual(len(tas1), 1)
+
+ with self.assertNumQueries(0):
+ tas1 = TestModelA.objects.filter(name__contains='f')
+ self.assertEqual(len(tas1), 1)
+
+ with self.assertNumQueries(2):
+ #the save() method triggers a SELECT using the ID, and that specific query hasn't been stored yet
+ tas1[0].save()
+
+ with self.assertNumQueries(1):
+ tas1 = TestModelA.objects.filter(name__contains='f')
+ self.assertEqual(len(tas1), 1)
+
+ with self.assertNumQueries(0):
+ tas1 = TestModelA.objects.filter(name__contains='f')
+ self.assertEqual(len(tas1), 1)
+ def test_cache_invalidation_on_single_table_when_updating(self):
+ "Test that a call to update() invalidates the cache and results in a new DB query."
+ with self.assertNumQueries(1):
+ tas1 = TestModelA.objects.filter(name__contains='f')
+ self.assertEqual(len(tas1), 1)
+
+ with self.assertNumQueries(0):
+ tas1 = TestModelA.objects.filter(name__contains='f')
+ self.assertEqual(len(tas1), 1)
+
+ with self.assertNumQueries(1):
+ TestModelA.objects.update(last_edited=datetime.now())
+
+ with self.assertNumQueries(1):
+ tas1 = TestModelA.objects.filter(name__contains='f')
+ self.assertEqual(len(tas1), 1)
+
+ with self.assertNumQueries(0):
+ tas1 = TestModelA.objects.filter(name__contains='f')
+ self.assertEqual(len(tas1), 1)
+
+ #TODO: test select_related(), invalidation from related model
+ #TODO: test one type of query on a model doesn't screw up other queries
+ #TODO: test that updating one entry will invalidate all cached entries for that table
+ #TODO: more testing for utility functions
+ #TODO: test extra(), annotate(), aggregate(), and raw SQL
+ #TODO: use proper func wrapping when patching
+ #TODO: test size limiting and table exclusion features
+ #TODO: test transaction rollback
+ def tearDown(self):
+ #TODO: add code to automatically clear cache between tests when installed
+ cache.clear()
@@ -142,17 +142,34 @@
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
+ 'formatters': {
+ 'verbose': {
+ 'format': '%(levelname)s %(asctime)s %(process)d %(thread)d %(name)s:%(lineno)s %(funcName)s() %(message)s'
+ },
+ 'verbose_sql': {
+ 'format': '%(levelname)s %(asctime)s %(process)d %(thread)d %(name)s:%(lineno)s %(funcName)s() %(message)s %(duration)s %(sql)s %(params)s'
+ },
+ },
'handlers': {
'mail_admins': {
'level': 'ERROR',
'class': 'django.utils.log.AdminEmailHandler'
- }
+ },
+ 'console': {
+ 'level': 'DEBUG',
+ 'class': 'logging.StreamHandler',
+ 'formatter': 'verbose',
+ },
},
'loggers': {
'django.request': {
'handlers': ['mail_admins'],
'level': 'ERROR',
'propagate': True,
},
+ },
+ 'root': {
+ 'handlers': ['console'],
+ 'level': 'INFO'#'DEBUG',
}
}
View
@@ -1,13 +1,15 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
+import os
from setuptools import setup
#import django_query_caching
package_name = 'django_query_caching'
+test_package_name = '%s_test_project' % package_name
setup(name='django-query-caching',
- version='0.1.2',
+ version='0.2.0dev',
description="Caches the results of SQL queries transparently.",
author='Seán Hayes',
author_email='sean@seanhayes.name',
@@ -28,7 +30,7 @@
url='http://seanhayes.name/',
download_url='https://github.com/SeanHayes/django-query-caching',
license='BSD',
- package_dir={'test_app': 'django_query_caching_test_project/apps/test_app/'},
+ package_dir={'test_app': os.path.join(test_package_name, 'apps', 'test_app')},
packages=[
'django_query_caching',
'django_query_caching.test',
@@ -37,6 +39,6 @@
],
include_package_data=True,
install_requires=['Django',],
- test_suite = '%s_test_project.runtests.runtests' % package_name,
+ test_suite = '%s.runtests.runtests' % test_package_name,
)

0 comments on commit d2b093d

Please sign in to comment.