Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Refactored db cache backend to use ORM
  • Loading branch information
akaariai committed May 29, 2012
1 parent 28e4245 commit d0133ce
Show file tree
Hide file tree
Showing 4 changed files with 92 additions and 142 deletions.
185 changes: 72 additions & 113 deletions django/core/cache/backends/db.py
@@ -1,77 +1,68 @@
"Database cache backend."
import base64
import time
from datetime import datetime
from datetime import timedelta

try:
import cPickle as pickle
except ImportError:
import pickle

from django.conf import settings
from django.core.cache.backends.base import BaseCache
from django.db import connections, router, transaction, DatabaseError
from django.db import connections, router, transaction, DatabaseError, models
from django.utils import timezone


class Options(object):
"""A class that will quack like a Django model _meta class.
This allows cache operations to be controlled by the router
def create_cache_model(table):
"""
This function will create a new cache table model to use for caching. The
model is created dynamically, and isn't part of app-loaded Django models.
"""
def __init__(self, table):
self.db_table = table
self.app_label = 'django_cache'
self.module_name = 'cacheentry'
self.verbose_name = 'cache entry'
self.verbose_name_plural = 'cache entries'
self.object_name = 'CacheEntry'
self.abstract = False
self.managed = True
self.proxy = False
class CacheEntry(models.Model):
cache_key = models.CharField(max_length=255, unique=True, primary_key=True)
value = models.TextField()
expires = models.DateTimeField(db_index=True)

class Meta:
db_table = table
verbose_name = 'cache entry'
verbose_name_plural = 'cache entries'
# We need to be able to create multiple different instances of
# this same class, and we don't want to leak entries into the
# app-cache. This model must not be part of the app-cache also
# because get_models() must not list any CacheEntry classes. So
# use this internal flag to skip this class totally.
_skip_app_cache = True

opts = CacheEntry._meta
opts.app_label = 'django_cache'
opts.module_name = 'cacheentry'
return CacheEntry


class BaseDatabaseCache(BaseCache):
def __init__(self, table, params):
BaseCache.__init__(self, params)
self._table = table

class CacheEntry(object):
_meta = Options(table)
self.cache_model_class = CacheEntry
self.cache_model_class = create_cache_model(table)
self.objects = self.cache_model_class.objects

class DatabaseCache(BaseDatabaseCache):

# This class uses cursors provided by the database connection. This means
# it reads expiration values as aware or naive datetimes depending on the
# value of USE_TZ. They must be compared to aware or naive representations
# of "now" respectively.

# But it bypasses the ORM for write operations. As a consequence, aware
# datetimes aren't made naive for databases that don't support time zones.
# We work around this problem by always using naive datetimes when writing
# expiration values, in UTC when USE_TZ = True and in local time otherwise.

def get(self, key, default=None, version=None):
key = self.make_key(key, version=version)
db = router.db_for_write(self.cache_model_class)
self.validate_key(key)
db = router.db_for_read(self.cache_model_class)
table = connections[db].ops.quote_name(self._table)
cursor = connections[db].cursor()

cursor.execute("SELECT cache_key, value, expires FROM %s "
"WHERE cache_key = %%s" % table, [key])
row = cursor.fetchone()
if row is None:
try:
obj = self.objects.using(db).get(cache_key=key)
except self.cache_model_class.DoesNotExist:
return default
now = timezone.now()
if row[2] < now:
db = router.db_for_write(self.cache_model_class)
cursor = connections[db].cursor()
cursor.execute("DELETE FROM %s "
"WHERE cache_key = %%s" % table, [key])
if obj.expires < now:
obj.delete()
transaction.commit_unless_managed(using=db)
return default
value = connections[db].ops.process_clob(row[1])
# Note: we must commit_unless_managed even for read-operations to
# avoid transaction leaks.
transaction.commit_unless_managed(using=db)
value = connections[db].ops.process_clob(obj.value)
return pickle.loads(base64.decodestring(value))

def set(self, key, value, timeout=None, version=None):
Expand All @@ -85,38 +76,28 @@ def add(self, key, value, timeout=None, version=None):
return self._base_set('add', key, value, timeout)

def _base_set(self, mode, key, value, timeout=None):
db = router.db_for_write(self.cache_model_class)
if timeout is None:
timeout = self.default_timeout
db = router.db_for_write(self.cache_model_class)
table = connections[db].ops.quote_name(self._table)
cursor = connections[db].cursor()

cursor.execute("SELECT COUNT(*) FROM %s" % table)
num = cursor.fetchone()[0]
now = timezone.now()
now = now.replace(microsecond=0)
if settings.USE_TZ:
exp = datetime.utcfromtimestamp(time.time() + timeout)
else:
exp = datetime.fromtimestamp(time.time() + timeout)
exp = exp.replace(microsecond=0)
exp = now + timedelta(seconds=timeout)
num = self.objects.using(db).count()
if num > self._max_entries:
self._cull(db, cursor, now)
self._cull(db, now)
pickled = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
encoded = base64.encodestring(pickled).strip()
cursor.execute("SELECT cache_key, expires FROM %s "
"WHERE cache_key = %%s" % table, [key])
try:
result = cursor.fetchone()
if result and (mode == 'set' or
(mode == 'add' and result[1] < now)):
cursor.execute("UPDATE %s SET value = %%s, expires = %%s "
"WHERE cache_key = %%s" % table,
[encoded, connections[db].ops.value_to_db_datetime(exp), key])
else:
cursor.execute("INSERT INTO %s (cache_key, value, expires) "
"VALUES (%%s, %%s, %%s)" % table,
[key, encoded, connections[db].ops.value_to_db_datetime(exp)])
try:
obj = self.objects.using(db).only('cache_key').get(cache_key=key)
if mode == 'set' or (mode == 'add' and obj.expires < now):
obj.expires = exp
obj.value = encoded
obj.save(using=db)
else:
return False
except self.cache_model_class.DoesNotExist:
self.objects.using(db).create(cache_key=key, expires=exp, value=encoded)
except DatabaseError:
# To be threadsafe, updates/inserts are allowed to fail silently
transaction.rollback_unless_managed(using=db)
Expand All @@ -130,63 +111,41 @@ def delete(self, key, version=None):
self.validate_key(key)

db = router.db_for_write(self.cache_model_class)
table = connections[db].ops.quote_name(self._table)
cursor = connections[db].cursor()

cursor.execute("DELETE FROM %s WHERE cache_key = %%s" % table, [key])
self.objects.using(db).filter(cache_key=key).delete()
transaction.commit_unless_managed(using=db)

def has_key(self, key, version=None):
key = self.make_key(key, version=version)
self.validate_key(key)

db = router.db_for_read(self.cache_model_class)
table = connections[db].ops.quote_name(self._table)
cursor = connections[db].cursor()

if settings.USE_TZ:
now = datetime.utcnow()
else:
now = datetime.now()
now = timezone.now()
now = now.replace(microsecond=0)
cursor.execute("SELECT cache_key FROM %s "
"WHERE cache_key = %%s and expires > %%s" % table,
[key, connections[db].ops.value_to_db_datetime(now)])
return cursor.fetchone() is not None
ret = self.objects.using(db).filter(cache_key=key, expires__gt=now).exists()
transaction.commit_unless_managed(using=db)
return ret

def _cull(self, db, cursor, now):
def _cull(self, db, now):
if self._cull_frequency == 0:
self.clear()
# cull might be used inside other dbcache operations possibly already
# doing commits themselves - so do not commit in clear.
self.clear(commit=False)
else:
# When USE_TZ is True, 'now' will be an aware datetime in UTC.
now = now.replace(tzinfo=None)
table = connections[db].ops.quote_name(self._table)
cursor.execute("DELETE FROM %s WHERE expires < %%s" % table,
[connections[db].ops.value_to_db_datetime(now)])
cursor.execute("SELECT COUNT(*) FROM %s" % table)
num = cursor.fetchone()[0]
self.objects.using(db).filter(expires__lt=now).delete()
num = self.objects.using(db).count()
if num > self._max_entries:
cull_num = num / self._cull_frequency
if connections[db].vendor == 'oracle':
# Oracle doesn't support LIMIT + OFFSET
cursor.execute("""SELECT cache_key FROM
(SELECT ROW_NUMBER() OVER (ORDER BY cache_key) AS counter, cache_key FROM %s)
WHERE counter > %%s AND COUNTER <= %%s""" % table, [cull_num, cull_num + 1])
else:
# This isn't standard SQL, it's likely to break
# with some non officially supported databases
cursor.execute("SELECT cache_key FROM %s "
"ORDER BY cache_key "
"LIMIT 1 OFFSET %%s" % table, [cull_num])
cursor.execute("DELETE FROM %s "
"WHERE cache_key < %%s" % table,
[cursor.fetchone()[0]])

def clear(self):
limit = self.objects.using(db).values_list(
'cache_key').order_by('cache_key')[cull_num][0]
self.objects.using(db).filter(cache_key__lt=limit).delete()

def clear(self, commit=True):
db = router.db_for_write(self.cache_model_class)
table = connections[db].ops.quote_name(self._table)
cursor = connections[db].cursor()
cursor.execute('DELETE FROM %s' % table)
self.objects.using(db).delete()
if commit:
transaction.commit_unless_managed(using=db)

# For backwards compatibility
class CacheClass(DatabaseCache):
Expand Down
38 changes: 10 additions & 28 deletions django/core/management/commands/createcachetable.py
Expand Up @@ -2,7 +2,8 @@

from django.core.cache.backends.db import BaseDatabaseCache
from django.core.management.base import LabelCommand, CommandError
from django.db import connections, router, transaction, models, DEFAULT_DB_ALIAS
from django.core.management.color import no_style
from django.db import connections, router, transaction, DEFAULT_DB_ALIAS
from django.db.utils import DatabaseError

class Command(LabelCommand):
Expand All @@ -25,35 +26,16 @@ def handle_label(self, tablename, **options):
if not router.allow_syncdb(db, cache.cache_model_class):
return
connection = connections[db]
fields = (
# "key" is a reserved word in MySQL, so use "cache_key" instead.
models.CharField(name='cache_key', max_length=255, unique=True, primary_key=True),
models.TextField(name='value'),
models.DateTimeField(name='expires', db_index=True),
)
table_output = []
index_output = []
qn = connection.ops.quote_name
for f in fields:
field_output = [qn(f.name), f.db_type(connection=connection)]
field_output.append("%sNULL" % (not f.null and "NOT " or ""))
if f.primary_key:
field_output.append("PRIMARY KEY")
elif f.unique:
field_output.append("UNIQUE")
if f.db_index:
unique = f.unique and "UNIQUE " or ""
index_output.append("CREATE %sINDEX %s ON %s (%s);" % \
(unique, qn('%s_%s' % (tablename, f.name)), qn(tablename),
qn(f.name)))
table_output.append(" ".join(field_output))
full_statement = ["CREATE TABLE %s (" % qn(tablename)]
for i, line in enumerate(table_output):
full_statement.append(' %s%s' % (line, i < len(table_output)-1 and ',' or ''))
full_statement.append(');')
creation = connection.creation
style = no_style()
tbl_output, _ = creation.sql_create_model(cache.cache_model_class,
style)
index_output = creation.sql_indexes_for_model(cache.cache_model_class,
style)
curs = connection.cursor()
try:
curs.execute("\n".join(full_statement))
for statement in tbl_output:
curs.execute(statement)
except DatabaseError as e:
transaction.rollback_unless_managed(using=db)
raise CommandError(
Expand Down
5 changes: 4 additions & 1 deletion django/db/models/base.py
Expand Up @@ -91,7 +91,7 @@ def __new__(cls, name, bases, attrs):
# Bail out early if we have already created this class.
m = get_model(new_class._meta.app_label, name,
seed_cache=False, only_installed=False)
if m is not None:
if m is not None and not new_class._meta._skip_app_cache:
return m

# Add all attributes to the class.
Expand Down Expand Up @@ -196,6 +196,9 @@ def __new__(cls, name, bases, attrs):
return new_class

new_class._prepare()
if new_class._meta._skip_app_cache:
return new_class

register_models(new_class._meta.app_label, new_class)

# Because of the way imports happen (recursively), we may or may not be
Expand Down
6 changes: 6 additions & 0 deletions django/db/models/options.py
Expand Up @@ -62,6 +62,10 @@ def __init__(self, meta, app_label=None):
# List of all lookups defined in ForeignKey 'limit_choices_to' options
# from *other* models. Needed for some admin checks. Internal use only.
self.related_fkey_lookups = []
# An internal flag used to keep track of which models to load into the
# app-cache. Sometimes (for example db caching) we want to use
# Django's models, but not make them part of the app-models.
self._skip_app_cache = False

def contribute_to_class(self, cls, name):
from django.db import connection
Expand All @@ -82,6 +86,8 @@ def contribute_to_class(self, cls, name):
# NOTE: We can't modify a dictionary's contents while looping
# over it, so we loop over the *original* dictionary instead.
if name.startswith('_'):
if name == '_skip_app_cache':
self._skip_app_cache = meta_attrs[name]
del meta_attrs[name]
for attr_name in DEFAULT_NAMES:
if attr_name in meta_attrs:
Expand Down

0 comments on commit d0133ce

Please sign in to comment.