diff --git a/django/core/cache/backends/db.py b/django/core/cache/backends/db.py index 62ea5c420bac2..af808108f9b5c 100644 --- a/django/core/cache/backends/db.py +++ b/django/core/cache/backends/db.py @@ -1,77 +1,68 @@ "Database cache backend." import base64 -import time -from datetime import datetime +from datetime import timedelta try: import cPickle as pickle except ImportError: import pickle -from django.conf import settings from django.core.cache.backends.base import BaseCache -from django.db import connections, router, transaction, DatabaseError +from django.db import connections, router, transaction, DatabaseError, models from django.utils import timezone - -class Options(object): - """A class that will quack like a Django model _meta class. - - This allows cache operations to be controlled by the router +def create_cache_model(table): + """ + This function will create a new cache table model to use for caching. The + model is created dynamically, and isn't part of app-loaded Django models. """ - def __init__(self, table): - self.db_table = table - self.app_label = 'django_cache' - self.module_name = 'cacheentry' - self.verbose_name = 'cache entry' - self.verbose_name_plural = 'cache entries' - self.object_name = 'CacheEntry' - self.abstract = False - self.managed = True - self.proxy = False + class CacheEntry(models.Model): + cache_key = models.CharField(max_length=255, unique=True, primary_key=True) + value = models.TextField() + expires = models.DateTimeField(db_index=True) + + class Meta: + db_table = table + verbose_name = 'cache entry' + verbose_name_plural = 'cache entries' + # We need to be able to create multiple different instances of + # this same class, and we don't want to leak entries into the + # app-cache. This model must not be part of the app-cache also + # because get_models() must not list any CacheEntry classes. So + # use this internal flag to skip this class totally. + _skip_app_cache = True + + opts = CacheEntry._meta + opts.app_label = 'django_cache' + opts.module_name = 'cacheentry' + return CacheEntry + class BaseDatabaseCache(BaseCache): def __init__(self, table, params): BaseCache.__init__(self, params) - self._table = table - - class CacheEntry(object): - _meta = Options(table) - self.cache_model_class = CacheEntry + self.cache_model_class = create_cache_model(table) + self.objects = self.cache_model_class.objects class DatabaseCache(BaseDatabaseCache): - # This class uses cursors provided by the database connection. This means - # it reads expiration values as aware or naive datetimes depending on the - # value of USE_TZ. They must be compared to aware or naive representations - # of "now" respectively. - - # But it bypasses the ORM for write operations. As a consequence, aware - # datetimes aren't made naive for databases that don't support time zones. - # We work around this problem by always using naive datetimes when writing - # expiration values, in UTC when USE_TZ = True and in local time otherwise. - def get(self, key, default=None, version=None): key = self.make_key(key, version=version) + db = router.db_for_write(self.cache_model_class) self.validate_key(key) - db = router.db_for_read(self.cache_model_class) - table = connections[db].ops.quote_name(self._table) - cursor = connections[db].cursor() - - cursor.execute("SELECT cache_key, value, expires FROM %s " - "WHERE cache_key = %%s" % table, [key]) - row = cursor.fetchone() - if row is None: + try: + obj = self.objects.using(db).get(cache_key=key) + except self.cache_model_class.DoesNotExist: return default now = timezone.now() - if row[2] < now: - db = router.db_for_write(self.cache_model_class) - cursor = connections[db].cursor() - cursor.execute("DELETE FROM %s " - "WHERE cache_key = %%s" % table, [key]) + if obj.expires < now: + obj.delete() transaction.commit_unless_managed(using=db) return default - value = connections[db].ops.process_clob(row[1]) + # Note: we must commit_unless_managed even for read-operations to + # avoid transaction leaks. + transaction.commit_unless_managed(using=db) + value = connections[db].ops.process_clob(obj.value) return pickle.loads(base64.decodestring(value)) def set(self, key, value, timeout=None, version=None): @@ -85,38 +76,28 @@ def add(self, key, value, timeout=None, version=None): return self._base_set('add', key, value, timeout) def _base_set(self, mode, key, value, timeout=None): + db = router.db_for_write(self.cache_model_class) if timeout is None: timeout = self.default_timeout - db = router.db_for_write(self.cache_model_class) - table = connections[db].ops.quote_name(self._table) - cursor = connections[db].cursor() - - cursor.execute("SELECT COUNT(*) FROM %s" % table) - num = cursor.fetchone()[0] now = timezone.now() now = now.replace(microsecond=0) - if settings.USE_TZ: - exp = datetime.utcfromtimestamp(time.time() + timeout) - else: - exp = datetime.fromtimestamp(time.time() + timeout) - exp = exp.replace(microsecond=0) + exp = now + timedelta(seconds=timeout) + num = self.objects.using(db).count() if num > self._max_entries: - self._cull(db, cursor, now) + self._cull(db, now) pickled = pickle.dumps(value, pickle.HIGHEST_PROTOCOL) encoded = base64.encodestring(pickled).strip() - cursor.execute("SELECT cache_key, expires FROM %s " - "WHERE cache_key = %%s" % table, [key]) try: - result = cursor.fetchone() - if result and (mode == 'set' or - (mode == 'add' and result[1] < now)): - cursor.execute("UPDATE %s SET value = %%s, expires = %%s " - "WHERE cache_key = %%s" % table, - [encoded, connections[db].ops.value_to_db_datetime(exp), key]) - else: - cursor.execute("INSERT INTO %s (cache_key, value, expires) " - "VALUES (%%s, %%s, %%s)" % table, - [key, encoded, connections[db].ops.value_to_db_datetime(exp)]) + try: + obj = self.objects.using(db).only('cache_key').get(cache_key=key) + if mode == 'set' or (mode == 'add' and obj.expires < now): + obj.expires = exp + obj.value = encoded + obj.save(using=db) + else: + return False + except self.cache_model_class.DoesNotExist: + self.objects.using(db).create(cache_key=key, expires=exp, value=encoded) except DatabaseError: # To be threadsafe, updates/inserts are allowed to fail silently transaction.rollback_unless_managed(using=db) @@ -130,10 +111,7 @@ def delete(self, key, version=None): self.validate_key(key) db = router.db_for_write(self.cache_model_class) - table = connections[db].ops.quote_name(self._table) - cursor = connections[db].cursor() - - cursor.execute("DELETE FROM %s WHERE cache_key = %%s" % table, [key]) + self.objects.using(db).filter(cache_key=key).delete() transaction.commit_unless_managed(using=db) def has_key(self, key, version=None): @@ -141,52 +119,33 @@ def has_key(self, key, version=None): self.validate_key(key) db = router.db_for_read(self.cache_model_class) - table = connections[db].ops.quote_name(self._table) - cursor = connections[db].cursor() - if settings.USE_TZ: - now = datetime.utcnow() - else: - now = datetime.now() + now = timezone.now() now = now.replace(microsecond=0) - cursor.execute("SELECT cache_key FROM %s " - "WHERE cache_key = %%s and expires > %%s" % table, - [key, connections[db].ops.value_to_db_datetime(now)]) - return cursor.fetchone() is not None + ret = self.objects.using(db).filter(cache_key=key, expires__gt=now).exists() + transaction.commit_unless_managed(using=db) + return ret - def _cull(self, db, cursor, now): + def _cull(self, db, now): if self._cull_frequency == 0: - self.clear() + # cull might be used inside other dbcache operations possibly already + # doing commits themselves - so do not commit in clear. + self.clear(commit=False) else: # When USE_TZ is True, 'now' will be an aware datetime in UTC. - now = now.replace(tzinfo=None) - table = connections[db].ops.quote_name(self._table) - cursor.execute("DELETE FROM %s WHERE expires < %%s" % table, - [connections[db].ops.value_to_db_datetime(now)]) - cursor.execute("SELECT COUNT(*) FROM %s" % table) - num = cursor.fetchone()[0] + self.objects.using(db).filter(expires__lt=now).delete() + num = self.objects.using(db).count() if num > self._max_entries: cull_num = num / self._cull_frequency - if connections[db].vendor == 'oracle': - # Oracle doesn't support LIMIT + OFFSET - cursor.execute("""SELECT cache_key FROM -(SELECT ROW_NUMBER() OVER (ORDER BY cache_key) AS counter, cache_key FROM %s) -WHERE counter > %%s AND COUNTER <= %%s""" % table, [cull_num, cull_num + 1]) - else: - # This isn't standard SQL, it's likely to break - # with some non officially supported databases - cursor.execute("SELECT cache_key FROM %s " - "ORDER BY cache_key " - "LIMIT 1 OFFSET %%s" % table, [cull_num]) - cursor.execute("DELETE FROM %s " - "WHERE cache_key < %%s" % table, - [cursor.fetchone()[0]]) - - def clear(self): + limit = self.objects.using(db).values_list( + 'cache_key').order_by('cache_key')[cull_num][0] + self.objects.using(db).filter(cache_key__lt=limit).delete() + + def clear(self, commit=True): db = router.db_for_write(self.cache_model_class) - table = connections[db].ops.quote_name(self._table) - cursor = connections[db].cursor() - cursor.execute('DELETE FROM %s' % table) + self.objects.using(db).delete() + if commit: + transaction.commit_unless_managed(using=db) # For backwards compatibility class CacheClass(DatabaseCache): diff --git a/django/core/management/commands/createcachetable.py b/django/core/management/commands/createcachetable.py index bcc47e17c80ad..ee6c5ad465638 100644 --- a/django/core/management/commands/createcachetable.py +++ b/django/core/management/commands/createcachetable.py @@ -2,7 +2,8 @@ from django.core.cache.backends.db import BaseDatabaseCache from django.core.management.base import LabelCommand, CommandError -from django.db import connections, router, transaction, models, DEFAULT_DB_ALIAS +from django.core.management.color import no_style +from django.db import connections, router, transaction, DEFAULT_DB_ALIAS from django.db.utils import DatabaseError class Command(LabelCommand): @@ -25,35 +26,16 @@ def handle_label(self, tablename, **options): if not router.allow_syncdb(db, cache.cache_model_class): return connection = connections[db] - fields = ( - # "key" is a reserved word in MySQL, so use "cache_key" instead. - models.CharField(name='cache_key', max_length=255, unique=True, primary_key=True), - models.TextField(name='value'), - models.DateTimeField(name='expires', db_index=True), - ) - table_output = [] - index_output = [] - qn = connection.ops.quote_name - for f in fields: - field_output = [qn(f.name), f.db_type(connection=connection)] - field_output.append("%sNULL" % (not f.null and "NOT " or "")) - if f.primary_key: - field_output.append("PRIMARY KEY") - elif f.unique: - field_output.append("UNIQUE") - if f.db_index: - unique = f.unique and "UNIQUE " or "" - index_output.append("CREATE %sINDEX %s ON %s (%s);" % \ - (unique, qn('%s_%s' % (tablename, f.name)), qn(tablename), - qn(f.name))) - table_output.append(" ".join(field_output)) - full_statement = ["CREATE TABLE %s (" % qn(tablename)] - for i, line in enumerate(table_output): - full_statement.append(' %s%s' % (line, i < len(table_output)-1 and ',' or '')) - full_statement.append(');') + creation = connection.creation + style = no_style() + tbl_output, _ = creation.sql_create_model(cache.cache_model_class, + style) + index_output = creation.sql_indexes_for_model(cache.cache_model_class, + style) curs = connection.cursor() try: - curs.execute("\n".join(full_statement)) + for statement in tbl_output: + curs.execute(statement) except DatabaseError as e: transaction.rollback_unless_managed(using=db) raise CommandError( diff --git a/django/db/models/base.py b/django/db/models/base.py index 13238fc9dcfc3..f582b88064416 100644 --- a/django/db/models/base.py +++ b/django/db/models/base.py @@ -91,7 +91,7 @@ def __new__(cls, name, bases, attrs): # Bail out early if we have already created this class. m = get_model(new_class._meta.app_label, name, seed_cache=False, only_installed=False) - if m is not None: + if m is not None and not new_class._meta._skip_app_cache: return m # Add all attributes to the class. @@ -196,6 +196,9 @@ def __new__(cls, name, bases, attrs): return new_class new_class._prepare() + if new_class._meta._skip_app_cache: + return new_class + register_models(new_class._meta.app_label, new_class) # Because of the way imports happen (recursively), we may or may not be diff --git a/django/db/models/options.py b/django/db/models/options.py index 44f8891942294..80a612d4c02e1 100644 --- a/django/db/models/options.py +++ b/django/db/models/options.py @@ -62,6 +62,10 @@ def __init__(self, meta, app_label=None): # List of all lookups defined in ForeignKey 'limit_choices_to' options # from *other* models. Needed for some admin checks. Internal use only. self.related_fkey_lookups = [] + # An internal flag used to keep track of which models to load into the + # app-cache. Sometimes (for example db caching) we want to use + # Django's models, but not make them part of the app-models. + self._skip_app_cache = False def contribute_to_class(self, cls, name): from django.db import connection @@ -82,6 +86,8 @@ def contribute_to_class(self, cls, name): # NOTE: We can't modify a dictionary's contents while looping # over it, so we loop over the *original* dictionary instead. if name.startswith('_'): + if name == '_skip_app_cache': + self._skip_app_cache = meta_attrs[name] del meta_attrs[name] for attr_name in DEFAULT_NAMES: if attr_name in meta_attrs: