Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Added TransactionTestCase.available_apps.
This makes Django's test suite significantly faster by reducing the
number of models for which content types and permissions must be created
and tables must be flushed in each non-transactional test.

Most of the credit goes to Anssi. He got the idea and did the research.

It's documented for Django contributors and committers but it's branded
as a private API to preserve our freedom to change it in the future.

Refs #20483.
  • Loading branch information
aaugustin committed Jun 10, 2013
1 parent 13b7f29 commit 4e7dcc9
Show file tree
Hide file tree
Showing 6 changed files with 127 additions and 31 deletions.
15 changes: 12 additions & 3 deletions django/contrib/auth/management/__init__.py
Expand Up @@ -11,7 +11,7 @@
from django.core import exceptions
from django.core.management.base import CommandError
from django.db import DEFAULT_DB_ALIAS, router
from django.db.models import get_models, signals
from django.db.models import get_model, get_models, signals, UnavailableApp
from django.utils.encoding import DEFAULT_LOCALE_ENCODING
from django.utils import six
from django.utils.six.moves import input
Expand Down Expand Up @@ -60,6 +60,11 @@ def _check_permission_clashing(custom, builtin, ctype):
pool.add(codename)

def create_permissions(app, created_models, verbosity, db=DEFAULT_DB_ALIAS, **kwargs):
try:
get_model('auth', 'Permission')
except UnavailableApp:
return

if not router.allow_syncdb(db, auth_app.Permission):
return

Expand Down Expand Up @@ -101,9 +106,13 @@ def create_permissions(app, created_models, verbosity, db=DEFAULT_DB_ALIAS, **kw


def create_superuser(app, created_models, verbosity, db, **kwargs):
from django.core.management import call_command
try:
get_model('auth', 'Permission')
UserModel = get_user_model()
except UnavailableApp:
return

UserModel = get_user_model()
from django.core.management import call_command

if UserModel in created_models and kwargs.get('interactive', True):
msg = ("\nYou just installed Django's auth system, which means you "
Expand Down
7 changes: 6 additions & 1 deletion django/contrib/contenttypes/management.py
@@ -1,6 +1,6 @@
from django.contrib.contenttypes.models import ContentType
from django.db import DEFAULT_DB_ALIAS, router
from django.db.models import get_apps, get_models, signals
from django.db.models import get_apps, get_model, get_models, signals, UnavailableApp
from django.utils.encoding import smart_text
from django.utils import six
from django.utils.six.moves import input
Expand All @@ -11,6 +11,11 @@ def update_contenttypes(app, created_models, verbosity=2, db=DEFAULT_DB_ALIAS, *
Creates content types for models in the given app, removing any model
entries that no longer have a matching model class.
"""
try:
get_model('contenttypes', 'ContentType')
except UnavailableApp:
return

if not router.allow_syncdb(db, ContentType):
return

Expand Down
2 changes: 1 addition & 1 deletion django/db/models/__init__.py
@@ -1,7 +1,7 @@
from functools import wraps

from django.core.exceptions import ObjectDoesNotExist, ImproperlyConfigured
from django.db.models.loading import get_apps, get_app_paths, get_app, get_models, get_model, register_models
from django.db.models.loading import get_apps, get_app_paths, get_app, get_models, get_model, register_models, UnavailableApp
from django.db.models.query import Q
from django.db.models.expressions import F
from django.db.models.manager import Manager
Expand Down
62 changes: 50 additions & 12 deletions django/db/models/loading.py
Expand Up @@ -15,6 +15,8 @@
__all__ = ('get_apps', 'get_app', 'get_models', 'get_model', 'register_models',
'load_app', 'app_cache_ready')

class UnavailableApp(Exception):
pass

class AppCache(object):
"""
Expand Down Expand Up @@ -43,6 +45,7 @@ class AppCache(object):
postponed=[],
nesting_level=0,
_get_models_cache={},
available_apps=None,
)

def __init__(self):
Expand Down Expand Up @@ -135,12 +138,17 @@ def get_apps(self):
"""
self._populate()

apps = self.app_store.items()
if self.available_apps is not None:
apps = [elt for elt in apps
if self._label_for(elt[0]) in self.available_apps]

# Ensure the returned list is always in the same order (with new apps
# added at the end). This avoids unstable ordering on the admin app
# list page, for example.
apps = [(v, k) for k, v in self.app_store.items()]
apps.sort()
return [elt[1] for elt in apps]
apps = sorted(apps, key=lambda elt: elt[1])

return [elt[0] for elt in apps]

def get_app_paths(self):
"""
Expand All @@ -161,21 +169,24 @@ def get_app_paths(self):

def get_app(self, app_label, emptyOK=False):
"""
Returns the module containing the models for the given app_label. If
the app has no models in it and 'emptyOK' is True, returns None.
Returns the module containing the models for the given app_label.
Returns None if the app has no models in it and emptyOK is True.
Raises UnavailableApp when set_available_apps() in in effect and
doesn't include app_label.
"""
self._populate()
imp.acquire_lock()
try:
for app_name in settings.INSTALLED_APPS:
if app_label == app_name.split('.')[-1]:
mod = self.load_app(app_name, False)
if mod is None:
if emptyOK:
return None
if mod is None and not emptyOK:
raise ImproperlyConfigured("App with label %s is missing a models.py module." % app_label)
else:
return mod
if self.available_apps is not None and app_label not in self.available_apps:
raise UnavailableApp("App with label %s isn't available." % app_label)
return mod
raise ImproperlyConfigured("App with label %s could not be found" % app_label)
finally:
imp.release_lock()
Expand Down Expand Up @@ -209,8 +220,13 @@ def get_models(self, app_mod=None,
include_swapped, they will be.
"""
cache_key = (app_mod, include_auto_created, include_deferred, only_installed, include_swapped)
model_list = None
try:
return self._get_models_cache[cache_key]
model_list = self._get_models_cache[cache_key]
if self.available_apps is not None and only_installed:
model_list = [m for m in model_list
if m._meta.app_label in self.available_apps]
return model_list
except KeyError:
pass
self._populate()
Expand All @@ -235,6 +251,9 @@ def get_models(self, app_mod=None,
(not model._meta.swapped or include_swapped))
)
self._get_models_cache[cache_key] = model_list
if self.available_apps is not None and only_installed:
model_list = [m for m in model_list
if m._meta.app_label in self.available_apps]
return model_list

def get_model(self, app_label, model_name,
Expand All @@ -244,12 +263,21 @@ def get_model(self, app_label, model_name,
model_name.
Returns None if no model is found.
Raises UnavailableApp when set_available_apps() in in effect and
doesn't include app_label.
"""
if seed_cache:
self._populate()
if only_installed and app_label not in self.app_labels:
return None
return self.app_models.get(app_label, SortedDict()).get(model_name.lower())
if (self.available_apps is not None and only_installed
and app_label not in self.available_apps):
raise UnavailableApp("App with label %s isn't available." % app_label)
try:
return self.app_models[app_label][model_name.lower()]
except KeyError:
return None

def register_models(self, app_label, *models):
"""
Expand All @@ -274,6 +302,16 @@ def register_models(self, app_label, *models):
model_dict[model_name] = model
self._get_models_cache.clear()

def set_available_apps(self, available):
if not set(available).issubset(set(settings.INSTALLED_APPS)):
extra = set(available) - set(settings.INSTALLED_APPS)
raise ValueError("Available apps isn't a subset of installed "
"apps, extra apps: " + ", ".join(extra))
self.available_apps = set(app.rsplit('.', 1)[-1] for app in available)

def unset_available_apps(self):
self.available_apps = None

cache = AppCache()

# These methods were always module level, so are kept that way for backwards
Expand Down
44 changes: 30 additions & 14 deletions django/test/testcases.py
Expand Up @@ -28,6 +28,7 @@
WSGIServerException)
from django.core.urlresolvers import clear_url_caches, set_urlconf
from django.db import connection, connections, DEFAULT_DB_ALIAS, transaction
from django.db.models.loading import cache
from django.forms.fields import CharField
from django.http import QueryDict
from django.test.client import Client
Expand Down Expand Up @@ -725,6 +726,9 @@ class TransactionTestCase(SimpleTestCase):
# test case
reset_sequences = False

# Subclasses can enable only a subset of apps for faster tests
available_apps = None

def _pre_setup(self):
"""Performs any pre-test setup. This includes:
Expand All @@ -733,7 +737,14 @@ def _pre_setup(self):
named fixtures.
"""
super(TransactionTestCase, self)._pre_setup()
self._fixture_setup()
if self.available_apps is not None:
cache.set_available_apps(self.available_apps)
try:
self._fixture_setup()
except Exception:
if self.available_apps is not None:
cache.unset_available_apps()
raise

def _databases_names(self, include_mirrors=True):
# If the test case has a multi_db=True flag, act on all databases,
Expand Down Expand Up @@ -775,22 +786,27 @@ def _post_teardown(self):
* Force closing the connection, so that the next test gets
a clean cursor.
"""
self._fixture_teardown()
super(TransactionTestCase, self)._post_teardown()
# Some DB cursors include SQL statements as part of cursor
# creation. If you have a test that does rollback, the effect
# of these statements is lost, which can effect the operation
# of tests (e.g., losing a timezone setting causing objects to
# be created with the wrong time).
# To make sure this doesn't happen, get a clean connection at the
# start of every test.
for conn in connections.all():
conn.close()
try:
self._fixture_teardown()
super(TransactionTestCase, self)._post_teardown()
# Some DB cursors include SQL statements as part of cursor
# creation. If you have a test that does rollback, the effect of
# these statements is lost, which can effect the operation of
# tests (e.g., losing a timezone setting causing objects to be
# created with the wrong time). To make sure this doesn't happen,
# get a clean connection at the start of every test.
for conn in connections.all():
conn.close()
finally:
cache.unset_available_apps()

def _fixture_teardown(self):
# Allow TRUNCATE ... CASCADE when flushing only a subset of the apps
allow_cascade = self.available_apps is not None
for db_name in self._databases_names(include_mirrors=False):
call_command('flush', verbosity=0, interactive=False, database=db_name,
skip_validation=True, reset_sequences=False)
call_command('flush', verbosity=0, interactive=False,
database=db_name, skip_validation=True,
reset_sequences=False, allow_cascade=allow_cascade)

def assertQuerysetEqual(self, qs, values, transform=repr, ordered=True):
items = six.moves.map(transform, qs)
Expand Down
28 changes: 28 additions & 0 deletions docs/topics/testing/overview.txt
Expand Up @@ -985,6 +985,34 @@ to test the effects of commit and rollback:
Using ``reset_sequences = True`` will slow down the test, since the primary
key reset is an relatively expensive database operation.

.. attribute:: TransactionTestCase.available_apps

.. warning::

This attribute is a private API. It may be changed or removed without
a deprecation period in the future, for instance to accomodate changes
in application loading.

It's used to optimize Django's own test suite, which contains hundreds
of models but no relations between models in different applications.

.. versionadded:: 1.6

By default, ``available_apps`` is set to ``None`` and has no effect.
Setting it to a list of applications tells Django to behave as if only the
models from these applications were available:

- Before each test, Django creates content types and permissions only for
these models.
- After each test, Django flushes only the corresponding tables. However,
at the database level, truncation may cascade to other related models,
even if they aren't in ``available_apps``.

Since the database isn't fully flushed, if a test creates instances of
models not included in ``available_apps``, they will leak and they may
cause unrelated tests to fail. Be careful with tests that use sessions;
the default session engine stores them in the database.

TestCase
~~~~~~~~

Expand Down

0 comments on commit 4e7dcc9

Please sign in to comment.