Browse files

Fixed #22487: Optional rollback emulation for migrated apps

  • Loading branch information...
1 parent 8721adc commit 8c12d51ea27479555e226894c50c83043211d71d @andrewgodwin andrewgodwin committed Jun 9, 2014
@@ -578,6 +578,10 @@
# The name of the class to use to run the test suite
TEST_RUNNER = 'django.test.runner.DiscoverRunner'
+# Apps that don't need to be serialized at test database creation time
+# (only apps with migrations are to start with)
@@ -22,10 +22,9 @@ class Command(NoArgsCommand):
make_option('--no-initial-data', action='store_false', dest='load_initial_data', default=True,
help='Tells Django not to load any initial data after database synchronization.'),
- help = ('Returns the database to the state it was in immediately after '
- 'migrate was first executed. This means that all data will be removed '
- 'from the database, any post-migration handlers will be '
- 're-executed, and the initial_data fixture will be re-installed.')
+ help = ('Removes ALL DATA from the database, including data added during '
+ 'migrations. Unmigrated apps will also have their initial_data '
+ 'fixture reloaded. Does not achieve a "fresh install" state.')
def handle_noargs(self, **options):
database = options.get('database')
@@ -54,7 +53,7 @@ def handle_noargs(self, **options):
if interactive:
confirm = input("""You have requested a flush of the database.
This will IRREVERSIBLY DESTROY all data currently in the %r database,
-and return each table to a fresh state.
+and return each table to an empty state.
Are you sure you want to do this?
Type 'yes' to continue, or 'no' to cancel: """ % connection.settings_dict['NAME'])
@@ -27,7 +27,7 @@ def handle(self, *fixture_labels, **options):
addrport = options.get('addrport')
# Create a test database.
- db_name = connection.creation.create_test_db(verbosity=verbosity, autoclobber=not interactive)
+ db_name = connection.creation.create_test_db(verbosity=verbosity, autoclobber=not interactive, serialize=False)
# Import the fixture data into the test database.
call_command('loaddata', *fixture_labels, **{'verbosity': verbosity})
@@ -7,6 +7,11 @@
from django.utils.encoding import force_bytes
from django.utils.functional import cached_property
from django.utils.six.moves import input
+from django.utils.six import StringIO
+from import sort_dependencies
+from django.db import router
+from django.apps import apps
+from django.core import serializers
from .utils import truncate_name
@@ -332,7 +337,7 @@ def sql_destroy_indexes_for_fields(self, model, fields, style):
- def create_test_db(self, verbosity=1, autoclobber=False, keepdb=False):
+ def create_test_db(self, verbosity=1, autoclobber=False, keepdb=False, serialize=True):
Creates a test database, prompting the user for confirmation if the
database already exists. Returns the name of the test database created.
@@ -364,25 +369,31 @@ def create_test_db(self, verbosity=1, autoclobber=False, keepdb=False):
settings.DATABASES[self.connection.alias]["NAME"] = test_database_name
self.connection.settings_dict["NAME"] = test_database_name
- # Report migrate messages at one level lower than that requested.
+ # We report migrate messages at one level lower than that requested.
# This ensures we don't get flooded with messages during testing
- # (unless you really ask to be flooded)
- call_command('migrate',
+ # (unless you really ask to be flooded).
+ call_command(
+ 'migrate',
verbosity=max(verbosity - 1, 0),
- load_initial_data=False,
- test_database=True)
- # We need to then do a flush to ensure that any data installed by
- # custom SQL has been removed. The only test data should come from
- # test fixtures, or autogenerated from post_migrate triggers.
- # This has the side effect of loading initial data (which was
- # intentionally skipped in the syncdb).
- call_command('flush',
+ test_database=True,
+ )
+ # We then serialize the current state of the database into a string
+ # and store it on the connection. This slightly horrific process is so people
+ # who are testing on databases without transactions or who are using
+ # a TransactionTestCase still get a clean database on every test run.
+ if serialize:
+ self.connection._test_serialized_contents = self.serialize_db_to_string()
+ # Finally, we flush the database to clean
+ call_command(
+ 'flush',
verbosity=max(verbosity - 1, 0),
- database=self.connection.alias)
+ database=self.connection.alias
+ )
call_command('createcachetable', database=self.connection.alias)
@@ -391,6 +402,44 @@ def create_test_db(self, verbosity=1, autoclobber=False, keepdb=False):
return test_database_name
+ def serialize_db_to_string(self):
+ """
+ Serializes all data in the database into a JSON string.
+ Designed only for test runner usage; will not handle large
+ amounts of data.
+ """
+ # Build list of all apps to serialize
+ from django.db.migrations.loader import MigrationLoader
+ loader = MigrationLoader(self.connection)
+ app_list = []
+ for app_config in apps.get_app_configs():
+ if (
+ app_config.models_module is not None and
+ app_config.label in loader.migrated_apps and
+ not in settings.TEST_NON_SERIALIZED_APPS
+ ):
+ app_list.append((app_config, None))
+ # Make a function to iteratively return every object
+ def get_objects():
+ for model in sort_dependencies(app_list):
+ if not model._meta.proxy and router.allow_migrate(self.connection.alias, model):
+ queryset = model._default_manager.using(self.connection.alias).order_by(
+ for obj in queryset.iterator():
+ yield obj
+ # Serialise to a string
+ out = StringIO()
+ serializers.serialize("json", get_objects(), indent=None, stream=out)
+ return out.getvalue()
+ def deserialize_db_from_string(self, data):
+ """
+ Reloads the database with data from a string generated by
+ the serialize_db_to_string method.
+ """
+ data = StringIO(data)
+ for obj in serializers.deserialize("json", data, using=self.connection.alias):
def _get_test_db_name(self):
Internal implementation - returns the name of the test DB that will be
@@ -298,7 +298,11 @@ def setup_databases(verbosity, interactive, keepdb=False, **kwargs):
connection = connections[alias]
if test_db_name is None:
test_db_name = connection.creation.create_test_db(
- verbosity, autoclobber=not interactive, keepdb=keepdb)
+ verbosity,
+ autoclobber=not interactive,
+ keepdb=keepdb,
+ serialize=connection.settings_dict.get("TEST_SERIALIZE", True),
+ )
destroy = True
connection.settings_dict['NAME'] = test_db_name
@@ -753,6 +753,12 @@ class TransactionTestCase(SimpleTestCase):
# Subclasses can define fixtures which will be automatically installed.
fixtures = None
+ # If transactions aren't available, Django will serialize the database
+ # contents into a fixture during setup and flush and reload them
+ # during teardown (as flush does not restore data from migrations).
+ # This can be slow; this flag allows enabling on a per-case basis.
+ serialized_rollback = False
def _pre_setup(self):
"""Performs any pre-test setup. This includes:
@@ -808,6 +814,17 @@ def _fixture_setup(self):
if self.reset_sequences:
+ # If we need to provide replica initial data from migrated apps,
+ # then do so.
+ if self.serialized_rollback and hasattr(connections[db_name], "_test_serialized_contents"):
+ if self.available_apps is not None:
+ apps.unset_available_apps()
+ connections[db_name].creation.deserialize_db_from_string(
+ connections[db_name]._test_serialized_contents
+ )
+ if self.available_apps is not None:
+ apps.set_available_apps(self.available_apps)
if self.fixtures:
# We have to use this slightly awkward syntax due to the fact
# that we're using *args and **kwargs together.
@@ -844,12 +861,14 @@ def _fixture_teardown(self):
# Allow TRUNCATE ... CASCADE and don't emit the post_migrate signal
# when flushing only a subset of the apps
for db_name in self._databases_names(include_mirrors=False):
+ # Flush the database
call_command('flush', verbosity=0, interactive=False,
database=db_name, skip_checks=True,
allow_cascade=self.available_apps is not None,
inhibit_post_migrate=self.available_apps is not None)
def assertQuerysetEqual(self, qs, values, transform=repr, ordered=True, msg=None):
items =, qs)
if not ordered:
@@ -199,8 +199,9 @@ model::
# We get the model from the versioned app registry;
# if we directly import it, it'll be the wrong version
Country = apps.get_model("myapp", "Country")
- Country.objects.create(name="USA", code="us")
- Country.objects.create(name="France", code="fr")
+ db_alias = schema_editor.connection.alias
+ Country.objects.create(name="USA", code="us", using=db_alias)
+ Country.objects.create(name="France", code="fr", using=db_alias)
class Migration(migrations.Migration):
@@ -236,6 +237,14 @@ Oracle). This should be safe, but may cause a crash if you attempt to use
the ``schema_editor`` provided on these backends; in this case, please
set ``atomic=False``.
+.. warning::
+ RunPython does not magically alter the connection of the models for you;
+ any model methods you call will go to the default database unless you
+ give them the current database alias (available from
+ ``schema_editor.connection.alias``, where ``schema_editor`` is the second
+ argument to your function).
@@ -2078,6 +2078,24 @@ Default: ``'django.test.runner.DiscoverRunner'``
The name of the class to use for starting the test suite. See
+Default: ``[]``
+In order to restore the database state between tests for TransactionTestCases
+and database backends without transactions, Django will :ref:`serialize the
+contents of all apps with migrations <test-case-serialized-rollback>` when it
+starts the test run so it can then reload from that copy before tests that
+need it.
+This slows down the startup time of the test runner; if you have apps that
+you know don't need this feature, you can add their full names in here (e.g.
+``django.contrib.contenttypes``) to exclude them from this serialization
@@ -63,6 +63,10 @@ but a few of the key features are:
* ``initial_data`` fixtures are no longer loaded for apps with migrations; if
you want to load initial data for an app, we suggest you do it in a migration.
+* Test rollback behaviour is different for apps with migrations; in particular,
+ Django will no longer emulate rollbacks on non-transactional databases or
+ inside ``TransactionTestCase`` :ref:`unless specifically asked <test-case-serialized-rollback>`.
App-loading refactor
@@ -485,7 +485,7 @@ django.db.connection.creation
The creation module of the database backend also provides some utilities that
can be useful during testing.
-.. function:: create_test_db([verbosity=1, autoclobber=False, keepdb=False])
+.. function:: create_test_db([verbosity=1, autoclobber=False, keepdb=False, serialize=True])
Creates a new test database and runs ``migrate`` against it.
@@ -507,6 +507,12 @@ can be useful during testing.
a new database will be created, prompting the user to remove
the existing one, if present.
+ ``serialize`` determines if Django serializes the database into an
+ in-memory JSON string before running tests (used to restore the database
+ state between tests if you don't have transactions). You can set this to
+ False to significantly speed up creation time if you know you don't need
+ data persistance outside of test fixtures.
Returns the name of the test database that it created.
``create_test_db()`` has the side effect of modifying the value of
@@ -234,6 +234,33 @@ the Django test runner reorders tests in the following way:
database by a given :class:`~django.test.TransactionTestCase` test, they
must be updated to be able to run independently.
+.. _test-case-serialized-rollback:
+Rollback emulation
+Any initial data loaded in migrations will only be available in ``TestCase``
+tests and not in ``TransactionTestCase`` tests, and additionally only on
+backends where transactions are supported (the most important exception being
+Django can re-load that data for you on a per-testcase basis by
+setting the ``serialized_rollback`` option to ``True`` in the body of the
+``TestCase`` or ``TransactionTestCase``, but note that this will slow down
+that test suite by approximately 3x.
+Third-party apps or those developing against MyISAM will need to set this;
+in general, however, you should be developing your own projects against a
+transactional database and be using ``TestCase`` for most tests, and thus
+not need this setting.
+The initial serialization is usually very quick, but if you wish to exclude
+some apps from this process (and speed up test runs slightly), you may add
+those apps to :setting:`TEST_NON_SERIALIZED_APPS`.
+Apps without migrations are not affected; ``initial_data`` fixtures are
+reloaded as usual.
Other test conditions
@@ -249,6 +276,7 @@ used. This behavior `may change`_ in the future.
.. _may change:
Understanding the test output
@@ -600,9 +600,17 @@ to test the effects of commit and rollback:
guarantees that the rollback at the end of the test restores the database to
its initial state.
- When running on a database that does not support rollback (e.g. MySQL with the
- MyISAM storage engine), ``TestCase`` falls back to initializing the database
- by truncating tables and reloading initial data.
+.. warning::
+ ``TestCase`` running on a database that does not support rollback (e.g. MySQL with the
+ MyISAM storage engine), and all instances of ``TransactionTestCase``, will
+ roll back at the end of the test by deleting all data from the test database
+ and reloading initial data for apps without migrations.
+ Apps with migrations :ref:`will not see their data reloaded <test-case-serialized-rollback>`;
+ if you need this functionality (for example, third-party apps should enable
+ this) you can set ``serialized_rollback = True`` inside the
+ ``TestCase`` body.
.. warning::
@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+from django.db import models, migrations
+def add_book(apps, schema_editor):
+ apps.get_model("migration_test_data_persistence", "Book").objects.using(
+ schema_editor.connection.alias,
+ ).create(
+ title="I Love Django",
+ )
+class Migration(migrations.Migration):
+ dependencies = [
+ ]
+ operations = [
+ migrations.CreateModel(
+ name='Book',
+ fields=[
+ ('id', models.AutoField(verbose_name='ID', primary_key=True, serialize=False, auto_created=True)),
+ ('title', models.CharField(max_length=100)),
+ ],
+ options={
+ },
+ bases=(models.Model,),
+ ),
+ migrations.RunPython(
+ add_book,
+ ),
+ ]
Oops, something went wrong. Retry.

0 comments on commit 8c12d51

Please sign in to comment.