Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Fixed #22487: Optional rollback emulation for migrated apps

  • Loading branch information...
commit 8c12d51ea27479555e226894c50c83043211d71d 1 parent 8721adc
@andrewgodwin andrewgodwin authored
View
4 django/conf/global_settings.py
@@ -578,6 +578,10 @@
# The name of the class to use to run the test suite
TEST_RUNNER = 'django.test.runner.DiscoverRunner'
+# Apps that don't need to be serialized at test database creation time
+# (only apps with migrations are to start with)
+TEST_NON_SERIALIZED_APPS = []
+
############
# FIXTURES #
############
View
9 django/core/management/commands/flush.py
@@ -22,10 +22,9 @@ class Command(NoArgsCommand):
make_option('--no-initial-data', action='store_false', dest='load_initial_data', default=True,
help='Tells Django not to load any initial data after database synchronization.'),
)
- help = ('Returns the database to the state it was in immediately after '
- 'migrate was first executed. This means that all data will be removed '
- 'from the database, any post-migration handlers will be '
- 're-executed, and the initial_data fixture will be re-installed.')
+ help = ('Removes ALL DATA from the database, including data added during '
+ 'migrations. Unmigrated apps will also have their initial_data '
+ 'fixture reloaded. Does not achieve a "fresh install" state.')
def handle_noargs(self, **options):
database = options.get('database')
@@ -54,7 +53,7 @@ def handle_noargs(self, **options):
if interactive:
confirm = input("""You have requested a flush of the database.
This will IRREVERSIBLY DESTROY all data currently in the %r database,
-and return each table to a fresh state.
+and return each table to an empty state.
Are you sure you want to do this?
Type 'yes' to continue, or 'no' to cancel: """ % connection.settings_dict['NAME'])
View
2  django/core/management/commands/testserver.py
@@ -27,7 +27,7 @@ def handle(self, *fixture_labels, **options):
addrport = options.get('addrport')
# Create a test database.
- db_name = connection.creation.create_test_db(verbosity=verbosity, autoclobber=not interactive)
+ db_name = connection.creation.create_test_db(verbosity=verbosity, autoclobber=not interactive, serialize=False)
# Import the fixture data into the test database.
call_command('loaddata', *fixture_labels, **{'verbosity': verbosity})
View
77 django/db/backends/creation.py
@@ -7,6 +7,11 @@
from django.utils.encoding import force_bytes
from django.utils.functional import cached_property
from django.utils.six.moves import input
+from django.utils.six import StringIO
+from django.core.management.commands.dumpdata import sort_dependencies
+from django.db import router
+from django.apps import apps
+from django.core import serializers
from .utils import truncate_name
@@ -332,7 +337,7 @@ def sql_destroy_indexes_for_fields(self, model, fields, style):
";",
]
- def create_test_db(self, verbosity=1, autoclobber=False, keepdb=False):
+ def create_test_db(self, verbosity=1, autoclobber=False, keepdb=False, serialize=True):
"""
Creates a test database, prompting the user for confirmation if the
database already exists. Returns the name of the test database created.
@@ -364,25 +369,31 @@ def create_test_db(self, verbosity=1, autoclobber=False, keepdb=False):
settings.DATABASES[self.connection.alias]["NAME"] = test_database_name
self.connection.settings_dict["NAME"] = test_database_name
- # Report migrate messages at one level lower than that requested.
+ # We report migrate messages at one level lower than that requested.
# This ensures we don't get flooded with messages during testing
- # (unless you really ask to be flooded)
- call_command('migrate',
+ # (unless you really ask to be flooded).
+ call_command(
+ 'migrate',
verbosity=max(verbosity - 1, 0),
interactive=False,
database=self.connection.alias,
- load_initial_data=False,
- test_database=True)
-
- # We need to then do a flush to ensure that any data installed by
- # custom SQL has been removed. The only test data should come from
- # test fixtures, or autogenerated from post_migrate triggers.
- # This has the side effect of loading initial data (which was
- # intentionally skipped in the syncdb).
- call_command('flush',
+ test_database=True,
+ )
+
+ # We then serialize the current state of the database into a string
+ # and store it on the connection. This slightly horrific process is so people
+ # who are testing on databases without transactions or who are using
+ # a TransactionTestCase still get a clean database on every test run.
+ if serialize:
+ self.connection._test_serialized_contents = self.serialize_db_to_string()
+
+ # Finally, we flush the database to clean
+ call_command(
+ 'flush',
verbosity=max(verbosity - 1, 0),
interactive=False,
- database=self.connection.alias)
+ database=self.connection.alias
+ )
call_command('createcachetable', database=self.connection.alias)
@@ -391,6 +402,44 @@ def create_test_db(self, verbosity=1, autoclobber=False, keepdb=False):
return test_database_name
+ def serialize_db_to_string(self):
+ """
+ Serializes all data in the database into a JSON string.
+ Designed only for test runner usage; will not handle large
+ amounts of data.
+ """
+ # Build list of all apps to serialize
+ from django.db.migrations.loader import MigrationLoader
+ loader = MigrationLoader(self.connection)
+ app_list = []
+ for app_config in apps.get_app_configs():
+ if (
+ app_config.models_module is not None and
+ app_config.label in loader.migrated_apps and
+ app_config.name not in settings.TEST_NON_SERIALIZED_APPS
+ ):
+ app_list.append((app_config, None))
+ # Make a function to iteratively return every object
+ def get_objects():
+ for model in sort_dependencies(app_list):
+ if not model._meta.proxy and router.allow_migrate(self.connection.alias, model):
+ queryset = model._default_manager.using(self.connection.alias).order_by(model._meta.pk.name)
+ for obj in queryset.iterator():
+ yield obj
+ # Serialise to a string
+ out = StringIO()
+ serializers.serialize("json", get_objects(), indent=None, stream=out)
+ return out.getvalue()
+
+ def deserialize_db_from_string(self, data):
+ """
+ Reloads the database with data from a string generated by
+ the serialize_db_to_string method.
+ """
+ data = StringIO(data)
+ for obj in serializers.deserialize("json", data, using=self.connection.alias):
+ obj.save()
+
def _get_test_db_name(self):
"""
Internal implementation - returns the name of the test DB that will be
View
6 django/test/runner.py
@@ -298,7 +298,11 @@ def setup_databases(verbosity, interactive, keepdb=False, **kwargs):
connection = connections[alias]
if test_db_name is None:
test_db_name = connection.creation.create_test_db(
- verbosity, autoclobber=not interactive, keepdb=keepdb)
+ verbosity,
+ autoclobber=not interactive,
+ keepdb=keepdb,
+ serialize=connection.settings_dict.get("TEST_SERIALIZE", True),
+ )
destroy = True
else:
connection.settings_dict['NAME'] = test_db_name
View
19 django/test/testcases.py
@@ -753,6 +753,12 @@ class TransactionTestCase(SimpleTestCase):
# Subclasses can define fixtures which will be automatically installed.
fixtures = None
+ # If transactions aren't available, Django will serialize the database
+ # contents into a fixture during setup and flush and reload them
+ # during teardown (as flush does not restore data from migrations).
+ # This can be slow; this flag allows enabling on a per-case basis.
+ serialized_rollback = False
+
def _pre_setup(self):
"""Performs any pre-test setup. This includes:
@@ -808,6 +814,17 @@ def _fixture_setup(self):
if self.reset_sequences:
self._reset_sequences(db_name)
+ # If we need to provide replica initial data from migrated apps,
+ # then do so.
+ if self.serialized_rollback and hasattr(connections[db_name], "_test_serialized_contents"):
+ if self.available_apps is not None:
+ apps.unset_available_apps()
+ connections[db_name].creation.deserialize_db_from_string(
+ connections[db_name]._test_serialized_contents
+ )
+ if self.available_apps is not None:
+ apps.set_available_apps(self.available_apps)
+
if self.fixtures:
# We have to use this slightly awkward syntax due to the fact
# that we're using *args and **kwargs together.
@@ -844,12 +861,14 @@ def _fixture_teardown(self):
# Allow TRUNCATE ... CASCADE and don't emit the post_migrate signal
# when flushing only a subset of the apps
for db_name in self._databases_names(include_mirrors=False):
+ # Flush the database
call_command('flush', verbosity=0, interactive=False,
database=db_name, skip_checks=True,
reset_sequences=False,
allow_cascade=self.available_apps is not None,
inhibit_post_migrate=self.available_apps is not None)
+
def assertQuerysetEqual(self, qs, values, transform=repr, ordered=True, msg=None):
items = six.moves.map(transform, qs)
if not ordered:
View
13 docs/ref/migration-operations.txt
@@ -199,8 +199,9 @@ model::
# We get the model from the versioned app registry;
# if we directly import it, it'll be the wrong version
Country = apps.get_model("myapp", "Country")
- Country.objects.create(name="USA", code="us")
- Country.objects.create(name="France", code="fr")
+ db_alias = schema_editor.connection.alias
+ Country.objects.create(name="USA", code="us", using=db_alias)
+ Country.objects.create(name="France", code="fr", using=db_alias)
class Migration(migrations.Migration):
@@ -236,6 +237,14 @@ Oracle). This should be safe, but may cause a crash if you attempt to use
the ``schema_editor`` provided on these backends; in this case, please
set ``atomic=False``.
+.. warning::
+
+ RunPython does not magically alter the connection of the models for you;
+ any model methods you call will go to the default database unless you
+ give them the current database alias (available from
+ ``schema_editor.connection.alias``, where ``schema_editor`` is the second
+ argument to your function).
+
SeparateDatabaseAndState
------------------------
View
18 docs/ref/settings.txt
@@ -2078,6 +2078,24 @@ Default: ``'django.test.runner.DiscoverRunner'``
The name of the class to use for starting the test suite. See
:ref:`other-testing-frameworks`.
+.. setting:: TEST_NON_SERIALIZED_APPS
+
+TEST_NON_SERIALIZED_APPS
+------------------------
+
+Default: ``[]``
+
+In order to restore the database state between tests for TransactionTestCases
+and database backends without transactions, Django will :ref:`serialize the
+contents of all apps with migrations <test-case-serialized-rollback>` when it
+starts the test run so it can then reload from that copy before tests that
+need it.
+
+This slows down the startup time of the test runner; if you have apps that
+you know don't need this feature, you can add their full names in here (e.g.
+``django.contrib.contenttypes``) to exclude them from this serialization
+process.
+
.. setting:: THOUSAND_SEPARATOR
THOUSAND_SEPARATOR
View
4 docs/releases/1.7.txt
@@ -63,6 +63,10 @@ but a few of the key features are:
* ``initial_data`` fixtures are no longer loaded for apps with migrations; if
you want to load initial data for an app, we suggest you do it in a migration.
+* Test rollback behaviour is different for apps with migrations; in particular,
+ Django will no longer emulate rollbacks on non-transactional databases or
+ inside ``TransactionTestCase`` :ref:`unless specifically asked <test-case-serialized-rollback>`.
+
App-loading refactor
~~~~~~~~~~~~~~~~~~~~
View
8 docs/topics/testing/advanced.txt
@@ -485,7 +485,7 @@ django.db.connection.creation
The creation module of the database backend also provides some utilities that
can be useful during testing.
-.. function:: create_test_db([verbosity=1, autoclobber=False, keepdb=False])
+.. function:: create_test_db([verbosity=1, autoclobber=False, keepdb=False, serialize=True])
Creates a new test database and runs ``migrate`` against it.
@@ -507,6 +507,12 @@ can be useful during testing.
a new database will be created, prompting the user to remove
the existing one, if present.
+ ``serialize`` determines if Django serializes the database into an
+ in-memory JSON string before running tests (used to restore the database
+ state between tests if you don't have transactions). You can set this to
+ False to significantly speed up creation time if you know you don't need
+ data persistance outside of test fixtures.
+
Returns the name of the test database that it created.
``create_test_db()`` has the side effect of modifying the value of
View
28 docs/topics/testing/overview.txt
@@ -234,6 +234,33 @@ the Django test runner reorders tests in the following way:
database by a given :class:`~django.test.TransactionTestCase` test, they
must be updated to be able to run independently.
+.. _test-case-serialized-rollback:
+
+Rollback emulation
+------------------
+
+Any initial data loaded in migrations will only be available in ``TestCase``
+tests and not in ``TransactionTestCase`` tests, and additionally only on
+backends where transactions are supported (the most important exception being
+MyISAM).
+
+Django can re-load that data for you on a per-testcase basis by
+setting the ``serialized_rollback`` option to ``True`` in the body of the
+``TestCase`` or ``TransactionTestCase``, but note that this will slow down
+that test suite by approximately 3x.
+
+Third-party apps or those developing against MyISAM will need to set this;
+in general, however, you should be developing your own projects against a
+transactional database and be using ``TestCase`` for most tests, and thus
+not need this setting.
+
+The initial serialization is usually very quick, but if you wish to exclude
+some apps from this process (and speed up test runs slightly), you may add
+those apps to :setting:`TEST_NON_SERIALIZED_APPS`.
+
+Apps without migrations are not affected; ``initial_data`` fixtures are
+reloaded as usual.
+
Other test conditions
---------------------
@@ -249,6 +276,7 @@ used. This behavior `may change`_ in the future.
.. _may change: https://code.djangoproject.com/ticket/11505
+
Understanding the test output
-----------------------------
View
14 docs/topics/testing/tools.txt
@@ -600,9 +600,17 @@ to test the effects of commit and rollback:
guarantees that the rollback at the end of the test restores the database to
its initial state.
- When running on a database that does not support rollback (e.g. MySQL with the
- MyISAM storage engine), ``TestCase`` falls back to initializing the database
- by truncating tables and reloading initial data.
+.. warning::
+
+ ``TestCase`` running on a database that does not support rollback (e.g. MySQL with the
+ MyISAM storage engine), and all instances of ``TransactionTestCase``, will
+ roll back at the end of the test by deleting all data from the test database
+ and reloading initial data for apps without migrations.
+
+ Apps with migrations :ref:`will not see their data reloaded <test-case-serialized-rollback>`;
+ if you need this functionality (for example, third-party apps should enable
+ this) you can set ``serialized_rollback = True`` inside the
+ ``TestCase`` body.
.. warning::
View
0  tests/migration_test_data_persistence/__init__.py
No changes.
View
34 tests/migration_test_data_persistence/migrations/0001_initial.py
@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+def add_book(apps, schema_editor):
+ apps.get_model("migration_test_data_persistence", "Book").objects.using(
+ schema_editor.connection.alias,
+ ).create(
+ title="I Love Django",
+ )
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='Book',
+ fields=[
+ ('id', models.AutoField(verbose_name='ID', primary_key=True, serialize=False, auto_created=True)),
+ ('title', models.CharField(max_length=100)),
+ ],
+ options={
+ },
+ bases=(models.Model,),
+ ),
+ migrations.RunPython(
+ add_book,
+ ),
+ ]
View
0  tests/migration_test_data_persistence/migrations/__init__.py
No changes.
View
5 tests/migration_test_data_persistence/models.py
@@ -0,0 +1,5 @@
+from django.db import models
+
+
+class Book(models.Model):
+ title = models.CharField(max_length=100)
View
33 tests/migration_test_data_persistence/tests.py
@@ -0,0 +1,33 @@
+from django.test import TransactionTestCase
+from .models import Book
+
+
+class MigrationDataPersistenceTestCase(TransactionTestCase):
+ """
+ Tests that data loaded in migrations is available if we set
+ serialized_rollback = True.
+ """
+
+ available_apps = ["migration_test_data_persistence"]
+ serialized_rollback = True
+
+ def test_persistence(self):
+ self.assertEqual(
+ Book.objects.count(),
+ 1,
+ )
+
+
+class MigrationDataNoPersistenceTestCase(TransactionTestCase):
+ """
+ Tests the failure case
+ """
+
+ available_apps = ["migration_test_data_persistence"]
+ serialized_rollback = False
+
+ def test_no_persistence(self):
+ self.assertEqual(
+ Book.objects.count(),
+ 0,
+ )
Please sign in to comment.
Something went wrong with that request. Please try again.