From 5354642830307094abaa3fd4ee229141faf150f2 Mon Sep 17 00:00:00 2001 From: lukeplant Date: Sat, 14 Jan 2012 17:26:32 +0000 Subject: [PATCH] Fixed #17530 - Fixture loading with deferred constraints broken for Postgres for fixtures from multiple dirs/files Thanks to claudep for the review. git-svn-id: http://code.djangoproject.com/svn/django/trunk@17372 bcc190cf-cafb-0310-a4f2-bffc1f526a37 --- django/core/management/commands/loaddata.py | 218 +++++++++--------- .../fixtures_1/forward_ref_1.json | 10 + .../fixtures_2/forward_ref_2.json | 9 + .../regressiontests/fixtures_regress/tests.py | 20 ++ 4 files changed, 150 insertions(+), 107 deletions(-) create mode 100644 tests/regressiontests/fixtures_regress/fixtures_1/forward_ref_1.json create mode 100644 tests/regressiontests/fixtures_regress/fixtures_2/forward_ref_2.json diff --git a/django/core/management/commands/loaddata.py b/django/core/management/commands/loaddata.py index d96421f73c..0c234fb746 100644 --- a/django/core/management/commands/loaddata.py +++ b/django/core/management/commands/loaddata.py @@ -104,87 +104,89 @@ def read(self): app_module_paths.append(app.__file__) app_fixtures = [os.path.join(os.path.dirname(path), 'fixtures') for path in app_module_paths] - for fixture_label in fixture_labels: - parts = fixture_label.split('.') - if len(parts) > 1 and parts[-1] in compression_types: - compression_formats = [parts[-1]] - parts = parts[:-1] - else: - compression_formats = compression_types.keys() + try: + with connection.constraint_checks_disabled(): + for fixture_label in fixture_labels: + parts = fixture_label.split('.') - if len(parts) == 1: - fixture_name = parts[0] - formats = serializers.get_public_serializer_formats() - else: - fixture_name, format = '.'.join(parts[:-1]), parts[-1] - if format in serializers.get_public_serializer_formats(): - formats = [format] - else: - formats = [] + if len(parts) > 1 and parts[-1] in compression_types: + compression_formats = [parts[-1]] + parts = parts[:-1] + else: + compression_formats = compression_types.keys() - if formats: - if verbosity >= 2: - self.stdout.write("Loading '%s' fixtures...\n" % fixture_name) - else: - self.stderr.write( - self.style.ERROR("Problem installing fixture '%s': %s is not a known serialization format.\n" % - (fixture_name, format))) - if commit: - transaction.rollback(using=using) - transaction.leave_transaction_management(using=using) - return - - if os.path.isabs(fixture_name): - fixture_dirs = [fixture_name] - else: - fixture_dirs = app_fixtures + list(settings.FIXTURE_DIRS) + [''] + if len(parts) == 1: + fixture_name = parts[0] + formats = serializers.get_public_serializer_formats() + else: + fixture_name, format = '.'.join(parts[:-1]), parts[-1] + if format in serializers.get_public_serializer_formats(): + formats = [format] + else: + formats = [] - for fixture_dir in fixture_dirs: - if verbosity >= 2: - self.stdout.write("Checking %s for fixtures...\n" % humanize(fixture_dir)) - - label_found = False - for combo in product([using, None], formats, compression_formats): - database, format, compression_format = combo - file_name = '.'.join( - p for p in [ - fixture_name, database, format, compression_format - ] - if p - ) - - if verbosity >= 3: - self.stdout.write("Trying %s for %s fixture '%s'...\n" % \ - (humanize(fixture_dir), file_name, fixture_name)) - full_path = os.path.join(fixture_dir, file_name) - open_method = compression_types[compression_format] - try: - fixture = open_method(full_path, 'r') - except IOError: + if formats: if verbosity >= 2: - self.stdout.write("No %s fixture '%s' in %s.\n" % \ - (format, fixture_name, humanize(fixture_dir))) + self.stdout.write("Loading '%s' fixtures...\n" % fixture_name) else: - if label_found: - fixture.close() - self.stderr.write(self.style.ERROR("Multiple fixtures named '%s' in %s. Aborting.\n" % - (fixture_name, humanize(fixture_dir)))) - if commit: - transaction.rollback(using=using) - transaction.leave_transaction_management(using=using) - return - else: - fixture_count += 1 - objects_in_fixture = 0 - loaded_objects_in_fixture = 0 - if verbosity >= 2: - self.stdout.write("Installing %s fixture '%s' from %s.\n" % \ - (format, fixture_name, humanize(fixture_dir))) + self.stderr.write( + self.style.ERROR("Problem installing fixture '%s': %s is not a known serialization format.\n" % + (fixture_name, format))) + if commit: + transaction.rollback(using=using) + transaction.leave_transaction_management(using=using) + return + + if os.path.isabs(fixture_name): + fixture_dirs = [fixture_name] + else: + fixture_dirs = app_fixtures + list(settings.FIXTURE_DIRS) + [''] + + for fixture_dir in fixture_dirs: + if verbosity >= 2: + self.stdout.write("Checking %s for fixtures...\n" % humanize(fixture_dir)) + + label_found = False + for combo in product([using, None], formats, compression_formats): + database, format, compression_format = combo + file_name = '.'.join( + p for p in [ + fixture_name, database, format, compression_format + ] + if p + ) + + if verbosity >= 3: + self.stdout.write("Trying %s for %s fixture '%s'...\n" % \ + (humanize(fixture_dir), file_name, fixture_name)) + full_path = os.path.join(fixture_dir, file_name) + open_method = compression_types[compression_format] try: - objects = serializers.deserialize(format, fixture, using=using) + fixture = open_method(full_path, 'r') + except IOError: + if verbosity >= 2: + self.stdout.write("No %s fixture '%s' in %s.\n" % \ + (format, fixture_name, humanize(fixture_dir))) + else: + try: + if label_found: + self.stderr.write(self.style.ERROR("Multiple fixtures named '%s' in %s. Aborting.\n" % + (fixture_name, humanize(fixture_dir)))) + if commit: + transaction.rollback(using=using) + transaction.leave_transaction_management(using=using) + return + + fixture_count += 1 + objects_in_fixture = 0 + loaded_objects_in_fixture = 0 + if verbosity >= 2: + self.stdout.write("Installing %s fixture '%s' from %s.\n" % \ + (format, fixture_name, humanize(fixture_dir))) + + objects = serializers.deserialize(format, fixture, using=using) - with connection.constraint_checks_disabled(): for obj in objects: objects_in_fixture += 1 if router.allow_syncdb(using, obj.object.__class__): @@ -201,41 +203,43 @@ def read(self): } raise e.__class__, e.__class__(msg), sys.exc_info()[2] - # Since we disabled constraint checks, we must manually check for - # any invalid keys that might have been added - table_names = [model._meta.db_table for model in models] - connection.check_constraints(table_names=table_names) - - loaded_object_count += loaded_objects_in_fixture - fixture_object_count += objects_in_fixture - label_found = True - except (SystemExit, KeyboardInterrupt): - raise - except Exception: - fixture.close() - if commit: - transaction.rollback(using=using) - transaction.leave_transaction_management(using=using) - if show_traceback: - traceback.print_exc() - else: + loaded_object_count += loaded_objects_in_fixture + fixture_object_count += objects_in_fixture + label_found = True + finally: + fixture.close() + + # If the fixture we loaded contains 0 objects, assume that an + # error was encountered during fixture loading. + if objects_in_fixture == 0: self.stderr.write( - self.style.ERROR("Problem installing fixture '%s': %s\n" % - (full_path, ''.join(traceback.format_exception(sys.exc_type, - sys.exc_value, sys.exc_traceback))))) - return - fixture.close() - - # If the fixture we loaded contains 0 objects, assume that an - # error was encountered during fixture loading. - if objects_in_fixture == 0: - self.stderr.write( - self.style.ERROR("No fixture data found for '%s'. (File format may be invalid.)\n" % - (fixture_name))) - if commit: - transaction.rollback(using=using) - transaction.leave_transaction_management(using=using) - return + self.style.ERROR("No fixture data found for '%s'. (File format may be invalid.)\n" % + (fixture_name))) + if commit: + transaction.rollback(using=using) + transaction.leave_transaction_management(using=using) + return + + # Since we disabled constraint checks, we must manually check for + # any invalid keys that might have been added + table_names = [model._meta.db_table for model in models] + connection.check_constraints(table_names=table_names) + + except (SystemExit, KeyboardInterrupt): + raise + except Exception: + if commit: + transaction.rollback(using=using) + transaction.leave_transaction_management(using=using) + if show_traceback: + traceback.print_exc() + else: + self.stderr.write( + self.style.ERROR("Problem installing fixture '%s': %s\n" % + (full_path, ''.join(traceback.format_exception(sys.exc_type, + sys.exc_value, sys.exc_traceback))))) + return + # If we found even one object in a fixture, we need to reset the # database sequences. diff --git a/tests/regressiontests/fixtures_regress/fixtures_1/forward_ref_1.json b/tests/regressiontests/fixtures_regress/fixtures_1/forward_ref_1.json new file mode 100644 index 0000000000..1a75037b48 --- /dev/null +++ b/tests/regressiontests/fixtures_regress/fixtures_1/forward_ref_1.json @@ -0,0 +1,10 @@ +[ + { + "pk": 1, + "model": "fixtures_regress.book", + "fields": { + "name": "Cryptonomicon", + "author": 4 + } + } +] diff --git a/tests/regressiontests/fixtures_regress/fixtures_2/forward_ref_2.json b/tests/regressiontests/fixtures_regress/fixtures_2/forward_ref_2.json new file mode 100644 index 0000000000..9cb63085a4 --- /dev/null +++ b/tests/regressiontests/fixtures_regress/fixtures_2/forward_ref_2.json @@ -0,0 +1,9 @@ +[ + { + "pk": "4", + "model": "fixtures_regress.person", + "fields": { + "name": "Neal Stephenson" + } + } +] diff --git a/tests/regressiontests/fixtures_regress/tests.py b/tests/regressiontests/fixtures_regress/tests.py index 5b8dc6659f..ed8b404788 100644 --- a/tests/regressiontests/fixtures_regress/tests.py +++ b/tests/regressiontests/fixtures_regress/tests.py @@ -16,6 +16,7 @@ from django.db.models import signals from django.test import (TestCase, TransactionTestCase, skipIfDBFeature, skipUnlessDBFeature) +from django.test.utils import override_settings from .models import (Animal, Stuff, Absolute, Parent, Child, Article, Widget, Store, Person, Book, NKChild, RefToNKChild, Circle1, Circle2, Circle3, @@ -390,6 +391,25 @@ def test_loaddata_raises_error_when_fixture_has_invalid_foreign_key(self): stderr.getvalue().startswith('Problem installing fixture') ) + _cur_dir = os.path.dirname(os.path.abspath(__file__)) + + @override_settings(FIXTURE_DIRS=[os.path.join(_cur_dir, 'fixtures_1'), + os.path.join(_cur_dir, 'fixtures_2')]) + def test_loaddata_forward_refs_split_fixtures(self): + """ + Regression for #17530 - should be able to cope with forward references + when the fixtures are not in the same files or directories. + """ + management.call_command( + 'loaddata', + 'forward_ref_1.json', + 'forward_ref_2.json', + verbosity=0, + commit=False + ) + self.assertEqual(Book.objects.all()[0].id, 1) + self.assertEqual(Person.objects.all()[0].id, 4) + def test_loaddata_no_fixture_specified(self): """ Regression for #7043 - Error is quickly reported when no fixtures is provided in the command line.