From 8d080d40cf671a9a0fa02182e8d14113dacf2ac0 Mon Sep 17 00:00:00 2001 From: Lubos Matl Date: Wed, 23 Dec 2020 20:32:23 +0100 Subject: [PATCH] Support django 3 --- .travis.yml | 9 +- chamber/importers/__init__.py | 13 +-- chamber/models/__init__.py | 104 +++++++++++------- chamber/multidomains/domain.py | 9 +- chamber/multidomains/urlresolvers.py | 6 +- chamber/utils/migrations/fixtures.py | 21 ++-- .../dj/apps/test_chamber/tests/importers.py | 2 +- .../test_chamber/tests/models/__init__.py | 47 +++++++- .../dj/apps/test_chamber/tests/shortcuts.py | 15 +++ example/requirements.txt | 15 +-- setup.py | 3 +- 11 files changed, 162 insertions(+), 82 deletions(-) diff --git a/.travis.yml b/.travis.yml index d8a6712..f9e51d1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,14 @@ language: python python: - - "3.5" - "3.6" + - "3.7" + - "3.8" + - "3.9" env: - - DJANGO_VERSION=1.11 - - DJANGO_VERSION=2.0 + - DJANGO_VERSION=2.2 + - DJANGO_VERSION=3.0 + - DJANGO_VERSION=3.1 # command to install dependencies install: diff --git a/chamber/importers/__init__.py b/chamber/importers/__init__.py index fefa0dd..6e81ac3 100644 --- a/chamber/importers/__init__.py +++ b/chamber/importers/__init__.py @@ -1,17 +1,13 @@ import csv import io +from itertools import zip_longest + from django.conf import settings import pyprind -try: - from itertools import zip_longest -except ImportError: - from itertools import izip_longest as zip_longest - - def simple_count(file): lines = 0 for _ in file: @@ -127,13 +123,16 @@ def import_rows(self, reader, row_count=0): self._post_batch_create(self.get_batch_size(), row_count) del batch[:] if any(row): # Skip blank lines - batch.append(self.model_class(**self.get_fields_dict(row))) + batch.append(self.create_instance(row)) created += self.create_batch(batch) self._post_batch_create(len(batch), row_count) self._post_import_rows(created) return created + def create_instance(self, row): + return self.model_class(**self.get_fields_dict(row)) + def get_delete_existing_objects(self): return self.delete_existing_objects diff --git a/chamber/models/__init__.py b/chamber/models/__init__.py index 6a6cf8a..05ddc71 100644 --- a/chamber/models/__init__.py +++ b/chamber/models/__init__.py @@ -1,7 +1,5 @@ import collections -from itertools import chain - from distutils.version import StrictVersion import django @@ -60,11 +58,24 @@ def __bool__(self): Unknown = UnknownSingleton() +@singleton +class DeferredSingleton: + + def __repr__(self): + return 'deferred' + + def __bool__(self): + return False + + +Deferred = DeferredSingleton() + + def unknown_model_fields_to_dict(instance, fields=None, exclude=None): return { field.name: Unknown - for field in chain(instance._meta.concrete_fields, instance._meta.many_to_many) # pylint: disable=W0212 + for field in instance._meta.concrete_fields # pylint: disable=W0212 if not should_exclude_field(field, fields, exclude) } @@ -73,10 +84,9 @@ def model_to_dict(instance, fields=None, exclude=None): """ The same implementation as django model_to_dict but editable fields are allowed """ - return { field.name: field_to_dict(field, instance) - for field in chain(instance._meta.concrete_fields, instance._meta.many_to_many) # pylint: disable=W0212 + for field in instance._meta.concrete_fields # pylint: disable=W0212 if not should_exclude_field(field, fields, exclude) } @@ -94,7 +104,7 @@ def __init__(self, initial_dict): @property def initial_values(self): - return self._initial_dict + return self._initial_dict.copy() @property def current_values(self): @@ -134,9 +144,6 @@ def has_key(self, k): def has_any_key(self, *keys): return bool(set(self.keys()) & set(keys)) - def update(self, *args, **kwargs): - raise AttributeError('Object is readonly') - def keys(self): return self.diff.keys() @@ -169,27 +176,41 @@ class DynamicChangedFields(ChangedFields): def __init__(self, instance): super().__init__( - self._get_unknown_dict(instance) if instance.is_adding else self._get_instance_dict(instance) + self._get_unknown_dict(instance) ) self.instance = instance def _get_unknown_dict(self, instance): - return unknown_model_fields_to_dict( - instance, fields=(field.name for field in instance._meta.fields) - ) - - def _get_instance_dict(self, instance): - return model_to_dict( - instance, fields=(field.name for field in instance._meta.fields) - ) + return unknown_model_fields_to_dict(instance) @property def current_values(self): - return self._get_instance_dict(self.instance) + deferred_values = { + field_name: value for field_name, value in self._initial_dict.items() + if field_name in self.instance.get_deferred_fields() + } + current_values = model_to_dict( + self.instance, + exclude=set(deferred_values.keys()) + ) + current_values.update(deferred_values) + return current_values def get_static_changes(self): return StaticChangedFields(self.initial_values, self.current_values) + def from_db(self, fields=None): + if fields is None: + fields = {field_name for field_name, value in self._initial_dict.items() if value is not Deferred} + + self._initial_dict.update( + model_to_dict(self.instance, fields=set(fields)) + ) + + for field_name, value in self._initial_dict.items(): + if value is Unknown: + self._initial_dict[field_name] = Deferred + class StaticChangedFields(ChangedFields): """ @@ -202,7 +223,7 @@ def __init__(self, initial_dict, current_dict): @property def current_values(self): - return self._current_dict + return self._current_dict.copy() class ComparableModelMixin: @@ -315,7 +336,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.is_adding = True self.is_changing = False - self.changed_fields = DynamicChangedFields(self) + self._changed_fields = DynamicChangedFields(self) self.post_save = Signal(self) class Meta: @@ -329,16 +350,24 @@ def from_db(cls, db, field_names, values): new = super().from_db(db, field_names, values) new.is_adding = False new.is_changing = True - new.changed_fields = DynamicChangedFields(new) + updating_fields = [ + f.name for f in cls._meta.concrete_fields + if len(values) == len(cls._meta.concrete_fields) or f.attname in field_names + ] + new._changed_fields.from_db(fields=updating_fields) return new @property def has_changed(self): - return bool(self.changed_fields) + return bool(self._changed_fields) + + @property + def changed_fields(self): + return self._changed_fields.get_static_changes() @property def initial_values(self): - return self.changed_fields.initial_values + return self._changed_fields.initial_values def full_clean(self, exclude=None, *args, **kwargs): errors = {} @@ -420,24 +449,24 @@ def _save(self, update_only_changed_fields=False, is_cleaned_pre_save=None, is_c kwargs.update(self._get_save_extra_kwargs()) self._call_pre_save( - changed=self.is_changing, changed_fields=self.changed_fields.get_static_changes(), *args, **kwargs + changed=self.is_changing, changed_fields=self.changed_fields, *args, **kwargs ) if is_cleaned_pre_save: self._clean_pre_save(*args, **kwargs) dispatcher_pre_save.send( sender=origin, instance=self, changed=self.is_changing, - changed_fields=self.changed_fields.get_static_changes(), + changed_fields=self.changed_fields, *args, **kwargs ) if not update_fields and update_only_changed_fields: - update_fields = list(self.changed_fields.keys()) + ['changed_at'] + update_fields = list(self._changed_fields.keys()) + ['changed_at'] # remove primary key from updating fields if self._meta.pk.name in update_fields: update_fields.remove(self._meta.pk.name) # Changed fields must be cached before save, for post_save and signal purposes - post_save_changed_fields = self.changed_fields.get_static_changes() + post_save_changed_fields = self.changed_fields post_save_is_changing = self.is_changing self.save_simple(force_insert=force_insert, force_update=force_update, using=using, update_fields=update_fields) @@ -468,7 +497,7 @@ def save_simple(self, *args, **kwargs): super().save(*args, **kwargs) self.is_adding = False self.is_changing = True - self.changed_fields = DynamicChangedFields(self) + self._changed_fields.from_db() def save(self, update_only_changed_fields=False, *args, **kwargs): if self._smart_meta.is_save_atomic: @@ -510,22 +539,17 @@ def delete(self, *args, **kwargs): else: self._delete(*args, **kwargs) - def refresh_from_db(self, *args, **kwargs): - super().refresh_from_db(*args, **kwargs) + def refresh_from_db(self, using=None, fields=None): + super().refresh_from_db(using=using, fields=fields) for key, value in self.__class__.__dict__.items(): if isinstance(value, cached_property): self.__dict__.pop(key, None) self.is_adding = False self.is_changing = True - self.changed_fields = DynamicChangedFields(self) - - if StrictVersion(get_main_version()) < StrictVersion('2.0'): - for field in [f for f in self._meta.get_fields() if f.is_relation]: - # For Generic relation related model is None - # https://docs.djangoproject.com/en/2.1/ref/models/meta/#migrating-from-the-old-api - cache_key = field.get_cache_name() if field.related_model else field.cache_attr - if cache_key in self.__dict__: - del self.__dict__[cache_key] + + self._changed_fields.from_db(fields={ + f.name for f in self._meta.concrete_fields if not fields or f.attname in fields or f.name in fields + }) return self diff --git a/chamber/multidomains/domain.py b/chamber/multidomains/domain.py index ab870de..ecdec01 100644 --- a/chamber/multidomains/domain.py +++ b/chamber/multidomains/domain.py @@ -1,5 +1,6 @@ from urllib.parse import urlparse +from django.apps import apps from django.core.exceptions import ImproperlyConfigured @@ -38,13 +39,7 @@ def url(self): @property def user_class(self): - try: - from django.apps import apps - get_model = apps.get_model - except ImportError: - from django.db.models.loading import get_model - - return get_model(*self.user_model.split('.', 1)) + return apps.get_model(*self.user_model.split('.', 1)) def get_domain(site_id): diff --git a/chamber/multidomains/urlresolvers.py b/chamber/multidomains/urlresolvers.py index 6d29b74..a095b22 100644 --- a/chamber/multidomains/urlresolvers.py +++ b/chamber/multidomains/urlresolvers.py @@ -1,11 +1,7 @@ from urllib.parse import urlencode from django.conf import settings - -try: - from django.core.urlresolvers import reverse as django_reverse -except ImportError: - from django.urls import reverse as django_reverse +from django.urls import reverse as django_reverse diff --git a/chamber/utils/migrations/fixtures.py b/chamber/utils/migrations/fixtures.py index 568804e..75153c8 100644 --- a/chamber/utils/migrations/fixtures.py +++ b/chamber/utils/migrations/fixtures.py @@ -2,8 +2,11 @@ from io import StringIO + +from django.db import DEFAULT_DB_ALIAS from django.core.management import call_command -from django.core.serializers import base, python +from django.core.serializers import python, base +from django.core.management.commands import loaddata class MigrationLoadFixture: @@ -26,9 +29,13 @@ def _get_model(model_identifier): raise base.DeserializationError("Invalid model identifier: '%s'" % model_identifier) get_model_tmp = python._get_model # pylint: disable=W0212 - python._get_model = _get_model - file = os.path.join(self.fixture_dir, self.fixture_filename) - if not os.path.isfile(file): - raise IOError('File "%s" does not exists' % file) - call_command('loaddata', file, stdout=StringIO()) - python._get_model = get_model_tmp # pylint: disable=W0212 + try: + python._get_model = _get_model + file = os.path.join(self.fixture_dir, self.fixture_filename) + if not os.path.isfile(file): + raise IOError('File "%s" does not exists' % file) + loaddata.Command().handle( + file, ignore=True, database=DEFAULT_DB_ALIAS, app_label=None, verbosity=0, exclude=[], format='json' + ) + finally: + python._get_model = get_model_tmp # pylint: disable=W0212 diff --git a/example/dj/apps/test_chamber/tests/importers.py b/example/dj/apps/test_chamber/tests/importers.py index ef3f1fc..9624514 100644 --- a/example/dj/apps/test_chamber/tests/importers.py +++ b/example/dj/apps/test_chamber/tests/importers.py @@ -1,4 +1,4 @@ -from six import StringIO +from io import StringIO from django.core.management import call_command from django.test import TestCase diff --git a/example/dj/apps/test_chamber/tests/models/__init__.py b/example/dj/apps/test_chamber/tests/models/__init__.py index 6d2695d..bbfef3e 100644 --- a/example/dj/apps/test_chamber/tests/models/__init__.py +++ b/example/dj/apps/test_chamber/tests/models/__init__.py @@ -1,11 +1,12 @@ from datetime import timedelta +from django.db import OperationalError from django.core.exceptions import ValidationError from django.test import TransactionTestCase from django.utils import timezone from chamber.exceptions import PersistenceException -from chamber.models import DynamicChangedFields, Comparator, Unknown +from chamber.models import DynamicChangedFields, Comparator, Unknown, Deferred from germanium.tools import assert_equal, assert_false, assert_raises, assert_true # pylint: disable=E0401 @@ -90,14 +91,48 @@ def test_smart_model_initial_values_should_be_unknown_for_not_saved_instance(sel assert_true(obj.is_changing) assert_true(all(v is not Unknown for v in obj.initial_values.values())) + assert_equal(str(Unknown), 'unknown') + + def test_smart_model_initial_values_should_be_deferred_for_partly_loaded_instance(self): + obj = DiffModel.objects.only('name').get( + pk=DiffModel.objects.create(name='test', datetime=timezone.now(), number=2).pk + ) + + assert_false(obj.has_changed) + assert_false(obj.changed_fields) + assert_false(obj.is_adding) + assert_true(obj.is_changing) + assert_true(all(v is Deferred for k, v in obj.initial_values.items() if k not in {'id', 'name'})) + assert_true(all(not bool(v) for k, v in obj.initial_values.items() if k not in {'id', 'name'})) + + assert_equal(obj.number, 2) + assert_false(obj.has_changed) + assert_false(obj.changed_fields) + assert_equal(obj.initial_values['number'], 2) + + obj.datetime = timezone.now() + assert_equal(obj.initial_values['datetime'], Deferred) + assert_true(obj.changed_fields) + assert_equal(obj.changed_fields.keys(), {'datetime'}) + assert_equal(str(Deferred), 'deferred') + def test_smart_model_changed_fields(self): obj = TestProxySmartModel.objects.create(name='a') changed_fields = DynamicChangedFields(obj) + assert_equal(len(changed_fields), 4) + changed_fields.from_db() assert_equal(len(changed_fields), 0) obj.name = 'b' assert_equal(len(changed_fields), 1) assert_equal(changed_fields['name'].initial, 'a') assert_equal(changed_fields['name'].current, 'b') + assert_equal(changed_fields.changed_values, {'name': 'b'}) + assert_equal(str(changed_fields), "{'name': ValueChange(initial='a', current='b')}") + assert_true(changed_fields.has_key('name')) + assert_false(changed_fields.has_key('changed_at')) + assert_equal(list(changed_fields.values()), [changed_fields['name']]) + assert_equal(changed_fields.keys(), {'name'}) + static_changed_fields = changed_fields.get_static_changes() obj.save() @@ -115,6 +150,7 @@ def test_smart_model_changed_fields(self): assert_raises(AttributeError, changed_fields.__delitem__, 'name') assert_raises(AttributeError, changed_fields.clear) assert_raises(AttributeError, changed_fields.pop, 'name') + assert_raises(AttributeError, changed_fields.__setitem__, 'name', 'value') obj.name = 'b' @@ -338,3 +374,12 @@ def test_smart_model_str_method(self): unstored_obj = TestSmartModel(name='1') assert_equal(str(unstored_obj), 'test smart model #None') + + def test_smart_model_get_locked_instance(self): + not_saved_obj = TestSmartModel() + + assert_raises(OperationalError, not_saved_obj.get_locked_instance) + + obj = TestSmartModel.objects.create(name='1') + assert_equal(obj, obj.get_locked_instance()) + diff --git a/example/dj/apps/test_chamber/tests/shortcuts.py b/example/dj/apps/test_chamber/tests/shortcuts.py index 74d3121..5075beb 100644 --- a/example/dj/apps/test_chamber/tests/shortcuts.py +++ b/example/dj/apps/test_chamber/tests/shortcuts.py @@ -82,6 +82,12 @@ def test_change_and_save_with_update_only_changed_fields_should_change_only_defi assert_equal(obj.name, 'test2') assert_equal(obj.number, 3) + def test_model_change(self): + obj = DiffModel.objects.create(name='test', datetime=timezone.now(), number=2) + DiffModel.objects.filter(pk=obj.pk).update(name='test2') + obj.change(number=3) + assert_equal(obj.number, 3) + def test_bulk_change_and_save(self): obj1 = ShortcutsModel.objects.create(name='test1', datetime=timezone.now(), number=1) obj2 = ShortcutsModel.objects.create(name='test2', datetime=timezone.now(), number=2) @@ -106,3 +112,12 @@ def test_bulk_change_and_bulk_save(self): bulk_save([obj1, obj2]) assert_equal(ShortcutsModel.objects.first().name, 'modified') # instance is changed but saved to DB assert_equal(ShortcutsModel.objects.last().name, 'modified') # instance is changed but saved to DB + + def test_queryset_change_and_save(self): + obj1 = DiffModel.objects.create(name='test', datetime=timezone.now(), number=2) + obj2 = DiffModel.objects.create(name='test', datetime=timezone.now(), number=2) + DiffModel.objects.all().change_and_save(name='modified') + obj1.refresh_from_db() + obj2.refresh_from_db() + assert_equal(obj1.name, 'modified') + assert_equal(obj2.name, 'modified') diff --git a/example/requirements.txt b/example/requirements.txt index 59245d4..730f274 100644 --- a/example/requirements.txt +++ b/example/requirements.txt @@ -1,13 +1,10 @@ -Django==2.0 +Django==3.1 diff-match-patch==20110725.1 django-germanium==2.1.0 six==1.10.0 -coverage==4.0.2 -pyprind==2.9.9 -coveralls==1.1 -filemagic==1.6 -unidecode==0.4.20 -pillow==6.2.0 -django-storages==1.7.1 -boto3==1.9.209 +coverage==5.3.1 +coveralls==2.2.0 +pillow==8.0.1 +boto3==1.16.47 +django-storages==1.11.1 -e ../ diff --git a/setup.py b/setup.py index 4e60cf2..1238265 100644 --- a/setup.py +++ b/setup.py @@ -24,10 +24,9 @@ 'Framework :: Django', ], install_requires=[ - 'Django>=1.11', + 'Django>=2.2', 'Unidecode>=1.1.1', 'pyprind>=2.11.2', - 'six>=1.12.0', 'filemagic>=1.6', ], extras_require={