Move add select related logic to Mixin, add only, iterator and tests

.travis.yml

@@ -35,7 +35,7 @@ before_install:
 - pip install codecov
 install:
 - pip install -U pip wheel setuptools
-- pip install django-composite-foreignkey
+- pip install django-composite-foreignkey>=1.0.1
 - travis_retry pip install $DJANGO -e .
 script:
 - coverage run --rcfile=.coveragerc runtests.py

example/article/tests.py

@@ -255,7 +255,6 @@ def test_admin_delete_translation(self):
         self.assertEqual(200, resp.status_code)
         self.assertTemplateUsed(resp, 'admin/parler/deletion_not_allowed.html')
 
-    @expectedFailure
     def test_admin_delete_translation_unavailable(self):
         """
         To be fixed : when trying to delete the last language when a translation

parler/fields.py

@@ -13,10 +13,6 @@
 
 import django
 from django.forms.forms import pretty_name
-from parler.utils.i18n import get_language
-
-if (1, 8) <= django.VERSION < (2, 0):
-    from compositefk.fields import RawFieldValue, CompositeOneToOneField
 
 
 # TODO: inherit RelatedField?
@@ -166,39 +162,3 @@ def __set__(self, instance, value):
 
     def __delete__(self, instance):
         raise AttributeError("The 'language_code' attribute cannot be deleted!")
-
-
-class DONOTHING(object):
-    pass
-
-
-if (1, 8) <= django.VERSION < (2, 0):
-    class CompositeOneToOneVirtualField(CompositeOneToOneField):
-        """
-        Class to fix problem with creation repetitive migrations
-        """
-        def deconstruct(self):
-            name, path, args, kwargs = super(CompositeOneToOneVirtualField, self).deconstruct()
-            if 'to_fields' in kwargs:
-                kwargs['to_fields'] = {'master_id': None, 'language_code': None}  # hack: Need always the same dict
-            if "on_delete" in kwargs:
-                kwargs['on_delete'] = DONOTHING  # hack: Need always the same global object with __module__ attr
-            if "null_if_equal" in kwargs:
-                del kwargs['null_if_equal']
-            return name, path, args, kwargs
-
-
-    class RawActiveLangFieldValue(RawFieldValue):
-        """
-        Raw value with active language
-        """
-        def __init__(self):
-            super(RawActiveLangFieldValue, self).__init__(None)
-
-        @property
-        def value(self):
-            return get_language()
-
-        @value.setter
-        def value(self, value):
-            pass

parler/managers.py

@@ -9,7 +9,11 @@
 from django.utils import six
 from parler import appsettings
 from parler.utils import get_active_language_choices
-from parler.utils.fields import get_extra_related_translalation_paths
+from parler.utils.fields import get_extra_related_translation_paths
+
+
+if (1, 8) <= django.VERSION < (1, 9):
+    from django.db.models.query import ValuesListQuerySet, ValuesQuerySet
 
 
 class SelectRelatedTranslationsQuerySetMixin(object):
@@ -24,46 +28,164 @@ class SelectRelatedTranslationsQuerySetMixin(object):
     def select_related(self, *fields):
         extra_paths = []
         for field in fields:
-            extra_paths += get_extra_related_translalation_paths(self.model, field)
+            extra_paths += get_extra_related_translation_paths(self.model, field)
         if extra_paths:
             fields = tuple(set(extra_paths)) + fields
         return super(SelectRelatedTranslationsQuerySetMixin, self).select_related(*fields)
 
 
-class TranslatableQuerySet(QuerySet):
+class AutoAddSelectRelatedQuerySetMixin(object):
+    """
+    Mixin auto adds select related models from the list self.select_related_to_auto_add
+    if it possible: QuerySet not for update and not returns values/values_list
+
+    select_related is not compatible with values/values_list method and raise error in Django 1.8+ if it used
+    so we check if select_related is applicable for the queryset
+
+    Set related fields in your qs like:
+        class YourQuerySet(AutoAddSelectRelatedQuerySetMixIn, query.QuerySet):
+            select_related_fields_to_auto_add = {
+                'field1': ['related_model1__field2, 'related_model2__field3, ...],
+                'field2': ['related_model3__field4, 'related_model4__field5, ...]
+            }
+            ...
+
+    Can be used for translated and normal model querysets to automate adding select related of any needed models to qs
+    """
+    select_related_fields_to_auto_add = dict()  # type: Dict[str, List[str]]
+
+    if django.VERSION < (1, 8):
+        @property
+        def select_related_is_applicable(self):
+            return False
+
+    elif (1, 8) <= django.VERSION < (1, 9):
+        @property
+        def select_related_is_applicable(self):
+            if self.model._meta.proxy:
+                return False
+            return not isinstance(self, ValuesListQuerySet) and not isinstance(self, ValuesQuerySet)
+
+    elif django.VERSION >= (1, 9):
+        def __init__(self, *args, **kwargs):
+            super(AutoAddSelectRelatedQuerySetMixin, self).__init__(*args, **kwargs)
+            self._use_values = False  # Will use _use_values as a flag if values/values_list is used
+
+        def _values(self, *fields):
+            result = super(AutoAddSelectRelatedQuerySetMixin, self)._values(*fields)
+            result._use_values = True
+            return result
+
+        def _clone(self, **kwargs):
+            c = super(AutoAddSelectRelatedQuerySetMixin, self)._clone(**kwargs)
+            c._use_values = self._use_values
+            return c
+
+        @property
+        def select_related_is_applicable(self):
+            if self.model._meta.proxy:
+                return False
+            return not self._use_values
+
+    def _add_select_related(self):
+        """
+        Adds select related fields based on select_related_fields_to_auto_add structure in format Dict[str, List[str]]
+        If there are not used only/defer on queryset: query.deferred_loading = (None, False)
+        we count all select_related_fields_to_auto_add are selecting and add all related fields,
+        else add only subset of them as intersection with query deferred field set
+        """
+        existing, defer = self.query.deferred_loading
+
+        used_fields = set(self.select_related_fields_to_auto_add.keys())
+        related_fields = set()
+
+        if defer:
+            used_fields = used_fields.difference(existing)
+        elif existing:
+            used_fields = used_fields.intersection(existing)
+
+        for field, related_field_list in six.iteritems(self.select_related_fields_to_auto_add):
+            if field in used_fields:
+                related_fields.update(related_field_list)
+
+        if not defer and existing:
+            existing.update(related_fields)
+
+        self.query.add_select_related(related_fields)
+
+    def _fetch_all(self):
+        # Add select_related only once just before run db-query
+        if self.select_related_is_applicable and not self._for_write:
+            self._add_select_related()
+        super(AutoAddSelectRelatedQuerySetMixin, self)._fetch_all()
+
+    def iterator(self):
+        # Add select_related only once just before run db-query
+        if self.select_related_is_applicable and not self._for_write:
+            self._add_select_related()
+        return super(AutoAddSelectRelatedQuerySetMixin, self).iterator()
+
+
+class TranslatableQuerySet(AutoAddSelectRelatedQuerySetMixin, QuerySet):
     """
     An enhancement of the QuerySet which sets the objects language before they are returned.
 
     When using this class in combination with *django-polymorphic*, make sure this
     class is first in the chain of inherited classes.
 
-    When force_select_related_translations set to True in your classes it will always
-    adds active and default languages to select_related. It could break values_list method in django 1.9+
-    You can always add translated models to select_related manually. When you call it with rel_name e.g: 'translations'
-    it automatically adds active and default virtual composite FKs.
+    When force_select_related_translations set to True it will always adds translated models with active and
+    default languages by using virtual composite FKs.
+    In light version QS with force select related False you can always add translated models to select_related manually.
+    When you call select_related with translations rel_name e.g: 'translations' it automatically adds active and
+    default translated models to select_related.
     """
 
-    force_select_related_translations = False
+    force_select_related_translations = True
 
     def __init__(self, *args, **kwargs):
         super(TranslatableQuerySet, self).__init__(*args, **kwargs)
         self._language = None
+        if not self.force_select_related_translations:
+            return
+        fields_dict = self.select_related_fields_to_auto_add.copy()
+        for extension in self.model._parler_meta:
+            fields_dict[extension.rel_name_active] = [extension.rel_name_active]
+            fields_dict[extension.rel_name_default] = [extension.rel_name_default]
+        self.select_related_fields_to_auto_add = fields_dict
 
     def select_related(self, *fields):
+        """
+        Replaces main field refer to translations ('translations') with 'translations_active' and 'translations_default'
+        """
         fields_to_add = set()
-        fields_to_exclude = set([None])  # if rel_name_active, rel_name_default is None
+        fields_to_exclude = set()
         for extension in self.model._parler_meta:
-            if extension.rel_name in fields:
+            select_related_translations_fields = extension.get_select_related_translations_fields()
+            fields_to_search = set(select_related_translations_fields + [extension.rel_name])
+            if fields_to_search.intersection(fields):
                 fields_to_exclude.add(extension.rel_name)  # Can not select related OneToMany field
-                fields_to_add.add(extension.rel_name_active)
-                fields_to_add.add(extension.rel_name_default)
-            if extension.rel_name_active in fields:
-                fields_to_add.add(extension.rel_name_default)
-            if extension.rel_name_default in fields:
-                fields_to_add.add(extension.rel_name_active)
+                fields_to_add.update(select_related_translations_fields)
         fields = set(fields).union(fields_to_add).difference(fields_to_exclude)
         return super(TranslatableQuerySet, self).select_related(*tuple(fields))
 
+    def only(self, *fields):
+        """
+        Replaces translated fields with 'translations_active' and 'translations_default'
+        pretending they are in original model so we can use .only
+        for translated fields as usual: .objects.only('some_translated_field')
+        """
+        fields_to_add = set()
+        fields_to_exclude = set()
+        for extension in self.model._parler_meta:
+            select_related_translations_fields = extension.get_select_related_translations_fields()
+            # List fields to replace with select_related_translations_fields
+            fields_to_search = set(extension.get_translated_fields() + [extension.rel_name])
+            if fields_to_search.intersection(fields):
+                fields_to_exclude.update(fields_to_search)
+                fields_to_add.update(select_related_translations_fields)
+        fields = set(fields).union(fields_to_add).difference(fields_to_exclude)
+        return super(TranslatableQuerySet, self).only(*tuple(fields))
+
     def _clone(self, klass=None, setup=False, **kw):
         if django.VERSION < (1, 9):
             kw['klass'] = klass
@@ -79,47 +201,7 @@ def create(self, **kwargs):
             kwargs['_current_language'] = self._language
         return super(TranslatableQuerySet, self).create(**kwargs)
 
-    def _add_active_default_select_related(self):
-        existing, defer = self.query.deferred_loading
-        related_to_add = set()
-        for extension in self.model._parler_meta:
-            if not extension.rel_name:
-                continue
-            if extension.rel_name_active:
-                related_to_add.add(extension.rel_name_active)
-            if extension.rel_name_default:
-                related_to_add.add(extension.rel_name_default)
-        if defer:
-            related_to_add = related_to_add.difference(existing)
-        elif existing:
-            related_to_add = related_to_add.intersection(existing)
-        self.query.add_select_related(related_to_add)
-
-    @property
-    def select_related_not_applicable(self):
-        # type: () -> Union[bool, None]
-        """
-        Returns is select_related not applicable for current qs.
-        Currently determine only for django ver 1.8, for others returns None
-        """
-        result = None
-        if self.model._meta.proxy:
-            return True
-
-        if (1, 7) < django.VERSION < (1, 9):
-            ValuesListQuerySet = getattr(django.db.models.query, 'ValuesListQuerySet')
-            result = isinstance(self, ValuesListQuerySet)
-
-        return result
-
     def _fetch_all(self):
-        # For django ver > 1.8 when values_list method is used
-        #  _iterable_class (FlatValuesListIterable, ValuesListIterable) is known only in iteration stage not here yet
-        # TODO: figure out how to determine non qs methods or
-        #       place _add_active_default_select_related in some other place
-        if self.force_select_related_translations and not self.select_related_not_applicable:
-            self._add_active_default_select_related()
-
         # Make sure the current language is assigned when Django fetches the data.
         # This low-level method is overwritten as that works better across Django versions.
         # Alternatives include:
@@ -263,12 +345,39 @@ def active_translations(self, language_code=None, **translated_fields):
         return self.all().active_translations(language_code, **translated_fields)
 
 
-class TranslatableAutoSelectRelatedQuerySet(TranslatableQuerySet):
-    force_select_related_translations = True
+class LightTranslatableQuerySet(TranslatableQuerySet):
+    force_select_related_translations = False
+
+
+class LightTranslatableManager(TranslatableManager):
+    """
+    Translatable manager does not auto add select related translation models
+    """
+    queryset_class = LightTranslatableQuerySet
+
 
+class DeepTranslatableQuerySet(SelectRelatedTranslationsQuerySetMixin, TranslatableQuerySet):
+    pass
 
-class TranslatableAutoSelectRelatedManager(TranslatableManager):
-    queryset_class = TranslatableAutoSelectRelatedQuerySet
+
+class DeepTranslatableManager(TranslatableManager):
+    """
+    Translatable manager does auto add select related translation models (for active and default languages)
+    for current model and all translatable models used in select_related method call
+    """
+    queryset_class = DeepTranslatableQuerySet
+
+
+class AutoAddTranslationsQuerySet(SelectRelatedTranslationsQuerySetMixin, models.query.QuerySet):
+    pass
+
+
+class AutoAddTranslationsManager(models.Manager.from_queryset(AutoAddTranslationsQuerySet)):
+    """
+    Manager does auto add select related translation models (for active and default languages)
+    for all translatable models used in select_related method call
+    """
+    queryset_class = AutoAddTranslationsQuerySet
 
 
 # Export the names in django-hvad style too: