Fixed #21169 -- Reworked RelatedManager methods use default filtering

The `remove()` and `clear()` methods of the related managers created by
`ForeignKey`, `GenericForeignKey`, and `ManyToManyField` suffered from a
number of issues. Some operations ran multiple data modifying queries without
wrapping them in a transaction, and some operations didn't respect default
filtering when it was present (i.e. when the default manager on the related
model implemented a custom `get_queryset()`).

Fixing the issues introduced some backward incompatible changes:

- The implementation of `remove()` for `ForeignKey` related managers changed
  from a series of `Model.save()` calls to a single `QuerySet.update()` call.
  The change means that `pre_save` and `post_save` signals aren't called anymore.

- The `remove()` and `clear()` methods for `GenericForeignKey` related
  managers now perform bulk delete so `Model.delete()` isn't called anymore.

- The `remove()` and `clear()` methods for `ManyToManyField` related
  managers perform nested queries when filtering is involved, which may
  or may not be an issue depending on the database and the data itself.

Refs. #3871, #21174.

Thanks Anssi Kääriäinen and Tim Graham for the reviews.

django/db/models/fields/related.py

@@ -2,7 +2,7 @@
 
 from django.db import connection, connections, router, transaction
 from django.db.backends import utils
-from django.db.models import signals
+from django.db.models import signals, Q
 from django.db.models.fields import (AutoField, Field, IntegerField,
     PositiveIntegerField, PositiveSmallIntegerField, FieldDoesNotExist)
 from django.db.models.related import RelatedObject, PathInfo
@@ -464,14 +464,21 @@ def get_or_create(self, **kwargs):
         # remove() and clear() are only provided if the ForeignKey can have a value of null.
         if rel_field.null:
             def remove(self, *objs):
+                # If there aren't any objects, there is nothing to do.
+                if not objs:
+                    return
+
                 val = rel_field.get_foreign_related_value(self.instance)
+
+                old_ids = set()
                 for obj in objs:
                     # Is obj actually part of this descriptor set?
                     if rel_field.get_local_related_value(obj) == val:
-                        setattr(obj, rel_field.name, None)
-                        obj.save()
+                        old_ids.add(obj.pk)
                     else:
                         raise rel_field.rel.to.DoesNotExist("%r is not related to %r." % (obj, self.instance))
+
+                self.filter(pk__in=old_ids).update(**{rel_field.name: None})
             remove.alters_data = True
 
             def clear(self):
@@ -536,6 +543,7 @@ def __init__(self, model=None, query_field_name=None, instance=None, symmetrical
             self.instance = instance
             self.symmetrical = symmetrical
             self.source_field = source_field
+            self.target_field = through._meta.get_field(target_field_name)
             self.source_field_name = source_field_name
             self.target_field_name = target_field_name
             self.reverse = reverse
@@ -572,6 +580,19 @@ def __call__(self, **kwargs):
             )
         do_not_call_in_templates = True
 
+        def _build_clear_filters(self, qs):
+            filters = Q(**{
+                self.source_field_name: self.related_val,
+                '%s__in' % self.target_field_name: qs
+            })
+
+            if self.symmetrical:
+                filters |= Q(**{
+                    self.target_field_name: self.related_val,
+                    '%s__in' % self.source_field_name: qs
+                })
+            return filters
+
         def get_queryset(self):
             try:
                 return self.instance._prefetched_objects_cache[self.prefetch_cache_name]
@@ -625,18 +646,20 @@ def add(self, *objs):
 
             def remove(self, *objs):
                 self._remove_items(self.source_field_name, self.target_field_name, *objs)
-
-                # If this is a symmetrical m2m relation to self, remove the mirror entry in the m2m table
-                if self.symmetrical:
-                    self._remove_items(self.target_field_name, self.source_field_name, *objs)
             remove.alters_data = True
 
         def clear(self):
-            self._clear_items(self.source_field_name)
+            db = router.db_for_write(self.through, instance=self.instance)
 
-            # If this is a symmetrical m2m relation to self, clear the mirror entry in the m2m table
-            if self.symmetrical:
-                self._clear_items(self.target_field_name)
+            signals.m2m_changed.send(sender=self.through, action="pre_clear",
+                instance=self.instance, reverse=self.reverse,
+                model=self.model, pk_set=None, using=db)
+            filters = self._build_clear_filters(self.using(db))
+            self.through._default_manager.using(db).filter(filters).delete()
+
+            signals.m2m_changed.send(sender=self.through, action="post_clear",
+                instance=self.instance, reverse=self.reverse,
+                model=self.model, pk_set=None, using=db)
         clear.alters_data = True
 
         def create(self, **kwargs):
@@ -722,55 +745,33 @@ def _remove_items(self, source_field_name, target_field_name, *objs):
             # *objs - objects to remove
 
             # If there aren't any objects, there is nothing to do.
-            if objs:
-                # Check that all the objects are of the right type
-                old_ids = set()
-                for obj in objs:
-                    if isinstance(obj, self.model):
-                        fk_val = self.through._meta.get_field(
-                            target_field_name).get_foreign_related_value(obj)[0]
-                        old_ids.add(fk_val)
-                    else:
-                        old_ids.add(obj)
-                # Work out what DB we're operating on
-                db = router.db_for_write(self.through, instance=self.instance)
-                # Send a signal to the other end if need be.
-                if self.reverse or source_field_name == self.source_field_name:
-                    # Don't send the signal when we are deleting the
-                    # duplicate data row for symmetrical reverse entries.
-                    signals.m2m_changed.send(sender=self.through, action="pre_remove",
-                        instance=self.instance, reverse=self.reverse,
-                        model=self.model, pk_set=old_ids, using=db)
-                # Remove the specified objects from the join table
-                self.through._default_manager.using(db).filter(**{
-                    source_field_name: self.related_val[0],
-                    '%s__in' % target_field_name: old_ids
-                }).delete()
-                if self.reverse or source_field_name == self.source_field_name:
-                    # Don't send the signal when we are deleting the
-                    # duplicate data row for symmetrical reverse entries.
-                    signals.m2m_changed.send(sender=self.through, action="post_remove",
-                        instance=self.instance, reverse=self.reverse,
-                        model=self.model, pk_set=old_ids, using=db)
+            if not objs:
+                return
+
+            # Check that all the objects are of the right type
+            old_ids = set()
+            for obj in objs:
+                if isinstance(obj, self.model):
+                    fk_val = self.target_field.get_foreign_related_value(obj)[0]
+                    old_ids.add(fk_val)
+                else:
+                    old_ids.add(obj)
 
-        def _clear_items(self, source_field_name):
             db = router.db_for_write(self.through, instance=self.instance)
-            # source_field_name: the PK colname in join table for the source object
-            if self.reverse or source_field_name == self.source_field_name:
-                # Don't send the signal when we are clearing the
-                # duplicate data rows for symmetrical reverse entries.
-                signals.m2m_changed.send(sender=self.through, action="pre_clear",
-                    instance=self.instance, reverse=self.reverse,
-                    model=self.model, pk_set=None, using=db)
-            self.through._default_manager.using(db).filter(**{
-                source_field_name: self.related_val
-            }).delete()
-            if self.reverse or source_field_name == self.source_field_name:
-                # Don't send the signal when we are clearing the
-                # duplicate data rows for symmetrical reverse entries.
-                signals.m2m_changed.send(sender=self.through, action="post_clear",
-                    instance=self.instance, reverse=self.reverse,
-                    model=self.model, pk_set=None, using=db)
+
+            # Send a signal to the other end if need be.
+            signals.m2m_changed.send(sender=self.through, action="pre_remove",
+                instance=self.instance, reverse=self.reverse,
+                model=self.model, pk_set=old_ids, using=db)
+
+            old_vals_qs = self.using(db).filter(**{
+                '%s__in' % self.target_field.related_field.attname: old_ids})
+            filters = self._build_clear_filters(old_vals_qs)
+            self.through._default_manager.using(db).filter(filters).delete()
+
+            signals.m2m_changed.send(sender=self.through, action="post_remove",
+                instance=self.instance, reverse=self.reverse,
+                model=self.model, pk_set=old_ids, using=db)
 
     return ManyRelatedManager
 

docs/ref/models/querysets.txt

@@ -2132,6 +2132,8 @@ extract two field values, where only one is expected::
     inner_qs = Blog.objects.filter(name__contains='Ch').values('name', 'id')
     entries = Entry.objects.filter(blog__name__in=inner_qs)
 
+.. _nested-queries-performance:
+
 .. admonition:: Performance considerations
 
     Be cautious about using nested queries and understand your database

docs/releases/1.7.txt

@@ -574,6 +574,32 @@ a :exc:`~exceptions.ValueError` when encountering them, you will have to
 install pytz_. You may be affected by this problem if you use Django's time
 zone-related date formats or :mod:`django.contrib.syndication`.
 
+``remove()`` and ``clear()`` methods of related managers
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The ``remove()`` and ``clear()`` methods of the related managers created by
+``ForeignKey``, ``GenericForeignKey``, and ``ManyToManyField`` suffered from a
+number of issues. Some operations ran multiple data modifying queries without
+wrapping them in a transaction, and some operations didn't respect default
+filtering when it was present (i.e. when the default manager on the related
+model implemented a custom ``get_queryset()``).
+
+Fixing the issues introduced some backward incompatible changes:
+
+- The default implementation of ``remove()`` for ``ForeignKey`` related managers
+  changed from a series of ``Model.save()`` calls to a single
+  ``QuerySet.update()`` call. The change means that ``pre_save`` and
+  ``post_save`` signals aren't sent anymore.
+
+- The ``remove()`` and ``clear()`` methods for ``GenericForeignKey`` related
+  managers now perform bulk delete. The ``Model.delete()`` method isn't called
+  on each instance anymore.
+
+- The ``remove()`` and ``clear()`` methods for ``ManyToManyField`` related
+  managers perform nested queries when filtering is involved, which may or
+  may not be an issue depending on your database and your data itself.
+  See :ref:`this note <nested-queries-performance>` for more details.
+
 .. _pytz: https://pypi.python.org/pypi/pytz/
 
 Miscellaneous

tests/custom_managers/models.py

@@ -101,17 +101,37 @@ def __str__(self):
         return "%s %s" % (self.first_name, self.last_name)
 
 
+@python_2_unicode_compatible
+class FunPerson(models.Model):
+    first_name = models.CharField(max_length=30)
+    last_name = models.CharField(max_length=30)
+    fun = models.BooleanField(default=True)
+
+    favorite_book = models.ForeignKey('Book', null=True, related_name='fun_people_favorite_books')
+    favorite_thing_type = models.ForeignKey('contenttypes.ContentType', null=True)
+    favorite_thing_id = models.IntegerField(null=True)
+    favorite_thing = generic.GenericForeignKey('favorite_thing_type', 'favorite_thing_id')
+
+    objects = FunPeopleManager()
+
+    def __str__(self):
+        return "%s %s" % (self.first_name, self.last_name)
+
 @python_2_unicode_compatible
 class Book(models.Model):
     title = models.CharField(max_length=50)
     author = models.CharField(max_length=30)
     is_published = models.BooleanField(default=False)
     published_objects = PublishedBookManager()
     authors = models.ManyToManyField(Person, related_name='books')
+    fun_authors = models.ManyToManyField(FunPerson, related_name='books')
 
     favorite_things = generic.GenericRelation(Person,
         content_type_field='favorite_thing_type', object_id_field='favorite_thing_id')
 
+    fun_people_favorite_things = generic.GenericRelation(FunPerson,
+        content_type_field='favorite_thing_type', object_id_field='favorite_thing_id')
+
     def __str__(self):
         return self.title