Permalink
Browse files

Fixed #21169 -- Reworked RelatedManager methods use default filtering

The `remove()` and `clear()` methods of the related managers created by
`ForeignKey`, `GenericForeignKey`, and `ManyToManyField` suffered from a
number of issues. Some operations ran multiple data modifying queries without
wrapping them in a transaction, and some operations didn't respect default
filtering when it was present (i.e. when the default manager on the related
model implemented a custom `get_queryset()`).

Fixing the issues introduced some backward incompatible changes:

- The implementation of `remove()` for `ForeignKey` related managers changed
  from a series of `Model.save()` calls to a single `QuerySet.update()` call.
  The change means that `pre_save` and `post_save` signals aren't called anymore.

- The `remove()` and `clear()` methods for `GenericForeignKey` related
  managers now perform bulk delete so `Model.delete()` isn't called anymore.

- The `remove()` and `clear()` methods for `ManyToManyField` related
  managers perform nested queries when filtering is involved, which may
  or may not be an issue depending on the database and the data itself.

Refs. #3871, #21174.

Thanks Anssi Kääriäinen and Tim Graham for the reviews.
  • Loading branch information...
1 parent 0b3c8fc commit 17c3997f6828e88e4646071a8187c1318b65597d @loic loic committed with akaariai Sep 26, 2013
@@ -2,7 +2,7 @@
from django.db import connection, connections, router, transaction
from django.db.backends import utils
-from django.db.models import signals
+from django.db.models import signals, Q
from django.db.models.fields import (AutoField, Field, IntegerField,
PositiveIntegerField, PositiveSmallIntegerField, FieldDoesNotExist)
from django.db.models.related import RelatedObject, PathInfo
@@ -464,14 +464,21 @@ def get_or_create(self, **kwargs):
# remove() and clear() are only provided if the ForeignKey can have a value of null.
if rel_field.null:
def remove(self, *objs):
+ # If there aren't any objects, there is nothing to do.
+ if not objs:
+ return
+
val = rel_field.get_foreign_related_value(self.instance)
+
+ old_ids = set()
for obj in objs:
# Is obj actually part of this descriptor set?
if rel_field.get_local_related_value(obj) == val:
- setattr(obj, rel_field.name, None)
- obj.save()
+ old_ids.add(obj.pk)
else:
raise rel_field.rel.to.DoesNotExist("%r is not related to %r." % (obj, self.instance))
+
+ self.filter(pk__in=old_ids).update(**{rel_field.name: None})
remove.alters_data = True
def clear(self):
@@ -536,6 +543,7 @@ def __init__(self, model=None, query_field_name=None, instance=None, symmetrical
self.instance = instance
self.symmetrical = symmetrical
self.source_field = source_field
+ self.target_field = through._meta.get_field(target_field_name)
self.source_field_name = source_field_name
self.target_field_name = target_field_name
self.reverse = reverse
@@ -572,6 +580,19 @@ def __call__(self, **kwargs):
)
do_not_call_in_templates = True
+ def _build_clear_filters(self, qs):
+ filters = Q(**{
+ self.source_field_name: self.related_val,
+ '%s__in' % self.target_field_name: qs
+ })
+
+ if self.symmetrical:
+ filters |= Q(**{
+ self.target_field_name: self.related_val,
+ '%s__in' % self.source_field_name: qs
+ })
+ return filters
+
def get_queryset(self):
try:
return self.instance._prefetched_objects_cache[self.prefetch_cache_name]
@@ -625,18 +646,20 @@ def add(self, *objs):
def remove(self, *objs):
self._remove_items(self.source_field_name, self.target_field_name, *objs)
-
- # If this is a symmetrical m2m relation to self, remove the mirror entry in the m2m table
- if self.symmetrical:
- self._remove_items(self.target_field_name, self.source_field_name, *objs)
remove.alters_data = True
def clear(self):
- self._clear_items(self.source_field_name)
+ db = router.db_for_write(self.through, instance=self.instance)
- # If this is a symmetrical m2m relation to self, clear the mirror entry in the m2m table
- if self.symmetrical:
- self._clear_items(self.target_field_name)
+ signals.m2m_changed.send(sender=self.through, action="pre_clear",
+ instance=self.instance, reverse=self.reverse,
+ model=self.model, pk_set=None, using=db)
+ filters = self._build_clear_filters(self.using(db))
+ self.through._default_manager.using(db).filter(filters).delete()
+
+ signals.m2m_changed.send(sender=self.through, action="post_clear",
+ instance=self.instance, reverse=self.reverse,
+ model=self.model, pk_set=None, using=db)
clear.alters_data = True
def create(self, **kwargs):
@@ -722,55 +745,33 @@ def _remove_items(self, source_field_name, target_field_name, *objs):
# *objs - objects to remove
# If there aren't any objects, there is nothing to do.
- if objs:
- # Check that all the objects are of the right type
- old_ids = set()
- for obj in objs:
- if isinstance(obj, self.model):
- fk_val = self.through._meta.get_field(
- target_field_name).get_foreign_related_value(obj)[0]
- old_ids.add(fk_val)
- else:
- old_ids.add(obj)
- # Work out what DB we're operating on
- db = router.db_for_write(self.through, instance=self.instance)
- # Send a signal to the other end if need be.
- if self.reverse or source_field_name == self.source_field_name:
- # Don't send the signal when we are deleting the
- # duplicate data row for symmetrical reverse entries.
- signals.m2m_changed.send(sender=self.through, action="pre_remove",
- instance=self.instance, reverse=self.reverse,
- model=self.model, pk_set=old_ids, using=db)
- # Remove the specified objects from the join table
- self.through._default_manager.using(db).filter(**{
- source_field_name: self.related_val[0],
- '%s__in' % target_field_name: old_ids
- }).delete()
- if self.reverse or source_field_name == self.source_field_name:
- # Don't send the signal when we are deleting the
- # duplicate data row for symmetrical reverse entries.
- signals.m2m_changed.send(sender=self.through, action="post_remove",
- instance=self.instance, reverse=self.reverse,
- model=self.model, pk_set=old_ids, using=db)
+ if not objs:
+ return
+
+ # Check that all the objects are of the right type
+ old_ids = set()
+ for obj in objs:
+ if isinstance(obj, self.model):
+ fk_val = self.target_field.get_foreign_related_value(obj)[0]
+ old_ids.add(fk_val)
+ else:
+ old_ids.add(obj)
- def _clear_items(self, source_field_name):
db = router.db_for_write(self.through, instance=self.instance)
- # source_field_name: the PK colname in join table for the source object
- if self.reverse or source_field_name == self.source_field_name:
- # Don't send the signal when we are clearing the
- # duplicate data rows for symmetrical reverse entries.
- signals.m2m_changed.send(sender=self.through, action="pre_clear",
- instance=self.instance, reverse=self.reverse,
- model=self.model, pk_set=None, using=db)
- self.through._default_manager.using(db).filter(**{
- source_field_name: self.related_val
- }).delete()
- if self.reverse or source_field_name == self.source_field_name:
- # Don't send the signal when we are clearing the
- # duplicate data rows for symmetrical reverse entries.
- signals.m2m_changed.send(sender=self.through, action="post_clear",
- instance=self.instance, reverse=self.reverse,
- model=self.model, pk_set=None, using=db)
+
+ # Send a signal to the other end if need be.
+ signals.m2m_changed.send(sender=self.through, action="pre_remove",
+ instance=self.instance, reverse=self.reverse,
+ model=self.model, pk_set=old_ids, using=db)
+
+ old_vals_qs = self.using(db).filter(**{
+ '%s__in' % self.target_field.related_field.attname: old_ids})
+ filters = self._build_clear_filters(old_vals_qs)
+ self.through._default_manager.using(db).filter(filters).delete()
+
+ signals.m2m_changed.send(sender=self.through, action="post_remove",
+ instance=self.instance, reverse=self.reverse,
+ model=self.model, pk_set=old_ids, using=db)
return ManyRelatedManager
@@ -2132,6 +2132,8 @@ extract two field values, where only one is expected::
inner_qs = Blog.objects.filter(name__contains='Ch').values('name', 'id')
entries = Entry.objects.filter(blog__name__in=inner_qs)
+.. _nested-queries-performance:
+
.. admonition:: Performance considerations
Be cautious about using nested queries and understand your database
View
@@ -574,6 +574,32 @@ a :exc:`~exceptions.ValueError` when encountering them, you will have to
install pytz_. You may be affected by this problem if you use Django's time
zone-related date formats or :mod:`django.contrib.syndication`.
+``remove()`` and ``clear()`` methods of related managers
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The ``remove()`` and ``clear()`` methods of the related managers created by
+``ForeignKey``, ``GenericForeignKey``, and ``ManyToManyField`` suffered from a
+number of issues. Some operations ran multiple data modifying queries without
+wrapping them in a transaction, and some operations didn't respect default
+filtering when it was present (i.e. when the default manager on the related
+model implemented a custom ``get_queryset()``).
+
+Fixing the issues introduced some backward incompatible changes:
+
+- The default implementation of ``remove()`` for ``ForeignKey`` related managers
+ changed from a series of ``Model.save()`` calls to a single
+ ``QuerySet.update()`` call. The change means that ``pre_save`` and
+ ``post_save`` signals aren't sent anymore.
+
+- The ``remove()`` and ``clear()`` methods for ``GenericForeignKey`` related
+ managers now perform bulk delete. The ``Model.delete()`` method isn't called
+ on each instance anymore.
+
+- The ``remove()`` and ``clear()`` methods for ``ManyToManyField`` related
+ managers perform nested queries when filtering is involved, which may or
+ may not be an issue depending on your database and your data itself.
+ See :ref:`this note <nested-queries-performance>` for more details.
+
.. _pytz: https://pypi.python.org/pypi/pytz/
Miscellaneous
@@ -102,16 +102,36 @@ def __str__(self):
@python_2_unicode_compatible
+class FunPerson(models.Model):
+ first_name = models.CharField(max_length=30)
+ last_name = models.CharField(max_length=30)
+ fun = models.BooleanField(default=True)
+
+ favorite_book = models.ForeignKey('Book', null=True, related_name='fun_people_favorite_books')
+ favorite_thing_type = models.ForeignKey('contenttypes.ContentType', null=True)
+ favorite_thing_id = models.IntegerField(null=True)
+ favorite_thing = generic.GenericForeignKey('favorite_thing_type', 'favorite_thing_id')
+
+ objects = FunPeopleManager()
+
+ def __str__(self):
+ return "%s %s" % (self.first_name, self.last_name)
+
+@python_2_unicode_compatible
class Book(models.Model):
title = models.CharField(max_length=50)
author = models.CharField(max_length=30)
is_published = models.BooleanField(default=False)
published_objects = PublishedBookManager()
authors = models.ManyToManyField(Person, related_name='books')
+ fun_authors = models.ManyToManyField(FunPerson, related_name='books')
favorite_things = generic.GenericRelation(Person,
content_type_field='favorite_thing_type', object_id_field='favorite_thing_id')
+ fun_people_favorite_things = generic.GenericRelation(FunPerson,
+ content_type_field='favorite_thing_type', object_id_field='favorite_thing_id')
+
def __str__(self):
return self.title
Oops, something went wrong.

0 comments on commit 17c3997

Please sign in to comment.