Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Fixed #19500 -- Solved a regression in join reuse

The ORM didn't reuse joins for direct foreign key traversals when using
chained filters. For example:
    qs.filter(fk__somefield=1).filter(fk__somefield=2))
produced two joins.

As a bonus, reverse onetoone filters can now reuse joins correctly

The regression was caused by the join() method refactor in commit
6884713

Thanks for Simon Charette for spotting some issues with the first draft
of the patch.
  • Loading branch information...
commit 3dcd435a0eb4380a3846e9c65140df480975687e 1 parent c04c03d
Anssi Kääriäinen authored December 20, 2012
8  django/db/models/sql/compiler.py
@@ -6,7 +6,7 @@
6 6
 from django.db.models.constants import LOOKUP_SEP
7 7
 from django.db.models.query_utils import select_related_descend
8 8
 from django.db.models.sql.constants import (SINGLE, MULTI, ORDER_DIR,
9  
-        GET_ITERATOR_CHUNK_SIZE, REUSE_ALL, SelectInfo)
  9
+        GET_ITERATOR_CHUNK_SIZE, SelectInfo)
10 10
 from django.db.models.sql.datastructures import EmptyResultSet
11 11
 from django.db.models.sql.expressions import SQLEvaluator
12 12
 from django.db.models.sql.query import get_order_dir, Query
@@ -317,7 +317,7 @@ def get_distinct(self):
317 317
 
318 318
         for name in self.query.distinct_fields:
319 319
             parts = name.split(LOOKUP_SEP)
320  
-            field, col, alias, _, _ = self._setup_joins(parts, opts, None)
  320
+            field, col, alias, _, _ = self._setup_joins(parts, opts)
321 321
             col, alias = self._final_join_removal(col, alias)
322 322
             result.append("%s.%s" % (qn(alias), qn2(col)))
323 323
         return result
@@ -450,7 +450,7 @@ def _setup_joins(self, pieces, opts, alias):
450 450
         if not alias:
451 451
             alias = self.query.get_initial_alias()
452 452
         field, target, opts, joins, _ = self.query.setup_joins(
453  
-            pieces, opts, alias, REUSE_ALL)
  453
+            pieces, opts, alias)
454 454
         # We will later on need to promote those joins that were added to the
455 455
         # query afresh above.
456 456
         joins_to_promote = [j for j in joins if self.query.alias_refcount[j] < 2]
@@ -688,7 +688,7 @@ def fill_related_selections(self, opts=None, root_alias=None, cur_depth=1,
688 688
                         int_opts = int_model._meta
689 689
                         alias = self.query.join(
690 690
                             (alias, int_opts.db_table, lhs_col, int_opts.pk.column),
691  
-                            promote=True,
  691
+                            promote=True
692 692
                         )
693 693
                         alias_chain.append(alias)
694 694
                 alias = self.query.join(
3  django/db/models/sql/constants.py
@@ -44,6 +44,3 @@
44 44
     'ASC': ('ASC', 'DESC'),
45 45
     'DESC': ('DESC', 'ASC'),
46 46
 }
47  
-
48  
-# A marker for join-reusability.
49  
-REUSE_ALL = object()
5  django/db/models/sql/expressions.py
... ...
@@ -1,10 +1,9 @@
1 1
 from django.core.exceptions import FieldError
2 2
 from django.db.models.constants import LOOKUP_SEP
3 3
 from django.db.models.fields import FieldDoesNotExist
4  
-from django.db.models.sql.constants import REUSE_ALL
5 4
 
6 5
 class SQLEvaluator(object):
7  
-    def __init__(self, expression, query, allow_joins=True, reuse=REUSE_ALL):
  6
+    def __init__(self, expression, query, allow_joins=True, reuse=None):
8 7
         self.expression = expression
9 8
         self.opts = query.get_meta()
10 9
         self.cols = []
@@ -54,7 +53,7 @@ def prepare_leaf(self, node, query, allow_joins):
54 53
                     field_list, query.get_meta(),
55 54
                     query.get_initial_alias(), self.reuse)
56 55
                 col, _, join_list = query.trim_joins(source, join_list, path)
57  
-                if self.reuse is not None and self.reuse != REUSE_ALL:
  56
+                if self.reuse is not None:
58 57
                     self.reuse.update(join_list)
59 58
                 self.cols.append((node, (join_list[-1], col)))
60 59
             except FieldDoesNotExist:
39  django/db/models/sql/query.py
@@ -20,7 +20,7 @@
20 20
 from django.db.models.loading import get_model
21 21
 from django.db.models.sql import aggregates as base_aggregates_module
22 22
 from django.db.models.sql.constants import (QUERY_TERMS, ORDER_DIR, SINGLE,
23  
-        ORDER_PATTERN, REUSE_ALL, JoinInfo, SelectInfo, PathInfo)
  23
+        ORDER_PATTERN, JoinInfo, SelectInfo, PathInfo)
24 24
 from django.db.models.sql.datastructures import EmptyResultSet, Empty, MultiJoin
25 25
 from django.db.models.sql.expressions import SQLEvaluator
26 26
 from django.db.models.sql.where import (WhereNode, Constraint, EverythingNode,
@@ -891,7 +891,7 @@ def count_active_tables(self):
891 891
         """
892 892
         return len([1 for count in self.alias_refcount.values() if count])
893 893
 
894  
-    def join(self, connection, reuse=REUSE_ALL, promote=False,
  894
+    def join(self, connection, reuse=None, promote=False,
895 895
              outer_if_first=False, nullable=False, join_field=None):
896 896
         """
897 897
         Returns an alias for the join in 'connection', either reusing an
@@ -902,10 +902,9 @@ def join(self, connection, reuse=REUSE_ALL, promote=False,
902 902
 
903 903
             lhs.lhs_col = table.col
904 904
 
905  
-        The 'reuse' parameter can be used in three ways: it can be REUSE_ALL
906  
-        which means all joins (matching the connection) are reusable, it can
907  
-        be a set containing the aliases that can be reused, or it can be None
908  
-        which means a new join is always created.
  905
+        The 'reuse' parameter can be either None which means all joins
  906
+        (matching the connection) are reusable, or it can be a set containing
  907
+        the aliases that can be reused.
909 908
 
910 909
         If 'promote' is True, the join type for the alias will be LOUTER (if
911 910
         the alias previously existed, the join type will be promoted from INNER
@@ -926,10 +925,8 @@ def join(self, connection, reuse=REUSE_ALL, promote=False,
926 925
         """
927 926
         lhs, table, lhs_col, col = connection
928 927
         existing = self.join_map.get(connection, ())
929  
-        if reuse == REUSE_ALL:
  928
+        if reuse is None:
930 929
             reuse = existing
931  
-        elif reuse is None:
932  
-            reuse = set()
933 930
         else:
934 931
             reuse = [a for a in existing if a in reuse]
935 932
         for alias in reuse:
@@ -1040,7 +1037,7 @@ def add_aggregate(self, aggregate, model, alias, is_summary):
1040 1037
             # then we need to explore the joins that are required.
1041 1038
 
1042 1039
             field, source, opts, join_list, path = self.setup_joins(
1043  
-                field_list, opts, self.get_initial_alias(), REUSE_ALL)
  1040
+                field_list, opts, self.get_initial_alias())
1044 1041
 
1045 1042
             # Process the join chain to see if it can be trimmed
1046 1043
             col, _, join_list = self.trim_joins(source, join_list, path)
@@ -1441,7 +1438,7 @@ def names_to_path(self, names, opts, allow_many=False,
1441 1438
             raise MultiJoin(multijoin_pos + 1)
1442 1439
         return path, final_field, target
1443 1440
 
1444  
-    def setup_joins(self, names, opts, alias, can_reuse, allow_many=True,
  1441
+    def setup_joins(self, names, opts, alias, can_reuse=None, allow_many=True,
1445 1442
                     allow_explicit_fk=False):
1446 1443
         """
1447 1444
         Compute the necessary table joins for the passage through the fields
@@ -1450,9 +1447,9 @@ def setup_joins(self, names, opts, alias, can_reuse, allow_many=True,
1450 1447
         the table to start the joining from.
1451 1448
 
1452 1449
         The 'can_reuse' defines the reverse foreign key joins we can reuse. It
1453  
-        can be sql.constants.REUSE_ALL in which case all joins are reusable
1454  
-        or a set of aliases that can be reused. Note that Non-reverse foreign
1455  
-        keys are always reusable.
  1450
+        can be None in which case all joins are reusable or a set of aliases
  1451
+        that can be reused. Note that non-reverse foreign keys are always
  1452
+        reusable when using setup_joins().
1456 1453
 
1457 1454
         If 'allow_many' is False, then any reverse foreign key seen will
1458 1455
         generate a MultiJoin exception.
@@ -1485,8 +1482,9 @@ def setup_joins(self, names, opts, alias, can_reuse, allow_many=True,
1485 1482
             else:
1486 1483
                 nullable = True
1487 1484
             connection = alias, opts.db_table, from_field.column, to_field.column
1488  
-            alias = self.join(connection, reuse=can_reuse, nullable=nullable,
1489  
-                              join_field=join_field)
  1485
+            reuse = None if direct or to_field.unique else can_reuse
  1486
+            alias = self.join(connection, reuse=reuse,
  1487
+                              nullable=nullable, join_field=join_field)
1490 1488
             joins.append(alias)
1491 1489
         return final_field, target, opts, joins, path
1492 1490
 
@@ -1643,7 +1641,7 @@ def add_fields(self, field_names, allow_m2m=True):
1643 1641
         try:
1644 1642
             for name in field_names:
1645 1643
                 field, target, u2, joins, u3 = self.setup_joins(
1646  
-                        name.split(LOOKUP_SEP), opts, alias, REUSE_ALL, allow_m2m,
  1644
+                        name.split(LOOKUP_SEP), opts, alias, None, allow_m2m,
1647 1645
                         True)
1648 1646
                 final_alias = joins[-1]
1649 1647
                 col = target.column
@@ -1729,8 +1727,9 @@ def add_count_column(self):
1729 1727
         else:
1730 1728
             opts = self.model._meta
1731 1729
             if not self.select:
1732  
-                count = self.aggregates_module.Count((self.join((None, opts.db_table, None, None)), opts.pk.column),
1733  
-                                         is_summary=True, distinct=True)
  1730
+                count = self.aggregates_module.Count(
  1731
+                    (self.join((None, opts.db_table, None, None)), opts.pk.column),
  1732
+                    is_summary=True, distinct=True)
1734 1733
             else:
1735 1734
                 # Because of SQL portability issues, multi-column, distinct
1736 1735
                 # counts need a sub-query -- see get_count() for details.
@@ -1934,7 +1933,7 @@ def set_start(self, start):
1934 1933
         opts = self.model._meta
1935 1934
         alias = self.get_initial_alias()
1936 1935
         field, col, opts, joins, extra = self.setup_joins(
1937  
-                start.split(LOOKUP_SEP), opts, alias, REUSE_ALL)
  1936
+                start.split(LOOKUP_SEP), opts, alias)
1938 1937
         select_col = self.alias_map[joins[1]].lhs_join_col
1939 1938
         select_alias = alias
1940 1939
 
1  django/db/models/sql/subqueries.py
@@ -232,7 +232,6 @@ def add_date_select(self, field_name, lookup_type, order='ASC'):
232 232
                 field_name.split(LOOKUP_SEP),
233 233
                 self.get_meta(),
234 234
                 self.get_initial_alias(),
235  
-                False
236 235
             )
237 236
         except FieldError:
238 237
             raise FieldDoesNotExist("%s has no field named '%s'" % (
8  tests/regressiontests/generic_relations_regress/tests.py
@@ -72,5 +72,11 @@ def test_q_object_or(self):
72 72
             Q(notes__note__icontains=r'other note'))
73 73
         self.assertTrue(org_contact in qs)
74 74
 
75  
-
  75
+    def test_join_reuse(self):
  76
+        qs = Person.objects.filter(
  77
+            addresses__street='foo'
  78
+        ).filter(
  79
+            addresses__street='bar'
  80
+        )
  81
+        self.assertEqual(str(qs.query).count('JOIN'), 2)
76 82
 
33  tests/regressiontests/queries/tests.py
@@ -2418,3 +2418,36 @@ def test_reverse_trimming(self):
2418 2418
         qs = Tag.objects.filter(annotation__tag=t.pk)
2419 2419
         self.assertIn('INNER JOIN', str(qs.query))
2420 2420
         self.assertEquals(list(qs), [])
  2421
+
  2422
+class JoinReuseTest(TestCase):
  2423
+    """
  2424
+    Test that the queries reuse joins sensibly (for example, direct joins
  2425
+    are always reused).
  2426
+    """
  2427
+    def test_fk_reuse(self):
  2428
+        qs = Annotation.objects.filter(tag__name='foo').filter(tag__name='bar')
  2429
+        self.assertEqual(str(qs.query).count('JOIN'), 1)
  2430
+
  2431
+    def test_fk_reuse_select_related(self):
  2432
+        qs = Annotation.objects.filter(tag__name='foo').select_related('tag')
  2433
+        self.assertEqual(str(qs.query).count('JOIN'), 1)
  2434
+
  2435
+    def test_fk_reuse_annotation(self):
  2436
+        qs = Annotation.objects.filter(tag__name='foo').annotate(cnt=Count('tag__name'))
  2437
+        self.assertEqual(str(qs.query).count('JOIN'), 1)
  2438
+
  2439
+    def test_fk_reuse_disjunction(self):
  2440
+        qs = Annotation.objects.filter(Q(tag__name='foo') | Q(tag__name='bar'))
  2441
+        self.assertEqual(str(qs.query).count('JOIN'), 1)
  2442
+
  2443
+    def test_fk_reuse_order_by(self):
  2444
+        qs = Annotation.objects.filter(tag__name='foo').order_by('tag__name')
  2445
+        self.assertEqual(str(qs.query).count('JOIN'), 1)
  2446
+
  2447
+    def test_revo2o_reuse(self):
  2448
+        qs = Detail.objects.filter(member__name='foo').filter(member__name='foo')
  2449
+        self.assertEqual(str(qs.query).count('JOIN'), 1)
  2450
+
  2451
+    def test_revfk_noreuse(self):
  2452
+        qs = Author.objects.filter(report__name='r4').filter(report__name='r1')
  2453
+        self.assertEqual(str(qs.query).count('JOIN'), 2)

0 notes on commit 3dcd435

Please sign in to comment.
Something went wrong with that request. Please try again.