Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

queryset-refactor: Fixed a large bag of order_by() problems.

This also picked up a small bug in some twisted select_related() handling.

Introduces a new syntax for cross-model ordering: foo__bar__baz, using field
names, instead of a strange combination of table names and field names. This
might turn out to be backwards compatible (although the old syntax leads to
bugs and is not to be recommended).

Still to come: fixes for extra() handling, since the new syntax can't handle
that and doc updates.

Things are starting to get a bit slow here, so we might eventually have to
remove ordering by many-many and other multiple-result fields, since they don't
make a lot of sense in any case. For now, it's legal.

Refs #2076, #2874, #3002 (although the admin bit doesn't work yet).


git-svn-id: http://code.djangoproject.com/svn/django/branches/queryset-refactor@6510 bcc190cf-cafb-0310-a4f2-bffc1f526a37
  • Loading branch information...
commit a1d160e2ea22cb010b276cfc2a085f9cd3d81b22 1 parent 6678de1
Malcolm Tredinnick authored October 14, 2007
5  django/core/management/validation.py
@@ -190,7 +190,10 @@ def get_validation_errors(outfile, app=None):
190 190
                     field_name = field_name[1:]
191 191
                 if opts.order_with_respect_to and field_name == '_order':
192 192
                     continue
193  
-                if '.' in field_name: continue # Skip ordering in the format 'table.field'.
  193
+                # Skip ordering in the format field1__field2 (FIXME: checking
  194
+                # this format would be nice, but it's a little fiddly).
  195
+                if '_' in field_name:
  196
+                    continue
194 197
                 try:
195 198
                     opts.get_field(field_name, many_to_many=False)
196 199
                 except models.FieldDoesNotExist:
149  django/db/models/sql/query.py
@@ -8,6 +8,7 @@
8 8
 """
9 9
 
10 10
 import copy
  11
+import re
11 12
 
12 13
 from django.utils import tree
13 14
 from django.db.models.sql.where import WhereNode, AND, OR
@@ -35,7 +36,7 @@
35 36
 # Separator used to split filter strings apart.
36 37
 LOOKUP_SEP = '__'
37 38
 
38  
-# Constants to make looking up tuple values clearerer.
  39
+# Constants to make looking up tuple values clearer.
39 40
 # Join lists
40 41
 TABLE_NAME = 0
41 42
 RHS_ALIAS = 1
@@ -43,7 +44,7 @@
43 44
 LHS_ALIAS = 3
44 45
 LHS_JOIN_COL = 4
45 46
 RHS_JOIN_COL = 5
46  
-# Alias maps lists
  47
+# Alias map lists
47 48
 ALIAS_TABLE = 0
48 49
 ALIAS_REFCOUNT = 1
49 50
 ALIAS_JOIN = 2
@@ -54,6 +55,8 @@
54 55
 SINGLE = 'single'
55 56
 NONE = None
56 57
 
  58
+ORDER_PATTERN = re.compile(r'\?|[-+]?\w+$')
  59
+
57 60
 class Query(object):
58 61
     """
59 62
     A single SQL query.
@@ -91,6 +94,7 @@ def __init__(self, model, connection):
91 94
         self.extra_tables = []
92 95
         self.extra_where = []
93 96
         self.extra_params = []
  97
+        self.extra_order_by = []
94 98
 
95 99
     def __str__(self):
96 100
         """
@@ -140,6 +144,7 @@ def clone(self, klass=None, **kwargs):
140 144
         obj.extra_tables = self.extra_tables[:]
141 145
         obj.extra_where = self.extra_where[:]
142 146
         obj.extra_params = self.extra_params[:]
  147
+        obj.extra_order_by = self.extra_order_by[:]
143 148
         obj.__dict__.update(kwargs)
144 149
         return obj
145 150
 
@@ -189,19 +194,22 @@ def as_sql(self, with_limits=True):
189 194
         in the query.
190 195
         """
191 196
         self.pre_sql_setup()
  197
+        out_cols = self.get_columns()
  198
+        ordering = self.get_ordering()
  199
+        # This must come after 'select' and 'ordering' -- see docstring of
  200
+        # get_from_clause() for details.
  201
+        from_, f_params = self.get_from_clause()
  202
+        where, w_params = self.where.as_sql()
  203
+
192 204
         result = ['SELECT']
193 205
         if self.distinct:
194 206
             result.append('DISTINCT')
195  
-        out_cols = self.get_columns()
196 207
         result.append(', '.join(out_cols))
197 208
 
198 209
         result.append('FROM')
199  
-        from_, f_params = self.get_from_clause()
200 210
         result.extend(from_)
201 211
         params = list(f_params)
202 212
 
203  
-        where, w_params = self.where.as_sql()
204  
-        params.extend(w_params)
205 213
         if where:
206 214
             result.append('WHERE %s' % where)
207 215
         if self.extra_where:
@@ -210,12 +218,12 @@ def as_sql(self, with_limits=True):
210 218
             else:
211 219
                 result.append('AND')
212 220
             result.append(' AND'.join(self.extra_where))
  221
+        params.extend(w_params)
213 222
 
214 223
         if self.group_by:
215 224
             grouping = self.get_grouping()
216 225
             result.append('GROUP BY %s' % ', '.join(grouping))
217 226
 
218  
-        ordering = self.get_ordering()
219 227
         if ordering:
220 228
             result.append('ORDER BY %s' % ', '.join(ordering))
221 229
 
@@ -312,11 +320,14 @@ def combine(self, rhs, connection):
312 320
         # Ordering uses the 'rhs' ordering, unless it has none, in which case
313 321
         # the current ordering is used.
314 322
         self.order_by = rhs.order_by and rhs.order_by[:] or self.order_by
  323
+        self.extra_order_by = (rhs.extra_order_by and rhs.extra_order_by[:] or
  324
+                self.extra_order_by)
315 325
 
316 326
     def pre_sql_setup(self):
317 327
         """
318  
-        Does any necessary class setup prior to producing SQL. This is for
319  
-        things that can't necessarily be done in __init__.
  328
+        Does any necessary class setup immediately prior to producing SQL. This
  329
+        is for things that can't necessarily be done in __init__ because we
  330
+        might not have all the pieces in place at that time.
320 331
         """
321 332
         if not self.tables:
322 333
             self.join((None, self.model._meta.db_table, None, None))
@@ -356,9 +367,14 @@ def get_from_clause(self):
356 367
         "FROM" part of the query, as well as any extra parameters that need to
357 368
         be included. Sub-classes, can override this to create a from-clause via
358 369
         a "select", for example (e.g. CountQuery).
  370
+
  371
+        This should only be called after any SQL construction methods that
  372
+        might change the tables we need. This means the select columns and
  373
+        ordering must be done first.
359 374
         """
360 375
         result = []
361 376
         qn = self.quote_name_unless_alias
  377
+        first = True
362 378
         for alias in self.tables:
363 379
             if not self.alias_map[alias][ALIAS_REFCOUNT]:
364 380
                 continue
@@ -370,12 +386,16 @@ def get_from_clause(self):
370 386
                         % (join_type, qn(name), alias_str, qn(lhs),
371 387
                             qn(lhs_col), qn(alias), qn(col)))
372 388
             else:
373  
-                result.append('%s%s' % (qn(name), alias_str))
  389
+                connector = not first and ', ' or ''
  390
+                result.append('%s%s%s' % (connector, qn(name), alias_str))
  391
+            first = False
374 392
         extra_tables = []
375 393
         for t in self.extra_tables:
376 394
             alias, created = self.table_alias(t)
377 395
             if created:
378  
-                result.append(', %s' % alias)
  396
+                connector = not first and ', ' or ''
  397
+                result.append('%s%s' % (connector, alias))
  398
+                first = False
379 399
         return result, []
380 400
 
381 401
     def get_grouping(self):
@@ -396,13 +416,20 @@ def get_grouping(self):
396 416
     def get_ordering(self):
397 417
         """
398 418
         Returns a tuple representing the SQL elements in the "order by" clause.
  419
+
  420
+        Determining the ordering SQL can change the tables we need to include,
  421
+        so this should be run *before* get_from_clause().
399 422
         """
400  
-        if self.order_by is None:
  423
+        if self.extra_order_by:
  424
+            ordering = self.extra_order_by
  425
+        elif self.order_by is None:
401 426
             ordering = []
402 427
         else:
  428
+            # Note that self.order_by can be empty in two ways: [] ("use the
  429
+            # default"), which is handled here, and None ("no ordering"), which
  430
+            # is handled in the previous test.
403 431
             ordering = self.order_by or self.model._meta.ordering
404 432
         qn = self.quote_name_unless_alias
405  
-        opts = self.model._meta
406 433
         result = []
407 434
         for field in ordering:
408 435
             if field == '?':
@@ -416,24 +443,62 @@ def get_ordering(self):
416 443
                     order = 'ASC'
417 444
                 result.append('%s %s' % (field, order))
418 445
                 continue
419  
-            if field[0] == '-':
420  
-                col = field[1:]
421  
-                order = 'DESC'
422  
-            else:
423  
-                col = field
424  
-                order = 'ASC'
425  
-            if '.' in col:
  446
+            if '.' in field:
  447
+                # This came in through an extra(ordering=...) addition. Pass it
  448
+                # on verbatim, after mapping the table name to an alias, if
  449
+                # necessary.
  450
+                col, order = get_order_dir(field)
426 451
                 table, col = col.split('.', 1)
427  
-                table = '%s.' % qn(self.table_alias(table)[0])
428  
-            elif col not in self.extra_select:
429  
-                # Use the root model's database table as the referenced table.
430  
-                table = '%s.' % qn(self.tables[0])
  452
+                result.append('%s.%s %s' % (qn(self.table_alias(table)[0]), col,
  453
+                        order))
  454
+            elif get_order_dir(field)[0] not in self.extra_select:
  455
+                # 'col' is of the form 'field' or 'field1__field2' or
  456
+                # 'field1__field2__field', etc.
  457
+                for table, col, order in self.find_ordering_name(field,
  458
+                        self.model._meta):
  459
+                    result.append('%s.%s %s' % (qn(table), qn(col), order))
431 460
             else:
432  
-                table = ''
433  
-            result.append('%s%s %s' % (table,
434  
-                    qn(orderfield_to_column(col, opts)), order))
  461
+                col, order = get_order_dir(field)
  462
+                result.append('%s %s' % (qn(col), order))
435 463
         return result
436 464
 
  465
+    def find_ordering_name(self, name, opts, alias=None, default_order='ASC'):
  466
+        """
  467
+        Returns the table alias (the name might not be unambiguous, the alias
  468
+        will be) and column name for ordering by the given 'name' parameter.
  469
+        The 'name' is of the form 'field1__field2__...__fieldN'.
  470
+        """
  471
+        name, order = get_order_dir(name, default_order)
  472
+        pieces = name.split(LOOKUP_SEP)
  473
+        if not alias:
  474
+            alias = self.join((None, opts.db_table, None, None))
  475
+        for elt in pieces:
  476
+            joins, opts, unused1, field, col, unused2 = \
  477
+                    self.get_next_join(elt, opts, alias, False)
  478
+            if joins:
  479
+                alias = joins[-1]
  480
+        col = col or field.column
  481
+
  482
+        # If we get to this point and the field is a relation to another model,
  483
+        # append the default ordering for that model.
  484
+        if joins and opts.ordering:
  485
+            results = []
  486
+            for item in opts.ordering:
  487
+                results.extend(self.find_ordering_name(item, opts, alias,
  488
+                        order))
  489
+            return results
  490
+
  491
+        if alias:
  492
+            # We have to do the same "final join" optimisation as in
  493
+            # add_filter, since the final column might not otherwise be part of
  494
+            # the select set (so we can't order on it).
  495
+            join = self.alias_map[alias][ALIAS_JOIN]
  496
+            if col == join[RHS_JOIN_COL]:
  497
+                self.unref_alias(alias)
  498
+                alias = join[LHS_ALIAS]
  499
+                col = join[LHS_JOIN_COL]
  500
+        return [(alias, col, order)]
  501
+
437 502
     def table_alias(self, table_name, create=False):
438 503
         """
439 504
         Returns a table alias for the given table_name and whether this is a
@@ -557,7 +622,7 @@ def fill_related_selections(self, opts=None, root_alias=None, cur_depth=0,
557 622
             alias = self.join((root_alias, table, f.column,
558 623
                     f.rel.get_related_field().column), exclusions=used)
559 624
             used.append(alias)
560  
-            self.select.extend([(table, f2.column)
  625
+            self.select.extend([(alias, f2.column)
561 626
                     for f2 in f.rel.to._meta.fields])
562 627
             self.fill_related_selections(f.rel.to._meta, alias, cur_depth + 1,
563 628
                     used)
@@ -802,6 +867,12 @@ def add_ordering(self, *ordering):
802 867
         possibly with a direction prefix ('-' or '?') -- or ordinals,
803 868
         corresponding to column positions in the 'select' list.
804 869
         """
  870
+        errors = []
  871
+        for item in ordering:
  872
+            if not ORDER_PATTERN.match(item):
  873
+                errors.append(item)
  874
+        if errors:
  875
+            raise TypeError('Invalid order_by arguments: %s' % errors)
805 876
         self.order_by.extend(ordering)
806 877
 
807 878
     def clear_ordering(self, force_empty=False):
@@ -813,6 +884,7 @@ def clear_ordering(self, force_empty=False):
813 884
             self.order_by = None
814 885
         else:
815 886
             self.order_by = []
  887
+        self.extra_order_by = []
816 888
 
817 889
     def add_count_column(self):
818 890
         """
@@ -1043,6 +1115,9 @@ def get_from_clause(self):
1043 1115
         result, params = self._query.as_sql()
1044 1116
         return ['(%s) AS A1' % result], params
1045 1117
 
  1118
+    def get_ordering(self):
  1119
+        return ()
  1120
+
1046 1121
 def find_field(name, field_list, related_query):
1047 1122
     """
1048 1123
     Finds a field with a specific name in a list of field instances.
@@ -1077,14 +1152,16 @@ def get_legal_fields(opts):
1077 1152
             + field_choices(opts.get_all_related_objects(), True)
1078 1153
             + field_choices(opts.fields, False))
1079 1154
 
1080  
-def orderfield_to_column(name, opts):
  1155
+def get_order_dir(field, default='ASC'):
1081 1156
     """
1082  
-    For a field name specified in an "order by" clause, returns the database
1083  
-    column name. If 'name' is not a field in the current model, it is returned
1084  
-    unchanged.
  1157
+    Returns the field name and direction for an order specification. For
  1158
+    example, '-foo' is returned as ('foo', 'DESC').
  1159
+
  1160
+    The 'default' param is used to indicate which way no prefix (or a '+'
  1161
+    prefix) should sort. The '-' prefix always sorts the opposite way.
1085 1162
     """
1086  
-    try:
1087  
-        return opts.get_field(name, False).column
1088  
-    except FieldDoesNotExist:
1089  
-        return name
  1163
+    dirn = {'ASC': ('ASC', 'DESC'), 'DESC': ('DESC', 'ASC')}[default]
  1164
+    if field[0] == '-':
  1165
+        return field[1:], dirn[1]
  1166
+    return field, dirn[0]
1090 1167
 
2  tests/modeltests/reserved_names/models.py
@@ -45,8 +45,6 @@ def __unicode__(self):
45 45
 b
46 46
 >>> print v.where
47 47
 2005-01-01
48  
->>> Thing.objects.order_by('select.when')
49  
-[<Thing: a>, <Thing: h>]
50 48
 
51 49
 >>> Thing.objects.dates('where', 'year')
52 50
 [datetime.datetime(2005, 1, 1, 0, 0), datetime.datetime(2006, 1, 1, 0, 0)]
128  tests/regressiontests/queries/models.py
... ...
@@ -1,5 +1,5 @@
1 1
 """
2  
-Various combination queries that have been problematic in the past.
  2
+Various complex queries that have been problematic in the past.
3 3
 """
4 4
 
5 5
 from django.db import models
@@ -12,9 +12,29 @@ class Tag(models.Model):
12 12
     def __unicode__(self):
13 13
         return self.name
14 14
 
  15
+class Note(models.Model):
  16
+    note = models.CharField(maxlength=100)
  17
+
  18
+    class Meta:
  19
+        ordering = ['note']
  20
+
  21
+    def __unicode__(self):
  22
+        return self.note
  23
+
  24
+class ExtraInfo(models.Model):
  25
+    info = models.CharField(maxlength=100)
  26
+    note = models.ForeignKey(Note)
  27
+
  28
+    class Meta:
  29
+        ordering = ['info']
  30
+
  31
+    def __unicode__(self):
  32
+        return self.info
  33
+
15 34
 class Author(models.Model):
16 35
     name = models.CharField(maxlength=10)
17 36
     num = models.IntegerField(unique=True)
  37
+    extra = models.ForeignKey(ExtraInfo)
18 38
 
19 39
     def __unicode__(self):
20 40
         return self.name
@@ -23,6 +43,10 @@ class Item(models.Model):
23 43
     name = models.CharField(maxlength=10)
24 44
     tags = models.ManyToManyField(Tag, blank=True, null=True)
25 45
     creator = models.ForeignKey(Author)
  46
+    note = models.ForeignKey(Note)
  47
+
  48
+    class Meta:
  49
+        ordering = ['-note', 'name']
26 50
 
27 51
     def __unicode__(self):
28 52
         return self.name
@@ -34,6 +58,26 @@ class Report(models.Model):
34 58
     def __unicode__(self):
35 59
         return self.name
36 60
 
  61
+class Ranking(models.Model):
  62
+    rank = models.IntegerField()
  63
+    author = models.ForeignKey(Author)
  64
+
  65
+    class Meta:
  66
+        # A complex ordering specification. Should stress the system a bit.
  67
+        ordering = ('author__extra__note', 'author__name', 'rank')
  68
+
  69
+    def __unicode__(self):
  70
+        return '%d: %s' % (self.rank, self.author.name)
  71
+
  72
+class Cover(models.Model):
  73
+    title = models.CharField(maxlength=50)
  74
+    item = models.ForeignKey(Item)
  75
+
  76
+    class Meta:
  77
+        ordering = ['item']
  78
+
  79
+    def __unicode__(self):
  80
+        return self.title
37 81
 
38 82
 __test__ = {'API_TESTS':"""
39 83
 >>> t1 = Tag(name='t1')
@@ -47,25 +91,38 @@ def __unicode__(self):
47 91
 >>> t5 = Tag(name='t5', parent=t3)
48 92
 >>> t5.save()
49 93
 
50  
-
51  
->>> a1 = Author(name='a1', num=1001)
  94
+>>> n1 = Note(note='n1')
  95
+>>> n1.save()
  96
+>>> n2 = Note(note='n2')
  97
+>>> n2.save()
  98
+>>> n3 = Note(note='n3')
  99
+>>> n3.save()
  100
+
  101
+Create these out of order so that sorting by 'id' will be different to sorting
  102
+by 'info'. Helps detect some problems later.
  103
+>>> e2 = ExtraInfo(info='e2', note=n2)
  104
+>>> e2.save()
  105
+>>> e1 = ExtraInfo(info='e1', note=n1)
  106
+>>> e1.save()
  107
+
  108
+>>> a1 = Author(name='a1', num=1001, extra=e1)
52 109
 >>> a1.save()
53  
->>> a2 = Author(name='a2', num=2002)
  110
+>>> a2 = Author(name='a2', num=2002, extra=e1)
54 111
 >>> a2.save()
55  
->>> a3 = Author(name='a3', num=3003)
  112
+>>> a3 = Author(name='a3', num=3003, extra=e2)
56 113
 >>> a3.save()
57  
->>> a4 = Author(name='a4', num=4004)
  114
+>>> a4 = Author(name='a4', num=4004, extra=e2)
58 115
 >>> a4.save()
59 116
 
60  
->>> i1 = Item(name='one', creator=a1)
  117
+>>> i1 = Item(name='one', creator=a1, note=n3)
61 118
 >>> i1.save()
62 119
 >>> i1.tags = [t1, t2]
63  
->>> i2 = Item(name='two', creator=a2)
  120
+>>> i2 = Item(name='two', creator=a2, note=n2)
64 121
 >>> i2.save()
65 122
 >>> i2.tags = [t1, t3]
66  
->>> i3 = Item(name='three', creator=a2)
  123
+>>> i3 = Item(name='three', creator=a2, note=n3)
67 124
 >>> i3.save()
68  
->>> i4 = Item(name='four', creator=a4)
  125
+>>> i4 = Item(name='four', creator=a4, note=n3)
69 126
 >>> i4.save()
70 127
 >>> i4.tags = [t4]
71 128
 
@@ -74,6 +131,20 @@ def __unicode__(self):
74 131
 >>> r2 = Report(name='r2', creator=a3)
75 132
 >>> r2.save()
76 133
 
  134
+Ordering by 'rank' gives us rank2, rank1, rank3. Ordering by the Meta.ordering
  135
+will be rank3, rank2, rank1.
  136
+>>> rank1 = Ranking(rank=2, author=a2)
  137
+>>> rank1.save()
  138
+>>> rank2 = Ranking(rank=1, author=a3)
  139
+>>> rank2.save()
  140
+>>> rank3 = Ranking(rank=3, author=a1)
  141
+>>> rank3.save()
  142
+
  143
+>>> c1 = Cover(title="first", item=i4)
  144
+>>> c1.save()
  145
+>>> c2 = Cover(title="second", item=i2)
  146
+>>> c2.save()
  147
+
77 148
 Bug #1050
78 149
 >>> Item.objects.filter(tags__isnull=True)
79 150
 [<Item: three>]
@@ -119,7 +190,7 @@ def __unicode__(self):
119 190
 
120 191
 # Create something with a duplicate 'name' so that we can test multi-column
121 192
 # cases (which require some tricky SQL transformations under the covers).
122  
->>> xx = Item(name='four', creator=a2)
  193
+>>> xx = Item(name='four', creator=a2, note=n1)
123 194
 >>> xx.save()
124 195
 >>> Item.objects.exclude(name='two').values('creator', 'name').distinct().count()
125 196
 4
@@ -206,5 +277,40 @@ def __unicode__(self):
206 277
 Bug #2496
207 278
 >>> Item.objects.extra(tables=['queries_author']).select_related().order_by('name')[:1]
208 279
 [<Item: four>]
  280
+
  281
+Bug #2076
  282
+# Ordering on related tables should be possible, even if the table is not
  283
+# otherwise involved.
  284
+>>> Item.objects.order_by('note__note', 'name')
  285
+[<Item: two>, <Item: four>, <Item: one>, <Item: three>]
  286
+
  287
+# Ordering on a related field should use the remote model's default ordering as
  288
+# a final step.
  289
+>>> Author.objects.order_by('extra', '-name')
  290
+[<Author: a2>, <Author: a1>, <Author: a4>, <Author: a3>]
  291
+
  292
+# If the remote model does not have a default ordering, we order by its 'id'
  293
+# field.
  294
+>>> Item.objects.order_by('creator', 'name')
  295
+[<Item: one>, <Item: three>, <Item: two>, <Item: four>]
  296
+
  297
+# Cross model ordering is possible in Meta, too.
  298
+>>> Ranking.objects.all()
  299
+[<Ranking: 3: a1>, <Ranking: 2: a2>, <Ranking: 1: a3>]
  300
+>>> Ranking.objects.all().order_by('rank')
  301
+[<Ranking: 1: a3>, <Ranking: 2: a2>, <Ranking: 3: a1>]
  302
+
  303
+>>> Cover.objects.all()
  304
+[<Cover: first>, <Cover: second>]
  305
+
  306
+Bugs #2874, #3002
  307
+>>> qs = Item.objects.select_related().order_by('note__note', 'name')
  308
+>>> list(qs)
  309
+[<Item: two>, <Item: four>, <Item: one>, <Item: three>]
  310
+
  311
+# This is also a good select_related() test because there are multiple Note
  312
+# entries in the SQL. The two Note items should be different.
  313
+>>> qs[0].note, qs[0].creator.extra.note
  314
+(<Note: n2>, <Note: n1>)
209 315
 """}
210 316
 

0 notes on commit a1d160e

Please sign in to comment.
Something went wrong with that request. Please try again.