Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

[1.2.X] Fixed #14700 - speed up RawQuerySet iterator.

This moves constant work out of the loop, and uses the much faster *args
based model instantiation where possible, to produce very large speed ups.

Thanks to akaariai for the report and patch.

Backport of [14692] from trunk.

git-svn-id: http://code.djangoproject.com/svn/django/branches/releases/1.2.X@14693 bcc190cf-cafb-0310-a4f2-bffc1f526a37
  • Loading branch information...
commit cb9cb65c6833c0477bbae19fd0cfe51e1df050ee 1 parent 8571316
@spookylukey spookylukey authored
Showing with 61 additions and 46 deletions.
  1. +61 −46 django/db/models/query.py
View
107 django/db/models/query.py
@@ -1387,8 +1387,67 @@ def __init__(self, raw_query, model=None, query=None, params=None,
self.translations = translations or {}
def __iter__(self):
- for row in self.query:
- yield self.transform_results(row)
+ # Mapping of attrnames to row column positions. Used for constructing
+ # the model using kwargs, needed when not all model's fields are present
+ # in the query.
+ model_init_field_names = {}
+ # A list of tuples of (column name, column position). Used for
+ # annotation fields.
+ annotation_fields = []
+
+ # Cache some things for performance reasons outside the loop.
+ db = self.db
+ compiler = connections[db].ops.compiler('SQLCompiler')(self.query, connections[db], db)
+ need_resolv_columns = hasattr(compiler, 'resolve_columns')
+
+ # Find out which columns are model's fields, and which ones should be
+ # annotated to the model.
+ for pos, column in enumerate(self.columns):
+ if column in self.model_fields:
+ model_init_field_names[self.model_fields[column].attname] = pos
+ else:
+ annotation_fields.append((column, pos))
+
+ # Find out which model's fields are not present in the query.
+ skip = set()
+ for field in self.model._meta.fields:
+ if field.attname not in model_init_field_names:
+ skip.add(field.attname)
+ if skip:
+ if self.model._meta.pk.attname in skip:
+ raise InvalidQuery('Raw query must include the primary key')
+ model_cls = deferred_class_factory(self.model, skip)
+ else:
+ model_cls = self.model
+ # All model's fields are present in the query. So, it is possible
+ # to use *args based model instantation. For each field of the model,
+ # record the query column position matching that field.
+ model_init_field_pos = []
+ for field in self.model._meta.fields:
+ model_init_field_pos.append(model_init_field_names[field.attname])
+ if need_resolv_columns:
+ fields = [self.model_fields.get(c, None) for c in self.columns]
+ # Begin looping through the query values.
+ for values in self.query:
+ if need_resolv_columns:
+ values = compiler.resolve_columns(values, fields)
+ # Associate fields to values
+ if skip:
+ model_init_kwargs = {}
+ for attname, pos in model_init_field_names.iteritems():
+ model_init_kwargs[attname] = values[pos]
+ instance = model_cls(**model_init_kwargs)
+ else:
+ model_init_args = [values[pos] for pos in model_init_field_pos]
+ instance = model_cls(*model_init_args)
+ if annotation_fields:
+ for column, pos in annotation_fields:
+ setattr(instance, column, values[pos])
+
+ instance._state.db = db
+ instance._state.adding = False
+
+ yield instance
def __repr__(self):
return "<RawQuerySet: %r>" % (self.raw_query % self.params)
@@ -1443,50 +1502,6 @@ def model_fields(self):
self._model_fields[converter(column)] = field
return self._model_fields
- def transform_results(self, values):
- model_init_kwargs = {}
- annotations = ()
-
- # Perform database backend type resolution
- connection = connections[self.db]
- compiler = connection.ops.compiler('SQLCompiler')(self.query, connection, self.db)
- if hasattr(compiler, 'resolve_columns'):
- fields = [self.model_fields.get(c,None) for c in self.columns]
- values = compiler.resolve_columns(values, fields)
-
- # Associate fields to values
- for pos, value in enumerate(values):
- column = self.columns[pos]
-
- # Separate properties from annotations
- if column in self.model_fields.keys():
- model_init_kwargs[self.model_fields[column].attname] = value
- else:
- annotations += (column, value),
-
- # Construct model instance and apply annotations
- skip = set()
- for field in self.model._meta.fields:
- if field.attname not in model_init_kwargs.keys():
- skip.add(field.attname)
-
- if skip:
- if self.model._meta.pk.attname in skip:
- raise InvalidQuery('Raw query must include the primary key')
- model_cls = deferred_class_factory(self.model, skip)
- else:
- model_cls = self.model
-
- instance = model_cls(**model_init_kwargs)
-
- for field, value in annotations:
- setattr(instance, field, value)
-
- instance._state.db = self.query.using
- instance._state.adding = False
-
- return instance
-
def insert_query(model, values, return_id=False, raw_values=False, using=None):
"""
Inserts a new record for the given model. This provides an interface to
Please sign in to comment.
Something went wrong with that request. Please try again.