Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Fixed #3566 -- Added support for aggregation to the ORM. See the docu…

…mentation for details on usage.

Many thanks to:
 * Nicolas Lara, who worked on this feature during the 2008 Google Summer of Code.
 * Alex Gaynor for his help debugging and fixing a number of issues.
 * Justin Bronn for his help integrating with contrib.gis.
 * Karen Tracey for her help with cross-platform testing.
 * Ian Kelly for his help testing and fixing Oracle support.
 * Malcolm Tredinnick for his invaluable review notes.

git-svn-id: http://code.djangoproject.com/svn/django/trunk@9742 bcc190cf-cafb-0310-a4f2-bffc1f526a37
  • Loading branch information...
commit cc4e4d9aee0b3ebfb45bee01aec79edc9e144c78 1 parent 50a293a
@freakboy3742 freakboy3742 authored
Showing with 2,361 additions and 329 deletions.
  1. +1 −0  AUTHORS
  2. +10 −0 django/contrib/gis/db/models/aggregates.py
  3. +68 −121 django/contrib/gis/db/models/query.py
  4. +36 −0 django/contrib/gis/db/models/sql/aggregates.py
  5. +85 −45 django/contrib/gis/db/models/sql/query.py
  6. +23 −0 django/db/backends/__init__.py
  7. +1 −0  django/db/backends/mysql/base.py
  8. +9 −11 django/db/backends/oracle/query.py
  9. +21 −1 django/db/backends/sqlite3/base.py
  10. +1 −0  django/db/models/__init__.py
  11. +66 −0 django/db/models/aggregates.py
  12. +6 −0 django/db/models/manager.py
  13. +82 −7 django/db/models/query.py
  14. +0 −1  django/db/models/query_utils.py
  15. +130 −0 django/db/models/sql/aggregates.py
  16. +0 −53 django/db/models/sql/datastructures.py
  17. +239 −68 django/db/models/sql/query.py
  18. +20 −10 django/db/models/sql/subqueries.py
  19. +15 −10 django/test/testcases.py
  20. +1 −1  docs/index.txt
  21. +1 −1  docs/ref/models/index.txt
  22. +186 −0 docs/ref/models/querysets.txt
  23. +323 −0 docs/topics/db/aggregation.txt
  24. +1 −0  docs/topics/db/index.txt
  25. 0  tests/modeltests/aggregation/__init__.py
  26. +229 −0 tests/modeltests/aggregation/fixtures/initial_data.json
  27. +379 −0 tests/modeltests/aggregation/models.py
  28. 0  tests/regressiontests/aggregation_regress/__init__.py
  29. +229 −0 tests/regressiontests/aggregation_regress/fixtures/initial_data.json
  30. +199 −0 tests/regressiontests/aggregation_regress/models.py
View
1  AUTHORS
@@ -31,6 +31,7 @@ answer newbie questions, and generally made Django that much better:
AgarFu <heaven@croasanaso.sytes.net>
Dagur Páll Ammendrup <dagurp@gmail.com>
Collin Anderson <cmawebsite@gmail.com>
+ Nicolas Lara <nicolaslara@gmail.com>
Jeff Anderson <jefferya@programmerq.net>
Marian Andre <django@andre.sk>
Andreas
View
10 django/contrib/gis/db/models/aggregates.py
@@ -0,0 +1,10 @@
+from django.db.models import Aggregate
+
+class Extent(Aggregate):
+ name = 'Extent'
+
+class MakeLine(Aggregate):
+ name = 'MakeLine'
+
+class Union(Aggregate):
+ name = 'Union'
View
189 django/contrib/gis/db/models/query.py
@@ -3,6 +3,7 @@
from django.db.models.query import sql, QuerySet, Q
from django.contrib.gis.db.backend import SpatialBackend
+from django.contrib.gis.db.models import aggregates
from django.contrib.gis.db.models.fields import GeometryField, PointField
from django.contrib.gis.db.models.sql import AreaField, DistanceField, GeomField, GeoQuery, GeoWhereNode
from django.contrib.gis.measure import Area, Distance
@@ -17,7 +18,7 @@ class GeomSQL(object):
"Simple wrapper object for geometric SQL."
def __init__(self, geo_sql):
self.sql = geo_sql
-
+
def as_sql(self, *args, **kwargs):
return self.sql
@@ -30,7 +31,7 @@ def __init__(self, model=None, query=None):
def area(self, tolerance=0.05, **kwargs):
"""
- Returns the area of the geographic field in an `area` attribute on
+ Returns the area of the geographic field in an `area` attribute on
each element of this GeoQuerySet.
"""
# Peforming setup here rather than in `_spatial_attribute` so that
@@ -75,21 +76,21 @@ def distance(self, geom, **kwargs):
Keyword Arguments:
`spheroid` => If the geometry field is geodetic and PostGIS is
- the spatial database, then the more accurate
+ the spatial database, then the more accurate
spheroid calculation will be used instead of the
quicker sphere calculation.
-
- `tolerance` => Used only for Oracle. The tolerance is
- in meters -- a default of 5 centimeters (0.05)
+
+ `tolerance` => Used only for Oracle. The tolerance is
+ in meters -- a default of 5 centimeters (0.05)
is used.
"""
return self._distance_attribute('distance', geom, **kwargs)
def envelope(self, **kwargs):
"""
- Returns a Geometry representing the bounding box of the
+ Returns a Geometry representing the bounding box of the
Geometry field in an `envelope` attribute on each element of
- the GeoQuerySet.
+ the GeoQuerySet.
"""
return self._geom_attribute('envelope', **kwargs)
@@ -98,20 +99,7 @@ def extent(self, **kwargs):
Returns the extent (aggregate) of the features in the GeoQuerySet. The
extent will be returned as a 4-tuple, consisting of (xmin, ymin, xmax, ymax).
"""
- convert_extent = None
- if SpatialBackend.postgis:
- def convert_extent(box, geo_field):
- # TODO: Parsing of BOX3D, Oracle support (patches welcome!)
- # Box text will be something like "BOX(-90.0 30.0, -85.0 40.0)";
- # parsing out and returning as a 4-tuple.
- ll, ur = box[4:-1].split(',')
- xmin, ymin = map(float, ll.split())
- xmax, ymax = map(float, ur.split())
- return (xmin, ymin, xmax, ymax)
- elif SpatialBackend.oracle:
- def convert_extent(wkt, geo_field):
- raise NotImplementedError
- return self._spatial_aggregate('extent', convert_func=convert_extent, **kwargs)
+ return self._spatial_aggregate(aggregates.Extent, **kwargs)
def gml(self, precision=8, version=2, **kwargs):
"""
@@ -120,7 +108,7 @@ def gml(self, precision=8, version=2, **kwargs):
"""
s = {'desc' : 'GML', 'procedure_args' : {'precision' : precision}}
if SpatialBackend.postgis:
- # PostGIS AsGML() aggregate function parameter order depends on the
+ # PostGIS AsGML() aggregate function parameter order depends on the
# version -- uggh.
major, minor1, minor2 = SpatialBackend.version
if major >= 1 and (minor1 > 3 or (minor1 == 3 and minor2 > 1)):
@@ -163,9 +151,7 @@ def make_line(self, **kwargs):
this GeoQuerySet and returns it. This is a spatial aggregate
method, and thus returns a geometry rather than a GeoQuerySet.
"""
- kwargs['geo_field_type'] = PointField
- kwargs['agg_field'] = GeometryField
- return self._spatial_aggregate('make_line', **kwargs)
+ return self._spatial_aggregate(aggregates.MakeLine, geo_field_type=PointField, **kwargs)
def mem_size(self, **kwargs):
"""
@@ -185,7 +171,7 @@ def num_geom(self, **kwargs):
def num_points(self, **kwargs):
"""
- Returns the number of points in the first linestring in the
+ Returns the number of points in the first linestring in the
Geometry field in a `num_points` attribute on each element of
this GeoQuerySet; otherwise sets with None.
"""
@@ -231,7 +217,7 @@ def svg(self, **kwargs):
def sym_difference(self, geom, **kwargs):
"""
- Returns the symmetric difference of the geographic field in a
+ Returns the symmetric difference of the geographic field in a
`sym_difference` attribute on each element of this GeoQuerySet.
"""
return self._geomset_attribute('sym_difference', geom, **kwargs)
@@ -265,7 +251,7 @@ def transform(self, srid=4326, **kwargs):
# when there's also a transformation we need to cascade the substitutions.
# For example, 'SDO_UTIL.TO_WKTGEOMETRY(SDO_CS.TRANSFORM( ... )'
geo_col = self.query.custom_select.get(geo_field, field_col)
-
+
# Setting the key for the field's column with the custom SELECT SQL to
# override the geometry column returned from the database.
custom_sel = '%s(%s, %s)' % (SpatialBackend.transform, geo_col, srid)
@@ -288,11 +274,10 @@ def unionagg(self, **kwargs):
None if the GeoQuerySet is empty. The `tolerance` keyword is for
Oracle backends only.
"""
- kwargs['agg_field'] = GeometryField
- return self._spatial_aggregate('unionagg', **kwargs)
+ return self._spatial_aggregate(aggregates.Union, **kwargs)
### Private API -- Abstracted DRY routines. ###
- def _spatial_setup(self, att, aggregate=False, desc=None, field_name=None, geo_field_type=None):
+ def _spatial_setup(self, att, desc=None, field_name=None, geo_field_type=None):
"""
Performs set up for executing the spatial function.
"""
@@ -301,86 +286,52 @@ def _spatial_setup(self, att, aggregate=False, desc=None, field_name=None, geo_f
if desc is None: desc = att
if not func: raise ImproperlyConfigured('%s stored procedure not available.' % desc)
- # Initializing the procedure arguments.
+ # Initializing the procedure arguments.
procedure_args = {'function' : func}
-
- # Is there a geographic field in the model to perform this
+
+ # Is there a geographic field in the model to perform this
# operation on?
geo_field = self.query._geo_field(field_name)
if not geo_field:
raise TypeError('%s output only available on GeometryFields.' % func)
- # If the `geo_field_type` keyword was used, then enforce that
+ # If the `geo_field_type` keyword was used, then enforce that
# type limitation.
- if not geo_field_type is None and not isinstance(geo_field, geo_field_type):
- raise TypeError('"%s" stored procedures may only be called on %ss.' % (func, geo_field_type.__name__))
+ if not geo_field_type is None and not isinstance(geo_field, geo_field_type):
+ raise TypeError('"%s" stored procedures may only be called on %ss.' % (func, geo_field_type.__name__))
# Setting the procedure args.
- procedure_args['geo_col'] = self._geocol_select(geo_field, field_name, aggregate)
+ procedure_args['geo_col'] = self._geocol_select(geo_field, field_name)
return procedure_args, geo_field
- def _spatial_aggregate(self, att, field_name=None,
- agg_field=None, convert_func=None,
- geo_field_type=None, tolerance=0.0005):
+ def _spatial_aggregate(self, aggregate, field_name=None,
+ geo_field_type=None, tolerance=0.05):
"""
DRY routine for calling aggregate spatial stored procedures and
returning their result to the caller of the function.
"""
- # Constructing the setup keyword arguments.
- setup_kwargs = {'aggregate' : True,
- 'field_name' : field_name,
- 'geo_field_type' : geo_field_type,
- }
- procedure_args, geo_field = self._spatial_setup(att, **setup_kwargs)
-
- if SpatialBackend.oracle:
- procedure_args['tolerance'] = tolerance
- # Adding in selection SQL for Oracle geometry columns.
- if agg_field is GeometryField:
- agg_sql = '%s' % SpatialBackend.select
- else:
- agg_sql = '%s'
- agg_sql = agg_sql % ('%(function)s(SDOAGGRTYPE(%(geo_col)s,%(tolerance)s))' % procedure_args)
- else:
- agg_sql = '%(function)s(%(geo_col)s)' % procedure_args
-
- # Wrapping our selection SQL in `GeomSQL` to bypass quoting, and
- # specifying the type of the aggregate field.
- self.query.select = [GeomSQL(agg_sql)]
- self.query.select_fields = [agg_field]
-
- try:
- # `asql` => not overriding `sql` module.
- asql, params = self.query.as_sql()
- except sql.datastructures.EmptyResultSet:
- return None
-
- # Getting a cursor, executing the query, and extracting the returned
- # value from the aggregate function.
- cursor = connection.cursor()
- cursor.execute(asql, params)
- result = cursor.fetchone()[0]
-
- # If the `agg_field` is specified as a GeometryField, then autmatically
- # set up the conversion function.
- if agg_field is GeometryField and not callable(convert_func):
- if SpatialBackend.postgis:
- def convert_geom(hex, geo_field):
- if hex: return SpatialBackend.Geometry(hex)
- else: return None
- elif SpatialBackend.oracle:
- def convert_geom(clob, geo_field):
- if clob: return SpatialBackend.Geometry(clob.read(), geo_field._srid)
- else: return None
- convert_func = convert_geom
-
- # Returning the callback function evaluated on the result culled
- # from the executed cursor.
- if callable(convert_func):
- return convert_func(result, geo_field)
- else:
- return result
+ # Getting the field the geographic aggregate will be called on.
+ geo_field = self.query._geo_field(field_name)
+ if not geo_field:
+ raise TypeError('%s aggregate only available on GeometryFields.' % aggregate.name)
+
+ # Checking if there are any geo field type limitations on this
+ # aggregate (e.g. ST_Makeline only operates on PointFields).
+ if not geo_field_type is None and not isinstance(geo_field, geo_field_type):
+ raise TypeError('%s aggregate may only be called on %ss.' % (aggregate.name, geo_field_type.__name__))
+
+ # Getting the string expression of the field name, as this is the
+ # argument taken by `Aggregate` objects.
+ agg_col = field_name or geo_field.name
+
+ # Adding any keyword parameters for the Aggregate object. Oracle backends
+ # in particular need an additional `tolerance` parameter.
+ agg_kwargs = {}
+ if SpatialBackend.oracle: agg_kwargs['tolerance'] = tolerance
+
+ # Calling the QuerySet.aggregate, and returning only the value of the aggregate.
+ return self.aggregate(_geoagg=aggregate(agg_col, **agg_kwargs))['_geoagg']
def _spatial_attribute(self, att, settings, field_name=None, model_att=None):
"""
@@ -393,7 +344,7 @@ def _spatial_attribute(self, att, settings, field_name=None, model_att=None):
SQL function to call.
settings:
- Dictonary of internal settings to customize for the spatial procedure.
+ Dictonary of internal settings to customize for the spatial procedure.
Public Keyword Arguments:
@@ -420,7 +371,7 @@ def _spatial_attribute(self, att, settings, field_name=None, model_att=None):
for k, v in default_args.iteritems(): settings['procedure_args'].setdefault(k, v)
else:
geo_field = settings['geo_field']
-
+
# The attribute to attach to the model.
if not isinstance(model_att, basestring): model_att = att
@@ -429,7 +380,7 @@ def _spatial_attribute(self, att, settings, field_name=None, model_att=None):
# Using the field's get_db_prep_lookup() to get any needed
# transformation SQL -- we pass in a 'dummy' `contains` lookup.
where, params = geo_field.get_db_prep_lookup('contains', settings['procedure_args'][name])
- # Replacing the procedure format with that of any needed
+ # Replacing the procedure format with that of any needed
# transformation SQL.
old_fmt = '%%(%s)s' % name
new_fmt = where[0] % '%%s'
@@ -438,7 +389,7 @@ def _spatial_attribute(self, att, settings, field_name=None, model_att=None):
# Getting the format for the stored procedure.
fmt = '%%(function)s(%s)' % settings['procedure_fmt']
-
+
# If the result of this function needs to be converted.
if settings.get('select_field', False):
sel_fld = settings['select_field']
@@ -446,10 +397,10 @@ def _spatial_attribute(self, att, settings, field_name=None, model_att=None):
self.query.custom_select[model_att] = SpatialBackend.select
self.query.extra_select_fields[model_att] = sel_fld
- # Finally, setting the extra selection attribute with
+ # Finally, setting the extra selection attribute with
# the format string expanded with the stored procedure
# arguments.
- return self.extra(select={model_att : fmt % settings['procedure_args']},
+ return self.extra(select={model_att : fmt % settings['procedure_args']},
select_params=settings['select_params'])
def _distance_attribute(self, func, geom=None, tolerance=0.05, spheroid=False, **kwargs):
@@ -471,10 +422,10 @@ def _distance_attribute(self, func, geom=None, tolerance=0.05, spheroid=False, *
distance = func == 'distance'
length = func == 'length'
perimeter = func == 'perimeter'
- if not (distance or length or perimeter):
+ if not (distance or length or perimeter):
raise ValueError('Unknown distance function: %s' % func)
- # The field's get_db_prep_lookup() is used to get any
+ # The field's get_db_prep_lookup() is used to get any
# extra distance parameters. Here we set up the
# parameters that will be passed in to field's function.
lookup_params = [geom or 'POINT (0 0)', 0]
@@ -482,12 +433,12 @@ def _distance_attribute(self, func, geom=None, tolerance=0.05, spheroid=False, *
# If the spheroid calculation is desired, either by the `spheroid`
# keyword or wehn calculating the length of geodetic field, make
# sure the 'spheroid' distance setting string is passed in so we
- # get the correct spatial stored procedure.
- if spheroid or (SpatialBackend.postgis and geo_field.geodetic and length):
- lookup_params.append('spheroid')
+ # get the correct spatial stored procedure.
+ if spheroid or (SpatialBackend.postgis and geo_field.geodetic and length):
+ lookup_params.append('spheroid')
where, params = geo_field.get_db_prep_lookup('distance_lte', lookup_params)
- # The `geom_args` flag is set to true if a geometry parameter was
+ # The `geom_args` flag is set to true if a geometry parameter was
# passed in.
geom_args = bool(geom)
@@ -505,7 +456,7 @@ def _distance_attribute(self, func, geom=None, tolerance=0.05, spheroid=False, *
geodetic = unit_name in geo_field.geodetic_units
else:
geodetic = geo_field.geodetic
-
+
if distance:
if self.query.transformed_srid:
# Setting the `geom_args` flag to false because we want to handle
@@ -515,7 +466,7 @@ def _distance_attribute(self, func, geom=None, tolerance=0.05, spheroid=False, *
geom_args = False
procedure_fmt = '%s(%%(geo_col)s, %s)' % (SpatialBackend.transform, self.query.transformed_srid)
if geom.srid is None or geom.srid == self.query.transformed_srid:
- # If the geom parameter srid is None, it is assumed the coordinates
+ # If the geom parameter srid is None, it is assumed the coordinates
# are in the transformed units. A placeholder is used for the
# geometry parameter.
procedure_fmt += ', %%s'
@@ -529,10 +480,10 @@ def _distance_attribute(self, func, geom=None, tolerance=0.05, spheroid=False, *
if geodetic:
# Spherical distance calculation is needed (because the geographic
- # field is geodetic). However, the PostGIS ST_distance_sphere/spheroid()
+ # field is geodetic). However, the PostGIS ST_distance_sphere/spheroid()
# procedures may only do queries from point columns to point geometries
# some error checking is required.
- if not isinstance(geo_field, PointField):
+ if not isinstance(geo_field, PointField):
raise TypeError('Spherical distance calculation only supported on PointFields.')
if not str(SpatialBackend.Geometry(buffer(params[0].wkb)).geom_type) == 'Point':
raise TypeError('Spherical distance calculation only supported with Point Geometry parameters')
@@ -553,12 +504,12 @@ def _distance_attribute(self, func, geom=None, tolerance=0.05, spheroid=False, *
# Setting up the settings for `_spatial_attribute`.
s = {'select_field' : DistanceField(dist_att),
- 'setup' : False,
+ 'setup' : False,
'geo_field' : geo_field,
'procedure_args' : procedure_args,
'procedure_fmt' : procedure_fmt,
}
- if geom_args:
+ if geom_args:
s['geom_args'] = ('geom',)
s['procedure_args']['geom'] = geom
elif geom:
@@ -577,12 +528,12 @@ def _geom_attribute(self, func, tolerance=0.05, **kwargs):
s['procedure_fmt'] = '%(geo_col)s,%(tolerance)s'
s['procedure_args'] = {'tolerance' : tolerance}
return self._spatial_attribute(func, s, **kwargs)
-
+
def _geomset_attribute(self, func, geom, tolerance=0.05, **kwargs):
"""
DRY routine for setting up a GeoQuerySet method that attaches a
Geometry attribute and takes a Geoemtry parameter. This is used
- for geometry set-like operations (e.g., intersection, difference,
+ for geometry set-like operations (e.g., intersection, difference,
union, sym_difference).
"""
s = {'geom_args' : ('geom',),
@@ -595,16 +546,12 @@ def _geomset_attribute(self, func, geom, tolerance=0.05, **kwargs):
s['procedure_args']['tolerance'] = tolerance
return self._spatial_attribute(func, s, **kwargs)
- def _geocol_select(self, geo_field, field_name, aggregate=False):
+ def _geocol_select(self, geo_field, field_name):
"""
Helper routine for constructing the SQL to select the geographic
column. Takes into account if the geographic field is in a
ForeignKey relation to the current model.
"""
- # If this is an aggregate spatial query, the flag needs to be
- # set on the `GeoQuery` object of this queryset.
- if aggregate: self.query.aggregate = True
-
opts = self.model._meta
if not geo_field in opts.fields:
# Is this operation going to be on a related geographic field?
View
36 django/contrib/gis/db/models/sql/aggregates.py
@@ -0,0 +1,36 @@
+from django.db.models.sql.aggregates import *
+
+from django.contrib.gis.db.models.fields import GeometryField
+from django.contrib.gis.db.backend import SpatialBackend
+
+if SpatialBackend.oracle:
+ geo_template = '%(function)s(SDOAGGRTYPE(%(field)s,%(tolerance)s))'
+else:
+ geo_template = '%(function)s(%(field)s)'
+
+class GeoAggregate(Aggregate):
+ # Overriding the SQL template with the geographic one.
+ sql_template = geo_template
+
+ is_extent = False
+
+ def __init__(self, col, source=None, is_summary=False, **extra):
+ super(GeoAggregate, self).__init__(col, source, is_summary, **extra)
+
+ # Can't use geographic aggregates on non-geometry fields.
+ if not isinstance(self.source, GeometryField):
+ raise ValueError('Geospatial aggregates only allowed on geometry fields.')
+
+ # Making sure the SQL function is available for this spatial backend.
+ if not self.sql_function:
+ raise NotImplementedError('This aggregate functionality not implemented for your spatial backend.')
+
+class Extent(GeoAggregate):
+ is_extent = True
+ sql_function = SpatialBackend.extent
+
+class MakeLine(GeoAggregate):
+ sql_function = SpatialBackend.make_line
+
+class Union(GeoAggregate):
+ sql_function = SpatialBackend.unionagg
View
130 django/contrib/gis/db/models/sql/query.py
@@ -5,6 +5,7 @@
from django.contrib.gis.db.backend import SpatialBackend
from django.contrib.gis.db.models.fields import GeometryField
+from django.contrib.gis.db.models.sql import aggregates as gis_aggregates_module
from django.contrib.gis.db.models.sql.where import GeoWhereNode
from django.contrib.gis.measure import Area, Distance
@@ -12,12 +13,35 @@
ALL_TERMS = sql.constants.QUERY_TERMS.copy()
ALL_TERMS.update(SpatialBackend.gis_terms)
+# Conversion functions used in normalizing geographic aggregates.
+if SpatialBackend.postgis:
+ def convert_extent(box):
+ # TODO: Parsing of BOX3D, Oracle support (patches welcome!)
+ # Box text will be something like "BOX(-90.0 30.0, -85.0 40.0)";
+ # parsing out and returning as a 4-tuple.
+ ll, ur = box[4:-1].split(',')
+ xmin, ymin = map(float, ll.split())
+ xmax, ymax = map(float, ur.split())
+ return (xmin, ymin, xmax, ymax)
+
+ def convert_geom(hex, geo_field):
+ if hex: return SpatialBackend.Geometry(hex)
+ else: return None
+else:
+ def convert_extent(box):
+ raise NotImplementedError('Aggregate extent not implemented for this spatial backend.')
+
+ def convert_geom(clob, geo_field):
+ if clob: return SpatialBackend.Geometry(clob.read(), geo_field._srid)
+ else: return None
+
class GeoQuery(sql.Query):
"""
A single spatial SQL query.
"""
# Overridding the valid query terms.
query_terms = ALL_TERMS
+ aggregates_module = gis_aggregates_module
#### Methods overridden from the base Query class ####
def __init__(self, model, conn):
@@ -25,7 +49,6 @@ def __init__(self, model, conn):
# The following attributes are customized for the GeoQuerySet.
# The GeoWhereNode and SpatialBackend classes contain backend-specific
# routines and functions.
- self.aggregate = False
self.custom_select = {}
self.transformed_srid = None
self.extra_select_fields = {}
@@ -34,7 +57,6 @@ def clone(self, *args, **kwargs):
obj = super(GeoQuery, self).clone(*args, **kwargs)
# Customized selection dictionary and transformed srid flag have
# to also be added to obj.
- obj.aggregate = self.aggregate
obj.custom_select = self.custom_select.copy()
obj.transformed_srid = self.transformed_srid
obj.extra_select_fields = self.extra_select_fields.copy()
@@ -50,12 +72,12 @@ def get_columns(self, with_aliases=False):
(without the table names) are given unique aliases. This is needed in
some cases to avoid ambiguitity with nested queries.
- This routine is overridden from Query to handle customized selection of
+ This routine is overridden from Query to handle customized selection of
geometry columns.
"""
qn = self.quote_name_unless_alias
qn2 = self.connection.ops.quote_name
- result = ['(%s) AS %s' % (self.get_extra_select_format(alias) % col[0], qn2(alias))
+ result = ['(%s) AS %s' % (self.get_extra_select_format(alias) % col[0], qn2(alias))
for alias, col in self.extra_select.iteritems()]
aliases = set(self.extra_select.keys())
if with_aliases:
@@ -67,38 +89,53 @@ def get_columns(self, with_aliases=False):
for col, field in izip(self.select, self.select_fields):
if isinstance(col, (list, tuple)):
r = self.get_field_select(field, col[0])
- if with_aliases and col[1] in col_aliases:
- c_alias = 'Col%d' % len(col_aliases)
- result.append('%s AS %s' % (r, c_alias))
- aliases.add(c_alias)
- col_aliases.add(c_alias)
+ if with_aliases:
+ if col[1] in col_aliases:
+ c_alias = 'Col%d' % len(col_aliases)
+ result.append('%s AS %s' % (r, c_alias))
+ aliases.add(c_alias)
+ col_aliases.add(c_alias)
+ else:
+ result.append('%s AS %s' % (r, col[1]))
+ aliases.add(r)
+ col_aliases.add(col[1])
else:
result.append(r)
aliases.add(r)
col_aliases.add(col[1])
else:
result.append(col.as_sql(quote_func=qn))
+
if hasattr(col, 'alias'):
aliases.add(col.alias)
col_aliases.add(col.alias)
+
elif self.default_cols:
cols, new_aliases = self.get_default_columns(with_aliases,
col_aliases)
result.extend(cols)
aliases.update(new_aliases)
+
+ result.extend([
+ '%s%s' % (
+ aggregate.as_sql(quote_func=qn),
+ alias is not None and ' AS %s' % alias or ''
+ )
+ for alias, aggregate in self.aggregate_select.items()
+ ])
+
# This loop customized for GeoQuery.
- if not self.aggregate:
- for (table, col), field in izip(self.related_select_cols, self.related_select_fields):
- r = self.get_field_select(field, table)
- if with_aliases and col in col_aliases:
- c_alias = 'Col%d' % len(col_aliases)
- result.append('%s AS %s' % (r, c_alias))
- aliases.add(c_alias)
- col_aliases.add(c_alias)
- else:
- result.append(r)
- aliases.add(r)
- col_aliases.add(col)
+ for (table, col), field in izip(self.related_select_cols, self.related_select_fields):
+ r = self.get_field_select(field, table)
+ if with_aliases and col in col_aliases:
+ c_alias = 'Col%d' % len(col_aliases)
+ result.append('%s AS %s' % (r, c_alias))
+ aliases.add(c_alias)
+ col_aliases.add(c_alias)
+ else:
+ result.append(r)
+ aliases.add(r)
+ col_aliases.add(col)
self._select_aliases = aliases
return result
@@ -112,7 +149,7 @@ def get_default_columns(self, with_aliases=False, col_aliases=None,
Returns a list of strings, quoted appropriately for use in SQL
directly, as well as a set of aliases used in the select statement.
- This routine is overridden from Query to handle customized selection of
+ This routine is overridden from Query to handle customized selection of
geometry columns.
"""
result = []
@@ -154,20 +191,10 @@ def get_default_columns(self, with_aliases=False, col_aliases=None,
return result, None
return result, aliases
- def get_ordering(self):
- """
- This routine is overridden to disable ordering for aggregate
- spatial queries.
- """
- if not self.aggregate:
- return super(GeoQuery, self).get_ordering()
- else:
- return ()
-
def resolve_columns(self, row, fields=()):
"""
This routine is necessary so that distances and geometries returned
- from extra selection SQL get resolved appropriately into Python
+ from extra selection SQL get resolved appropriately into Python
objects.
"""
values = []
@@ -183,7 +210,7 @@ def resolve_columns(self, row, fields=()):
# Converting any extra selection values (e.g., geometries and
# distance objects added by GeoQuerySet methods).
- values = [self.convert_values(v, self.extra_select_fields.get(a, None))
+ values = [self.convert_values(v, self.extra_select_fields.get(a, None))
for v, a in izip(row[rn_offset:index_start], aliases)]
if SpatialBackend.oracle:
# This is what happens normally in OracleQuery's `resolve_columns`.
@@ -212,6 +239,19 @@ def convert_values(self, value, field):
value = SpatialBackend.Geometry(value)
return value
+ def resolve_aggregate(self, value, aggregate):
+ """
+ Overridden from GeoQuery's normalize to handle the conversion of
+ GeoAggregate objects.
+ """
+ if isinstance(aggregate, self.aggregates_module.GeoAggregate):
+ if aggregate.is_extent:
+ return convert_extent(value)
+ else:
+ return convert_geom(value, aggregate.source)
+ else:
+ return super(GeoQuery, self).resolve_aggregate(value, aggregate)
+
#### Routines unique to GeoQuery ####
def get_extra_select_format(self, alias):
sel_fmt = '%s'
@@ -222,9 +262,9 @@ def get_extra_select_format(self, alias):
def get_field_select(self, fld, alias=None):
"""
Returns the SELECT SQL string for the given field. Figures out
- if any custom selection SQL is needed for the column The `alias`
- keyword may be used to manually specify the database table where
- the column exists, if not in the model associated with this
+ if any custom selection SQL is needed for the column The `alias`
+ keyword may be used to manually specify the database table where
+ the column exists, if not in the model associated with this
`GeoQuery`.
"""
sel_fmt = self.get_select_format(fld)
@@ -263,15 +303,15 @@ def _check_geo_field(self, model, name_param):
"""
Recursive utility routine for checking the given name parameter
on the given model. Initially, the name parameter is a string,
- of the field on the given model e.g., 'point', 'the_geom'.
- Related model field strings like 'address__point', may also be
+ of the field on the given model e.g., 'point', 'the_geom'.
+ Related model field strings like 'address__point', may also be
used.
- If a GeometryField exists according to the given name parameter
+ If a GeometryField exists according to the given name parameter
it will be returned, otherwise returns False.
"""
if isinstance(name_param, basestring):
- # This takes into account the situation where the name is a
+ # This takes into account the situation where the name is a
# lookup to a related geographic field, e.g., 'address__point'.
name_param = name_param.split(sql.constants.LOOKUP_SEP)
name_param.reverse() # Reversing so list operates like a queue of related lookups.
@@ -284,7 +324,7 @@ def _check_geo_field(self, model, name_param):
except (FieldDoesNotExist, IndexError):
return False
# TODO: ManyToManyField?
- if isinstance(fld, GeometryField):
+ if isinstance(fld, GeometryField):
return fld # A-OK.
elif isinstance(fld, ForeignKey):
# ForeignKey encountered, return the output of this utility called
@@ -297,12 +337,12 @@ def _field_column(self, field, table_alias=None):
"""
Helper function that returns the database column for the given field.
The table and column are returned (quoted) in the proper format, e.g.,
- `"geoapp_city"."point"`. If `table_alias` is not specified, the
+ `"geoapp_city"."point"`. If `table_alias` is not specified, the
database table associated with the model of this `GeoQuery` will be
used.
"""
if table_alias is None: table_alias = self.model._meta.db_table
- return "%s.%s" % (self.quote_name_unless_alias(table_alias),
+ return "%s.%s" % (self.quote_name_unless_alias(table_alias),
self.connection.ops.quote_name(field.column))
def _geo_field(self, field_name=None):
@@ -333,5 +373,5 @@ def __init__(self, distance_att):
# Rather than use GeometryField (which requires a SQL query
# upon instantiation), use this lighter weight class.
-class GeomField(object):
+class GeomField(object):
pass
View
23 django/db/backends/__init__.py
@@ -10,6 +10,12 @@
# Python 2.3 compat
from sets import Set as set
+try:
+ import decimal
+except ImportError:
+ # Python 2.3 fallback
+ from django.utils import _decimal as decimal
+
from django.db.backends import util
from django.utils import datetime_safe
@@ -62,6 +68,7 @@ def make_debug_cursor(self, cursor):
return util.CursorDebugWrapper(cursor, self)
class BaseDatabaseFeatures(object):
+ allows_group_by_pk = False
# True if django.db.backend.utils.typecast_timestamp is used on values
# returned from dates() calls.
needs_datetime_string_cast = True
@@ -376,6 +383,22 @@ def year_lookup_bounds_for_date_field(self, value):
"""
return self.year_lookup_bounds(value)
+ def convert_values(self, value, field):
+ """Coerce the value returned by the database backend into a consistent type that
+ is compatible with the field type.
+ """
+ internal_type = field.get_internal_type()
+ if internal_type == 'DecimalField':
+ return value
+ elif internal_type and internal_type.endswith('IntegerField') or internal_type == 'AutoField':
+ return int(value)
+ elif internal_type in ('DateField', 'DateTimeField', 'TimeField'):
+ return value
+ # No field, or the field isn't known to be a decimal or integer
+ # Default to a float
+ return float(value)
+
+
class BaseDatabaseIntrospection(object):
"""
This class encapsulates all backend-specific introspection utilities
View
1  django/db/backends/mysql/base.py
@@ -110,6 +110,7 @@ def __iter__(self):
class DatabaseFeatures(BaseDatabaseFeatures):
empty_fetchmany_value = ()
update_can_self_select = False
+ allows_group_by_pk = True
related_fields_match_type = True
class DatabaseOperations(BaseDatabaseOperations):
View
20 django/db/backends/oracle/query.py
@@ -53,21 +53,23 @@ def resolve_columns(self, row, fields=()):
return values
def convert_values(self, value, field):
- from django.db.models.fields import DateField, DateTimeField, \
- TimeField, BooleanField, NullBooleanField, DecimalField, Field
+ from django.db.models.fields import Field
if isinstance(value, Database.LOB):
value = value.read()
# Oracle stores empty strings as null. We need to undo this in
# order to adhere to the Django convention of using the empty
# string instead of null, but only if the field accepts the
# empty string.
- if value is None and isinstance(field, Field) and field.empty_strings_allowed:
+ if value is None and field and field.empty_strings_allowed:
value = u''
# Convert 1 or 0 to True or False
- elif value in (1, 0) and isinstance(field, (BooleanField, NullBooleanField)):
+ elif value in (1, 0) and field and field.get_internal_type() in ('BooleanField', 'NullBooleanField'):
value = bool(value)
+ # Force floats to the correct type
+ elif value is not None and field and field.get_internal_type() == 'FloatField':
+ value = float(value)
# Convert floats to decimals
- elif value is not None and isinstance(field, DecimalField):
+ elif value is not None and field and field.get_internal_type() == 'DecimalField':
value = util.typecast_decimal(field.format_number(value))
# cx_Oracle always returns datetime.datetime objects for
# DATE and TIMESTAMP columns, but Django wants to see a
@@ -86,13 +88,9 @@ def convert_values(self, value, field):
value = datetime.datetime(value.year, value.month,
value.day, value.hour, value.minute, value.second,
value.fsecond)
- if isinstance(field, DateTimeField):
- # DateTimeField subclasses DateField so must be checked
- # first.
- pass
- elif isinstance(field, DateField):
+ if field and field.get_internal_type() == 'DateField':
value = value.date()
- elif isinstance(field, TimeField) or (value.year == 1900 and value.month == value.day == 1):
+ elif field and field.get_internal_type() == 'TimeField' or (value.year == 1900 and value.month == value.day == 1):
value = value.time()
elif value.hour == value.minute == value.second == value.microsecond == 0:
value = value.date()
View
22 django/db/backends/sqlite3/base.py
@@ -10,7 +10,7 @@
from django.db.backends.sqlite3.client import DatabaseClient
from django.db.backends.sqlite3.creation import DatabaseCreation
from django.db.backends.sqlite3.introspection import DatabaseIntrospection
-from django.utils.safestring import SafeString
+from django.utils.safestring import SafeString
try:
try:
@@ -102,6 +102,26 @@ def year_lookup_bounds(self, value):
second = '%s-12-31 23:59:59.999999'
return [first % value, second % value]
+ def convert_values(self, value, field):
+ """SQLite returns floats when it should be returning decimals,
+ and gets dates and datetimes wrong.
+ For consistency with other backends, coerce when required.
+ """
+ internal_type = field.get_internal_type()
+ if internal_type == 'DecimalField':
+ return util.typecast_decimal(field.format_number(value))
+ elif internal_type and internal_type.endswith('IntegerField') or internal_type == 'AutoField':
+ return int(value)
+ elif internal_type == 'DateField':
+ return util.typecast_date(value)
+ elif internal_type == 'DateTimeField':
+ return util.typecast_timestamp(value)
+ elif internal_type == 'TimeField':
+ return util.typecast_time(value)
+
+ # No field, or the field isn't known to be a decimal or integer
+ return value
+
class DatabaseWrapper(BaseDatabaseWrapper):
# SQLite requires LIKE statements to include an ESCAPE clause if the value
View
1  django/db/models/__init__.py
@@ -5,6 +5,7 @@
from django.db.models.query import Q
from django.db.models.manager import Manager
from django.db.models.base import Model
+from django.db.models.aggregates import *
from django.db.models.fields import *
from django.db.models.fields.subclassing import SubfieldBase
from django.db.models.fields.files import FileField, ImageField
View
66 django/db/models/aggregates.py
@@ -0,0 +1,66 @@
+"""
+Classes to represent the definitions of aggregate functions.
+"""
+
+class Aggregate(object):
+ """
+ Default Aggregate definition.
+ """
+ def __init__(self, lookup, **extra):
+ """Instantiate a new aggregate.
+
+ * lookup is the field on which the aggregate operates.
+ * extra is a dictionary of additional data to provide for the
+ aggregate definition
+
+ Also utilizes the class variables:
+ * name, the identifier for this aggregate function.
+ """
+ self.lookup = lookup
+ self.extra = extra
+
+ def _default_alias(self):
+ return '%s__%s' % (self.lookup, self.name.lower())
+ default_alias = property(_default_alias)
+
+ def add_to_query(self, query, alias, col, source, is_summary):
+ """Add the aggregate to the nominated query.
+
+ This method is used to convert the generic Aggregate definition into a
+ backend-specific definition.
+
+ * query is the backend-specific query instance to which the aggregate
+ is to be added.
+ * col is a column reference describing the subject field
+ of the aggregate. It can be an alias, or a tuple describing
+ a table and column name.
+ * source is the underlying field or aggregate definition for
+ the column reference. If the aggregate is not an ordinal or
+ computed type, this reference is used to determine the coerced
+ output type of the aggregate.
+ * is_summary is a boolean that is set True if the aggregate is a
+ summary value rather than an annotation.
+ """
+ aggregate = getattr(query.aggregates_module, self.name)
+ query.aggregate_select[alias] = aggregate(col, source=source, is_summary=is_summary, **self.extra)
+
+class Avg(Aggregate):
+ name = 'Avg'
+
+class Count(Aggregate):
+ name = 'Count'
+
+class Max(Aggregate):
+ name = 'Max'
+
+class Min(Aggregate):
+ name = 'Min'
+
+class StdDev(Aggregate):
+ name = 'StdDev'
+
+class Sum(Aggregate):
+ name = 'Sum'
+
+class Variance(Aggregate):
+ name = 'Variance'
View
6 django/db/models/manager.py
@@ -101,6 +101,12 @@ def create(self, **kwargs):
def filter(self, *args, **kwargs):
return self.get_query_set().filter(*args, **kwargs)
+ def aggregate(self, *args, **kwargs):
+ return self.get_query_set().aggregate(*args, **kwargs)
+
+ def annotate(self, *args, **kwargs):
+ return self.get_query_set().annotate(*args, **kwargs)
+
def complex_filter(self, *args, **kwargs):
return self.get_query_set().complex_filter(*args, **kwargs)
View
89 django/db/models/query.py
@@ -4,6 +4,7 @@
from sets import Set as set # Python 2.3 fallback
from django.db import connection, transaction, IntegrityError
+from django.db.models.aggregates import Aggregate
from django.db.models.fields import DateField
from django.db.models.query_utils import Q, select_related_descend
from django.db.models import signals, sql
@@ -270,18 +271,47 @@ def iterator(self):
else:
requested = None
max_depth = self.query.max_depth
+
extra_select = self.query.extra_select.keys()
+ aggregate_select = self.query.aggregate_select.keys()
+
index_start = len(extra_select)
+ aggregate_start = index_start + len(self.model._meta.fields)
+
for row in self.query.results_iter():
if fill_cache:
- obj, _ = get_cached_row(self.model, row, index_start,
- max_depth, requested=requested)
+ obj, aggregate_start = get_cached_row(self.model, row,
+ index_start, max_depth, requested=requested)
else:
- obj = self.model(*row[index_start:])
+ # omit aggregates in object creation
+ obj = self.model(*row[index_start:aggregate_start])
+
for i, k in enumerate(extra_select):
setattr(obj, k, row[i])
+
+ # Add the aggregates to the model
+ for i, aggregate in enumerate(aggregate_select):
+ setattr(obj, aggregate, row[i+aggregate_start])
+
yield obj
+ def aggregate(self, *args, **kwargs):
+ """
+ Returns a dictionary containing the calculations (aggregation)
+ over the current queryset
+
+ If args is present the expression is passed as a kwarg ussing
+ the Aggregate object's default alias.
+ """
+ for arg in args:
+ kwargs[arg.default_alias] = arg
+
+ for (alias, aggregate_expr) in kwargs.items():
+ self.query.add_aggregate(aggregate_expr, self.model, alias,
+ is_summary=True)
+
+ return self.query.get_aggregation()
+
def count(self):
"""
Performs a SELECT COUNT() and returns the number of records as an
@@ -553,6 +583,25 @@ def dup_select_related(self, other):
"""
self.query.select_related = other.query.select_related
+ def annotate(self, *args, **kwargs):
+ """
+ Return a query set in which the returned objects have been annotated
+ with data aggregated from related fields.
+ """
+ for arg in args:
+ kwargs[arg.default_alias] = arg
+
+ obj = self._clone()
+
+ obj._setup_aggregate_query()
+
+ # Add the aggregates to the query
+ for (alias, aggregate_expr) in kwargs.items():
+ obj.query.add_aggregate(aggregate_expr, self.model, alias,
+ is_summary=False)
+
+ return obj
+
def order_by(self, *field_names):
"""
Returns a new QuerySet instance with the ordering changed.
@@ -641,6 +690,16 @@ def _merge_sanity_check(self, other):
"""
pass
+ def _setup_aggregate_query(self):
+ """
+ Prepare the query for computing a result that contains aggregate annotations.
+ """
+ opts = self.model._meta
+ if not self.query.group_by:
+ field_names = [f.attname for f in opts.fields]
+ self.query.add_fields(field_names, False)
+ self.query.set_group_by()
+
def as_sql(self):
"""
Returns the internal query's SQL and parameters (as a tuple).
@@ -669,6 +728,8 @@ def iterator(self):
len(self.field_names) != len(self.model._meta.fields)):
self.query.trim_extra_select(self.extra_names)
names = self.query.extra_select.keys() + self.field_names
+ names.extend(self.query.aggregate_select.keys())
+
for row in self.query.results_iter():
yield dict(zip(names, row))
@@ -682,20 +743,25 @@ def _setup_query(self):
"""
self.query.clear_select_fields()
self.extra_names = []
+ self.aggregate_names = []
+
if self._fields:
- if not self.query.extra_select:
+ if not self.query.extra_select and not self.query.aggregate_select:
field_names = list(self._fields)
else:
field_names = []
for f in self._fields:
if self.query.extra_select.has_key(f):
self.extra_names.append(f)
+ elif self.query.aggregate_select.has_key(f):
+ self.aggregate_names.append(f)
else:
field_names.append(f)
else:
# Default to all fields.
field_names = [f.attname for f in self.model._meta.fields]
+ self.query.select = []
self.query.add_fields(field_names, False)
self.query.default_cols = False
self.field_names = field_names
@@ -711,6 +777,7 @@ def _clone(self, klass=None, setup=False, **kwargs):
c._fields = self._fields[:]
c.field_names = self.field_names
c.extra_names = self.extra_names
+ c.aggregate_names = self.aggregate_names
if setup and hasattr(c, '_setup_query'):
c._setup_query()
return c
@@ -718,10 +785,18 @@ def _clone(self, klass=None, setup=False, **kwargs):
def _merge_sanity_check(self, other):
super(ValuesQuerySet, self)._merge_sanity_check(other)
if (set(self.extra_names) != set(other.extra_names) or
- set(self.field_names) != set(other.field_names)):
+ set(self.field_names) != set(other.field_names) or
+ self.aggregate_names != other.aggregate_names):
raise TypeError("Merging '%s' classes must involve the same values in each case."
% self.__class__.__name__)
+ def _setup_aggregate_query(self):
+ """
+ Prepare the query for computing a result that contains aggregate annotations.
+ """
+ self.query.set_group_by()
+
+ super(ValuesQuerySet, self)._setup_aggregate_query()
class ValuesListQuerySet(ValuesQuerySet):
def iterator(self):
@@ -729,14 +804,14 @@ def iterator(self):
if self.flat and len(self._fields) == 1:
for row in self.query.results_iter():
yield row[0]
- elif not self.query.extra_select:
+ elif not self.query.extra_select and not self.query.aggregate_select:
for row in self.query.results_iter():
yield tuple(row)
else:
# When extra(select=...) is involved, the extra cols come are
# always at the start of the row, so we need to reorder the fields
# to match the order in self._fields.
- names = self.query.extra_select.keys() + self.field_names
+ names = self.query.extra_select.keys() + self.field_names + self.query.aggregate_select.keys()
for row in self.query.results_iter():
data = dict(zip(names, row))
yield tuple([data[f] for f in self._fields])
View
1  django/db/models/query_utils.py
@@ -64,4 +64,3 @@ def select_related_descend(field, restricted, requested):
if not restricted and field.null:
return False
return True
-
View
130 django/db/models/sql/aggregates.py
@@ -0,0 +1,130 @@
+"""
+Classes to represent the default SQL aggregate functions
+"""
+
+class AggregateField(object):
+ """An internal field mockup used to identify aggregates in the
+ data-conversion parts of the database backend.
+ """
+ def __init__(self, internal_type):
+ self.internal_type = internal_type
+ def get_internal_type(self):
+ return self.internal_type
+
+ordinal_aggregate_field = AggregateField('IntegerField')
+computed_aggregate_field = AggregateField('FloatField')
+
+class Aggregate(object):
+ """
+ Default SQL Aggregate.
+ """
+ is_ordinal = False
+ is_computed = False
+ sql_template = '%(function)s(%(field)s)'
+
+ def __init__(self, col, source=None, is_summary=False, **extra):
+ """Instantiate an SQL aggregate
+
+ * col is a column reference describing the subject field
+ of the aggregate. It can be an alias, or a tuple describing
+ a table and column name.
+ * source is the underlying field or aggregate definition for
+ the column reference. If the aggregate is not an ordinal or
+ computed type, this reference is used to determine the coerced
+ output type of the aggregate.
+ * extra is a dictionary of additional data to provide for the
+ aggregate definition
+
+ Also utilizes the class variables:
+ * sql_function, the name of the SQL function that implements the
+ aggregate.
+ * sql_template, a template string that is used to render the
+ aggregate into SQL.
+ * is_ordinal, a boolean indicating if the output of this aggregate
+ is an integer (e.g., a count)
+ * is_computed, a boolean indicating if this output of this aggregate
+ is a computed float (e.g., an average), regardless of the input
+ type.
+
+ """
+ self.col = col
+ self.source = source
+ self.is_summary = is_summary
+ self.extra = extra
+
+ # Follow the chain of aggregate sources back until you find an
+ # actual field, or an aggregate that forces a particular output
+ # type. This type of this field will be used to coerce values
+ # retrieved from the database.
+ tmp = self
+
+ while tmp and isinstance(tmp, Aggregate):
+ if getattr(tmp, 'is_ordinal', False):
+ tmp = ordinal_aggregate_field
+ elif getattr(tmp, 'is_computed', False):
+ tmp = computed_aggregate_field
+ else:
+ tmp = tmp.source
+
+ self.field = tmp
+
+ def relabel_aliases(self, change_map):
+ if isinstance(self.col, (list, tuple)):
+ self.col = (change_map.get(self.col[0], self.col[0]), self.col[1])
+
+ def as_sql(self, quote_func=None):
+ "Return the aggregate, rendered as SQL."
+ if not quote_func:
+ quote_func = lambda x: x
+
+ if hasattr(self.col, 'as_sql'):
+ field_name = self.col.as_sql(quote_func)
+ elif isinstance(self.col, (list, tuple)):
+ field_name = '.'.join([quote_func(c) for c in self.col])
+ else:
+ field_name = self.col
+
+ params = {
+ 'function': self.sql_function,
+ 'field': field_name
+ }
+ params.update(self.extra)
+
+ return self.sql_template % params
+
+
+class Avg(Aggregate):
+ is_computed = True
+ sql_function = 'AVG'
+
+class Count(Aggregate):
+ is_ordinal = True
+ sql_function = 'COUNT'
+ sql_template = '%(function)s(%(distinct)s%(field)s)'
+
+ def __init__(self, col, distinct=False, **extra):
+ super(Count, self).__init__(col, distinct=distinct and 'DISTINCT ' or '', **extra)
+
+class Max(Aggregate):
+ sql_function = 'MAX'
+
+class Min(Aggregate):
+ sql_function = 'MIN'
+
+class StdDev(Aggregate):
+ is_computed = True
+
+ def __init__(self, col, sample=False, **extra):
+ super(StdDev, self).__init__(col, **extra)
+ self.sql_function = sample and 'STDDEV_SAMP' or 'STDDEV_POP'
+
+class Sum(Aggregate):
+ sql_function = 'SUM'
+
+class Variance(Aggregate):
+ is_computed = True
+
+ def __init__(self, col, sample=False, **extra):
+ super(Variance, self).__init__(col, **extra)
+ self.sql_function = sample and 'VAR_SAMP' or 'VAR_POP'
+
View
53 django/db/models/sql/datastructures.py
@@ -25,59 +25,6 @@ class RawValue(object):
def __init__(self, value):
self.value = value
-class Aggregate(object):
- """
- Base class for all aggregate-related classes (min, max, avg, count, sum).
- """
- def relabel_aliases(self, change_map):
- """
- Relabel the column alias, if necessary. Must be implemented by
- subclasses.
- """
- raise NotImplementedError
-
- def as_sql(self, quote_func=None):
- """
- Returns the SQL string fragment for this object.
-
- The quote_func function is used to quote the column components. If
- None, it defaults to doing nothing.
-
- Must be implemented by subclasses.
- """
- raise NotImplementedError
-
-class Count(Aggregate):
- """
- Perform a count on the given column.
- """
- def __init__(self, col='*', distinct=False):
- """
- Set the column to count on (defaults to '*') and set whether the count
- should be distinct or not.
- """
- self.col = col
- self.distinct = distinct
-
- def relabel_aliases(self, change_map):
- c = self.col
- if isinstance(c, (list, tuple)):
- self.col = (change_map.get(c[0], c[0]), c[1])
-
- def as_sql(self, quote_func=None):
- if not quote_func:
- quote_func = lambda x: x
- if isinstance(self.col, (list, tuple)):
- col = ('%s.%s' % tuple([quote_func(c) for c in self.col]))
- elif hasattr(self.col, 'as_sql'):
- col = self.col.as_sql(quote_func)
- else:
- col = self.col
- if self.distinct:
- return 'COUNT(DISTINCT %s)' % col
- else:
- return 'COUNT(%s)' % col
-
class Date(object):
"""
Add a date selection column.
View
307 django/db/models/sql/query.py
@@ -12,12 +12,13 @@
from django.utils.tree import Node
from django.utils.datastructures import SortedDict
from django.utils.encoding import force_unicode
+from django.db.backends.util import truncate_name
from django.db import connection
from django.db.models import signals
from django.db.models.fields import FieldDoesNotExist
from django.db.models.query_utils import select_related_descend
+from django.db.models.sql import aggregates as base_aggregates_module
from django.db.models.sql.where import WhereNode, Constraint, EverythingNode, AND, OR
-from django.db.models.sql.datastructures import Count
from django.core.exceptions import FieldError
from datastructures import EmptyResultSet, Empty, MultiJoin
from constants import *
@@ -40,6 +41,7 @@ class BaseQuery(object):
alias_prefix = 'T'
query_terms = QUERY_TERMS
+ aggregates_module = base_aggregates_module
def __init__(self, model, connection, where=WhereNode):
self.model = model
@@ -73,6 +75,9 @@ def __init__(self, model, connection, where=WhereNode):
self.select_related = False
self.related_select_cols = []
+ # SQL aggregate-related attributes
+ self.aggregate_select = SortedDict() # Maps alias -> SQL aggregate function
+
# Arbitrary maximum limit for select_related. Prevents infinite
# recursion. Can be changed by the depth parameter to select_related().
self.max_depth = 5
@@ -178,6 +183,7 @@ def clone(self, klass=None, **kwargs):
obj.distinct = self.distinct
obj.select_related = self.select_related
obj.related_select_cols = []
+ obj.aggregate_select = self.aggregate_select.copy()
obj.max_depth = self.max_depth
obj.extra_select = self.extra_select.copy()
obj.extra_tables = self.extra_tables
@@ -194,6 +200,35 @@ def clone(self, klass=None, **kwargs):
obj._setup_query()
return obj
+ def convert_values(self, value, field):
+ """Convert the database-returned value into a type that is consistent
+ across database backends.
+
+ By default, this defers to the underlying backend operations, but
+ it can be overridden by Query classes for specific backends.
+ """
+ return self.connection.ops.convert_values(value, field)
+
+ def resolve_aggregate(self, value, aggregate):
+ """Resolve the value of aggregates returned by the database to
+ consistent (and reasonable) types.
+
+ This is required because of the predisposition of certain backends
+ to return Decimal and long types when they are not needed.
+ """
+ if value is None:
+ # Return None as-is
+ return value
+ elif aggregate.is_ordinal:
+ # Any ordinal aggregate (e.g., count) returns an int
+ return int(value)
+ elif aggregate.is_computed:
+ # Any computed aggregate (e.g., avg) returns a float
+ return float(value)
+ else:
+ # Return value depends on the type of the field being processed.
+ return self.convert_values(value, aggregate.field)
+
def results_iter(self):
"""
Returns an iterator over the results from executing this query.
@@ -212,29 +247,78 @@ def results_iter(self):
else:
fields = self.model._meta.fields
row = self.resolve_columns(row, fields)
+
+ if self.aggregate_select:
+ aggregate_start = len(self.extra_select.keys()) + len(self.select)
+ row = tuple(row[:aggregate_start]) + tuple([
+ self.resolve_aggregate(value, aggregate)
+ for (alias, aggregate), value
+ in zip(self.aggregate_select.items(), row[aggregate_start:])
+ ])
+
yield row
+ def get_aggregation(self):
+ """
+ Returns the dictionary with the values of the existing aggregations.
+ """
+ if not self.aggregate_select:
+ return {}
+
+ # If there is a group by clause, aggregating does not add useful
+ # information but retrieves only the first row. Aggregate
+ # over the subquery instead.
+ if self.group_by:
+ from subqueries import AggregateQuery
+ query = AggregateQuery(self.model, self.connection)
+
+ obj = self.clone()
+
+ # Remove any aggregates marked for reduction from the subquery
+ # and move them to the outer AggregateQuery.
+ for alias, aggregate in self.aggregate_select.items():
+ if aggregate.is_summary:
+ query.aggregate_select[alias] = aggregate
+ del obj.aggregate_select[alias]
+
+ query.add_subquery(obj)
+ else:
+ query = self
+ self.select = []
+ self.default_cols = False
+ self.extra_select = {}
+
+ query.clear_ordering(True)
+ query.clear_limits()
+ query.select_related = False
+ query.related_select_cols = []
+ query.related_select_fields = []
+
+ return dict([
+ (alias, self.resolve_aggregate(val, aggregate))
+ for (alias, aggregate), val
+ in zip(query.aggregate_select.items(), query.execute_sql(SINGLE))
+ ])
+
def get_count(self):
"""
Performs a COUNT() query using the current filter constraints.
"""
- from subqueries import CountQuery
obj = self.clone()
- obj.clear_ordering(True)
- obj.clear_limits()
- obj.select_related = False
- obj.related_select_cols = []
- obj.related_select_fields = []
- if len(obj.select) > 1:
- obj = self.clone(CountQuery, _query=obj, where=self.where_class(),
- distinct=False)
- obj.select = []
- obj.extra_select = SortedDict()
+ if len(self.select) > 1:
+ # If a select clause exists, then the query has already started to
+ # specify the columns that are to be returned.
+ # In this case, we need to use a subquery to evaluate the count.
+ from subqueries import AggregateQuery
+ subquery = obj
+ subquery.clear_ordering(True)
+ subquery.clear_limits()
+
+ obj = AggregateQuery(obj.model, obj.connection)
+ obj.add_subquery(subquery)
+
obj.add_count_column()
- data = obj.execute_sql(SINGLE)
- if not data:
- return 0
- number = data[0]
+ number = obj.get_aggregation()[None]
# Apply offset and limit constraints manually, since using LIMIT/OFFSET
# in SQL (in variants that provide them) doesn't change the COUNT
@@ -450,25 +534,41 @@ def get_columns(self, with_aliases=False):
for col in self.select:
if isinstance(col, (list, tuple)):
r = '%s.%s' % (qn(col[0]), qn(col[1]))
- if with_aliases and col[1] in col_aliases:
- c_alias = 'Col%d' % len(col_aliases)
- result.append('%s AS %s' % (r, c_alias))
- aliases.add(c_alias)
- col_aliases.add(c_alias)
+ if with_aliases:
+ if col[1] in col_aliases:
+ c_alias = 'Col%d' % len(col_aliases)
+ result.append('%s AS %s' % (r, c_alias))
+ aliases.add(c_alias)
+ col_aliases.add(c_alias)
+ else:
+ result.append('%s AS %s' % (r, col[1]))
+ aliases.add(r)
+ col_aliases.add(col[1])
else:
result.append(r)
aliases.add(r)
col_aliases.add(col[1])
else:
result.append(col.as_sql(quote_func=qn))
+
if hasattr(col, 'alias'):
aliases.add(col.alias)
col_aliases.add(col.alias)
+
elif self.default_cols:
cols, new_aliases = self.get_default_columns(with_aliases,
col_aliases)
result.extend(cols)
aliases.update(new_aliases)
+
+ result.extend([
+ '%s%s' % (
+ aggregate.as_sql(quote_func=qn),
+ alias is not None and ' AS %s' % qn(alias) or ''
+ )
+ for alias, aggregate in self.aggregate_select.items()
+ ])
+
for table, col in self.related_select_cols:
r = '%s.%s' % (qn(table), qn(col))
if with_aliases and col in col_aliases:
@@ -538,7 +638,7 @@ def get_from_clause(self):
Returns a list of strings that are joined together to go after the
"FROM" part of the query, as well as a list any extra parameters that
need to be included. Sub-classes, can override this to create a
- from-clause via a "select", for example (e.g. CountQuery).
+ from-clause via a "select".
This should only be called after any SQL construction methods that
might change the tables we need. This means the select columns and
@@ -635,10 +735,13 @@ def get_ordering(self):
order = asc
result.append('%s %s' % (field, order))
continue
+ col, order = get_order_dir(field, asc)
+ if col in self.aggregate_select:
+ result.append('%s %s' % (col, order))
+ continue
if '.' in field:
# This came in through an extra(order_by=...) addition. Pass it
# on verbatim.
- col, order = get_order_dir(field, asc)
table, col = col.split('.', 1)
if (table, col) not in processed_pairs:
elt = '%s.%s' % (qn(table), col)
@@ -657,7 +760,6 @@ def get_ordering(self):
ordering_aliases.append(elt)
result.append('%s %s' % (elt, order))
else:
- col, order = get_order_dir(field, asc)
elt = qn2(col)
if distinct and col not in select_aliases:
ordering_aliases.append(elt)
@@ -1068,6 +1170,48 @@ def fill_related_selections(self, opts=None, root_alias=None, cur_depth=1,
self.fill_related_selections(f.rel.to._meta, alias, cur_depth + 1,
used, next, restricted, new_nullable, dupe_set, avoid)
+ def add_aggregate(self, aggregate, model, alias, is_summary):
+ """
+ Adds a single aggregate expression to the Query
+ """
+ opts = model._meta
+ field_list = aggregate.lookup.split(LOOKUP_SEP)
+ if (len(field_list) == 1 and
+ aggregate.lookup in self.aggregate_select.keys()):
+ # Aggregate is over an annotation
+ field_name = field_list[0]
+ col = field_name
+ source = self.aggregate_select[field_name]
+ elif (len(field_list) > 1 or
+ field_list[0] not in [i.name for i in opts.fields]):
+ field, source, opts, join_list, last, _ = self.setup_joins(
+ field_list, opts, self.get_initial_alias(), False)
+
+ # Process the join chain to see if it can be trimmed
+ _, _, col, _, join_list = self.trim_joins(source, join_list, last, False)
+
+ # If the aggregate references a model or field that requires a join,
+ # those joins must be LEFT OUTER - empty join rows must be returned
+ # in order for zeros to be returned for those aggregates.
+ for column_alias in join_list:
+ self.promote_alias(column_alias, unconditional=True)
+
+ col = (join_list[-1], col)
+ else:
+ # Aggregate references a normal field
+ field_name = field_list[0]
+ source = opts.get_field(field_name)
+ if not (self.group_by and is_summary):
+ # Only use a column alias if this is a
+ # standalone aggregate, or an annotation
+ col = (opts.db_table, source.column)
+ else:
+ col = field_name
+
+ # Add the aggregate to the query
+ alias = truncate_name(alias, self.connection.ops.max_name_length())
+ aggregate.add_to_query(self, alias, col=col, source=source, is_summary=is_summary)
+
def add_filter(self, filter_expr, connector=AND, negate=False, trim=False,
can_reuse=None, process_extras=True):
"""
@@ -1119,6 +1263,11 @@ def add_filter(self, filter_expr, connector=AND, negate=False, trim=False,
elif callable(value):
value = value()
+ for alias, aggregate in self.aggregate_select.items():
+ if alias == parts[0]:
+ self.having.add((aggregate, lookup_type, value), AND)
+ return
+
opts = self.get_meta()
alias = self.get_initial_alias()
allow_many = trim or not negate
@@ -1131,38 +1280,9 @@ def add_filter(self, filter_expr, connector=AND, negate=False, trim=False,
self.split_exclude(filter_expr, LOOKUP_SEP.join(parts[:e.level]),
can_reuse)
return
- final = len(join_list)
- penultimate = last.pop()
- if penultimate == final:
- penultimate = last.pop()
- if trim and len(join_list) > 1:
- extra = join_list[penultimate:]
- join_list = join_list[:penultimate]
- final = penultimate
- penultimate = last.pop()
- col = self.alias_map[extra[0]][LHS_JOIN_COL]
- for alias in extra:
- self.unref_alias(alias)
- else:
- col = target.column
- alias = join_list[-1]
- while final > 1:
- # An optimization: if the final join is against the same column as
- # we are comparing against, we can go back one step in the join
- # chain and compare against the lhs of the join instead (and then
- # repeat the optimization). The result, potentially, involves less
- # table joins.
- join = self.alias_map[alias]
- if col != join[RHS_JOIN_COL]:
- break
- self.unref_alias(alias)
- alias = join[LHS_ALIAS]
- col = join[LHS_JOIN_COL]
- join_list = join_list[:-1]
- final -= 1
- if final == penultimate:
- penultimate = last.pop()
+ # Process the join chain to see if it can be trimmed
+ final, penultimate, col, alias, join_list = self.trim_joins(target, join_list, last, trim)
if (lookup_type == 'isnull' and value is True and not negate and
final > 1):
@@ -1313,7 +1433,7 @@ def setup_joins(self, names, opts, alias, dupe_multis, allow_many=True,
field, model, direct, m2m = opts.get_field_by_name(f.name)
break
else:
- names = opts.get_all_field_names()
+ names = opts.get_all_field_names() + self.aggregate_select.keys()
raise FieldError("Cannot resolve keyword %r into field. "
"Choices are: %s" % (name, ", ".join(names)))
@@ -1462,6 +1582,43 @@ def setup_joins(self, names, opts, alias, dupe_multis, allow_many=True,
return field, target, opts, joins, last, extra_filters
+ def trim_joins(self, target, join_list, last, trim):
+ """An optimization: if the final join is against the same column as
+ we are comparing against, we can go back one step in a join
+ chain and compare against the LHS of the join instead (and then
+ repeat the optimization). The result, potentially, involves less
+ table joins.
+
+ Returns a tuple
+ """
+ final = len(join_list)
+ penultimate = last.pop()
+ if penultimate == final:
+ penultimate = last.pop()
+ if trim and len(join_list) > 1:
+ extra = join_list[penultimate:]
+ join_list = join_list[:penultimate]
+ final = penultimate
+ penultimate = last.pop()
+ col = self.alias_map[extra[0]][LHS_JOIN_COL]
+ for alias in extra:
+ self.unref_alias(alias)
+ else:
+ col = target.column
+ alias = join_list[-1]
+ while final > 1:
+ join = self.alias_map[alias]
+ if col != join[RHS_JOIN_COL]:
+ break
+ self.unref_alias(alias)
+ alias = join[LHS_ALIAS]
+ col = join[LHS_JOIN_COL]
+ join_list = join_list[:-1]
+ final -= 1
+ if final == penultimate:
+ penultimate = last.pop()
+ return final, penultimate, col, alias, join_list
+
def update_dupe_avoidance(self, opts, col, alias):
"""
For a column that is one of multiple pointing to the same table, update
@@ -1554,6 +1711,7 @@ def add_fields(self, field_names, allow_m2m=True):
"""
alias = self.get_initial_alias()
opts = self.get_meta()
+
try:
for name in field_names:
field, target, u2, joins, u3, u4 = self.setup_joins(
@@ -1574,7 +1732,7 @@ def add_fields(self, field_names, allow_m2m=True):
except MultiJoin:
raise FieldError("Invalid field name: '%s'" % name)
except FieldError:
- names = opts.get_all_field_names() + self.extra_select.keys()
+ names = opts.get_all_field_names() + self.extra_select.keys() + self.aggregate_select.keys()
names.sort()
raise FieldError("Cannot resolve keyword %r into field. "
"Choices are: %s" % (name, ", ".join(names)))
@@ -1609,38 +1767,52 @@ def clear_ordering(self, force_empty=False):
if force_empty:
self.default_ordering = False
+ def set_group_by(self):
+ """
+ Expands the GROUP BY clause required by the query.
+
+ This will usually be the set of all non-aggregate fields in the
+ return data. If the database backend supports grouping by the
+ primary key, and the query would be equivalent, the optimization
+ will be made automatically.
+ """
+ if self.connection.features.allows_group_by_pk:
+ if len(self.select) == len(self.model._meta.fields):
+ self.group_by.append('.'.join([self.model._meta.db_table,
+ self.model._meta.pk.column]))
+ return
+
+ for sel in self.select:
+ self.group_by.append(sel)
+
def add_count_column(self):
"""
Converts the query to do count(...) or count(distinct(pk)) in order to
get its size.
"""
- # TODO: When group_by support is added, this needs to be adjusted so
- # that it doesn't totally overwrite the select list.
if not self.distinct:
if not self.select:
- select = Count()
+ count = self.aggregates_module.Count('*', is_summary=True)
else:
assert len(self.select) == 1, \
"Cannot add count col with multiple cols in 'select': %r" % self.select
- select = Count(self.select[0])
+ count = self.aggregates_module.Count(self.select[0])
else:
opts = self.model._meta
if not self.select:
- select = Count((self.join((None, opts.db_table, None, None)),
- opts.pk.column), True)
+ count = self.aggregates_module.Count((self.join((None, opts.db_table, None, None)), opts.pk.column),
+ is_summary=True, distinct=True)
else:
# Because of SQL portability issues, multi-column, distinct
# counts need a sub-query -- see get_count() for details.
assert len(self.select) == 1, \
"Cannot add count col with multiple cols in 'select'."
- select = Count(self.select[0], True)
+ count = self.aggregates_module.Count(self.select[0], distinct=True)
# Distinct handling is done in Count(), so don't do it at this
# level.
self.distinct = False
- self.select = [select]
- self.select_fields = [None]
- self.extra_select = {}
+ self.aggregate_select = {None: count}
def add_select_related(self, fields):
"""
@@ -1758,7 +1930,6 @@ def execute_sql(self, result_type=MULTI):
return empty_iter()
else:
return
-
cursor = self.connection.cursor()
cursor.execute(sql, params)
View
30 django/db/models/sql/subqueries.py
@@ -9,7 +9,7 @@
from django.db.models.sql.where import AND, Constraint
__all__ = ['DeleteQuery', 'UpdateQuery', 'InsertQuery', 'DateQuery',
- 'CountQuery']
+ 'AggregateQuery']
class DeleteQuery(Query):
"""
@@ -400,15 +400,25 @@ def add_date_select(self, field, lookup_type, order='ASC'):
self.distinct = True
self.order_by = order == 'ASC' and [1] or [-1]
-class CountQuery(Query):
+class AggregateQuery(Query):
"""
- A CountQuery knows how to take a normal query which would select over
- multiple distinct columns and turn it into SQL that can be used on a
- variety of backends (it requires a select in the FROM clause).
+ An AggregateQuery takes another query as a parameter to the FROM
+ clause and only selects the elements in the provided list.
"""
- def get_from_clause(self):
- result, params = self._query.as_sql()
- return ['(%s) A1' % result], params
+ def add_subquery(self, query):
+ self.subquery, self.sub_params = query.as_sql(with_col_aliases=True)
- def get_ordering(self):
- return ()
+ def as_sql(self, quote_func=None):
+ """
+ Creates the SQL for this query. Returns the SQL string and list of
+ parameters.
+ """
+ sql = ('SELECT %s FROM (%s) subquery' % (
+ ', '.join([
+ aggregate.as_sql()
+ for aggregate in self.aggregate_select.values()
+ ]),
+ self.subquery)
+ )
+ params = self.sub_params
+ return (sql, params)
View
25 django/test/testcases.py
@@ -14,6 +14,7 @@
from django.utils import simplejson
normalize_long_ints = lambda s: re.sub(r'(?<![\w])(\d+)L(?![\w])', '\\1', s)
+normalize_decimals = lambda s: re.sub(r"Decimal\('(\d+(\.\d*)?)'\)", lambda m: "Decimal(\"%s\")" % m.groups()[0], s)
def to_list(value):
"""
@@ -31,7 +32,7 @@ class OutputChecker(doctest.OutputChecker):
def check_output(self, want, got, optionflags):
"The entry method for doctest output checking. Defers to a sequence of child checkers"
checks = (self.check_output_default,
- self.check_output_long,
+ self.check_output_numeric,
self.check_output_xml,
self.check_output_json)
for check in checks:
@@ -43,19 +44,23 @@ def check_output_default(self, want, got, optionflags):
"The default comparator provided by doctest - not perfect, but good for most purposes"
return doctest.OutputChecker.check_output(self, want, got, optionflags)
- def check_output_long(self, want, got, optionflags):
- """Doctest does an exact string comparison of output, which means long
- integers aren't equal to normal integers ("22L" vs. "22"). The
- following code normalizes long integers so that they equal normal
- integers.
+ def check_output_numeric(self, want, got, optionflags):
+ """Doctest does an exact string comparison of output, which means that
+ some numerically equivalent values aren't equal. This check normalizes
+ * long integers (22L) so that they equal normal integers. (22)
+ * Decimals so that they are comparable, regardless of the change
+ made to __repr__ in Python 2.6.
"""
- return normalize_long_ints(want) == normalize_long_ints(got)
+ return doctest.OutputChecker.check_output(self,
+ normalize_decimals(normalize_long_ints(want)),
+ normalize_decimals(normalize_long_ints(got)),
+ optionflags)
def check_output_xml(self, want, got, optionsflags):
"""Tries to do a 'xml-comparision' of want and got. Plain string
comparision doesn't always work because, for example, attribute
ordering should not be important.
-
+
Based on http://codespeak.net/svn/lxml/trunk/src/lxml/doctestcompare.py
"""
_norm_whitespace_re = re.compile(r'[ \t\n][ \t\n]+')
@@ -102,7 +107,7 @@ def check_element(want_element, got_element):
wrapper = '<root>%s</root>'
want = wrapper % want
got = wrapper % got
-
+
# Parse the want and got strings, and compare the parsings.
try:
want_root = parseString(want).firstChild
@@ -174,7 +179,7 @@ def _pre_setup(self):
"""Performs any pre-test setup. This includes:
* Flushing the database.
- * If the Test Case class has a 'fixtures' member, installing the
+ * If the Test Case class has a 'fixtures' member, installing the
named fixtures.
* If the Test Case class has a 'urls' member, replace the
ROOT_URLCONF with it.
View
2  docs/index.txt
@@ -42,7 +42,7 @@ The model layer
* **Models:** :ref:`Model syntax <topics-db-models>` | :ref:`Field types <ref-models-fields>` | :ref:`Meta options <ref-models-options>`
* **QuerySets:** :ref:`Executing queries <topics-db-queries>` | :ref:`QuerySet method reference <ref-models-querysets>`
* **Model instances:** :ref:`Instance methods <ref-models-instances>` | :ref:`Accessing related objects <ref-models-relations>`
- * **Advanced:** :ref:`Managers <topics-db-managers>` | :ref:`Raw SQL <topics-db-sql>` | :ref:`Transactions <topics-db-transactions>` | :ref:`Custom fields <howto-custom-model-fields>`
+ * **Advanced:** :ref:`Managers <topics-db-managers>` | :ref:`Raw SQL <topics-db-sql>` | :ref:`Transactions <topics-db-transactions>` | :ref:`Aggregation <topics-db-aggregation>` | :ref:`Custom fields <howto-custom-model-fields>`
* **Other:** :ref:`Supported databases <ref-databases>` | :ref:`Legacy databases <howto-legacy-databases>` | :ref:`Providing initial data <howto-initial-data>`
The template layer
View
2  docs/ref/models/index.txt
@@ -7,7 +7,7 @@ Model API reference. For introductory material, see :ref:`topics-db-models`.
.. toctree::
:maxdepth: 1
-
+
fields
relations
options
View
186 docs/ref/models/querysets.txt
@@ -158,6 +158,48 @@ In SQL terms, that evaluates to::
Note the second example is more restrictive.
+``annotate(*args, **kwargs)``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 1.1
+
+Annotates each object in the ``QuerySet`` with the provided list of
+aggregate values (averages, sums, etc) that have been computed over
+the objects that are related to the objects in the ``QuerySet``.
+Each argument to ``annotate()`` is an annotation that will be added
+to each object in the ``QuerySet`` that is returned.
+
+The aggregation functions that are provided by Django are described
+in `Aggregation Functions`_ below.
+
+Annotations specified using keyword arguments will use the keyword as
+the alias for the annotation. Anonymous arguments will have an alias
+generated for them based upon the name of the aggregate function and
+the model field that is being aggregated.
+
+For example, if you were manipulating a list of blogs, you may want
+to determine how many entries have been made in each blog::
+
+ >>> q = Blog.objects.annotate(Count('entry'))
+ # The name of the first blog
+ >>> q[0].name
+ 'Blogasaurus'
+ # The number of entries on the first blog
+ >>> q[0].entry__count
+ 42
+
+The ``Blog`` model doesn't define an ``entry_count`` attribute by itself,
+but by using a keyword argument to specify the aggregate function, you can
+control the name of the annotation::
+
+ >>> q = Blog.objects.annotate(number_of_entries=Count('entry'))
+ # The number of entries on the first blog, using the name provided
+ >>> q[0].number_of_entries
+ 42
+
+For an in-depth discussion of aggregation, see :ref:`the topic guide on
+Aggregation <topics-db-aggregation>`.
+
``order_by(*fields)``
~~~~~~~~~~~~~~~~~~~~~
@@ -931,6 +973,38 @@ exist with the given parameters.
Note ``latest()`` exists purely for convenience and readability.
+``aggregate(*args, **kwargs)``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 1.1
+
+Returns a dictionary of aggregate values (averages, sums, etc) calculated
+over the ``QuerySet``. Each argument to ``aggregate()`` specifies
+a value that will be included in the dictionary that is returned.
+
+The aggregation functions that are provided by Django are described
+in `Aggregation Functions`_ below.
+
+Aggregates specified using keyword arguments will use the keyword as
+the name for the annotation. Anonymous arguments will have an name
+generated for them based upon the name of the aggregate function and
+the model field that is being aggregated.
+
+For example, if you were manipulating blog entries, you may want to know
+the average number of authors contributing to blog entries::
+
+ >>> q = Blog.objects.aggregate(Count('entry'))
+ {'entry__count': 16}
+
+By using a keyword argument to specify the aggregate function, you can
+control the name of the aggregation value that is returned::
+
+ >>> q = Blog.objects.aggregate(number_of_entries=Count('entry'))
+ {'number_of_entries': 2.34}
+
+For an in-depth discussion of aggregation, see :ref:`the topic guide on
+Aggregation <topics-db-aggregation>`.
+
.. _field-lookups:
Field lookups
@@ -1326,3 +1400,115 @@ SQL equivalents::
SELECT ... WHERE title REGEXP '(?i)^(an?|the) +'; -- SQLite
+.. _aggregation-functions:
+
+Aggregation Functions
+---------------------
+
+.. versionadded:: 1.1
+
+Django provides the following aggregation functions in the
+``django.db.models`` module.
+
+``Avg``
+~~~~~~~
+
+.. class:: Avg(field)
+
+Returns the mean value of the given field.
+
+ * Default alias: ``<field>__avg``
+ * Return type: float
+
+``Count``
+~~~~~~~~~
+
+.. class:: Count(field, distinct=False)
+
+Returns the number of objects that are related through the provided field.
+
+ * Default alias: ``<field>__count``
+ * Return type: integer
+
+Has one optional argument:
+
+.. attribute:: distinct
+
+ If distinct=True, the count will only include unique instances. This has
+ the SQL equivalent of ``COUNT(DISTINCT field)``. Default value is ``False``.
+
+``Max``
+~~~~~~~
+
+.. class:: Max(field)
+
+Returns the maximum value of the given field.
+
+ * Default alias: ``<field>__max``
+ * Return type: same as input field
+
+``Min``
+~~~~~~~
+
+.. class:: Min(field)
+
+Returns the minimum value of the given field.
+
+ * Default alias: ``<field>__min``
+ * Return type: same as input field
+
+``StdDev``
+~~~~~~~~~
+
+.. class:: StdDev(field, sample=False)
+
+Returns the standard deviation of the data in the provided field.
+
+ * Default alias: ``<field>__stddev``
+ * Return type: float
+
+Has one optional argument:
+
+.. attribute:: sample
+
+ By default, ``StdDev`` returns the population standard deviation. However,
+ if ``sample=True``, the return value will be the sample standard deviation.
+
+.. admonition:: SQLite
+
+ SQLite doesn't provide ``StdDev`` out of the box. An implementation is
+ available as an extension module for SQLite. Consult the SQlite
+ documentation for instructions on obtaining and installing this extension.
+
+``Sum``
+~~~~~~~
+
+.. class:: Sum(field)
+
+Computes the sum of all values of the given field.
+
+ * Default alias: ``<field>__sum``
+ * Return type: same as input field
+
+``Variance``
+~~~~~~~~~
+
+.. class:: Variance(field, sample=False)
+
+Returns the variance of the data in the provided field.
+
+ * Default alias: ``<field>__variance``
+ * Return type: float
+
+Has one optional argument:
+
+.. attribute:: sample
+
+ By default, ``Variance`` returns the population variance. However,
+ if ``sample=True``, the return value will be the sample variance.
+
+.. admonition:: SQLite
+
+ SQLite doesn't provide ``Variance`` out of the box. An implementation is
+ available as an extension module for SQLite. Consult the SQlite
+ documentation for instructions on obtaining and installing this extension.
View
323 docs/topics/db/aggregation.txt
@@ -0,0 +1,323 @@