Permalink
Browse files

Fixed #18757, #14462, #21565 -- Reworked database-python type convers…

…ions

Complete rework of translating data values from database

Deprecation of SubfieldBase, removal of resolve_columns and
convert_values in favour of a more general converter based approach and
public API Field.from_db_value(). Now works seamlessly with aggregation,
.values() and raw queries.

Thanks to akaariai in particular for extensive advice and inspiration,
also to shaib, manfre and timograham for their reviews.
  • Loading branch information...
1 parent 89559bc commit e9103402c0fa873aea58a6a11dba510cd308cb84 @mjtamlyn mjtamlyn committed Aug 12, 2014
Showing with 448 additions and 526 deletions.
  1. +0 −41 django/contrib/gis/db/backends/mysql/compiler.py
  2. +1 −1 django/contrib/gis/db/backends/mysql/operations.py
  3. +5 −0 django/contrib/gis/db/models/fields.py
  4. +1 −27 django/contrib/gis/db/models/query.py
  5. +10 −88 django/contrib/gis/db/models/sql/compiler.py
  6. +28 −7 django/contrib/gis/db/models/sql/conversion.py
  7. +1 −29 django/contrib/gis/db/models/sql/query.py
  8. +15 −0 django/contrib/gis/tests/geoapp/models.py
  9. +3 −10 django/contrib/gis/tests/geoapp/tests.py
  10. +5 −0 django/contrib/gis/tests/relatedapp/models.py
  11. +9 −1 django/contrib/gis/tests/relatedapp/tests.py
  12. +7 −14 django/db/backends/__init__.py
  13. +11 −0 django/db/backends/mysql/base.py
  14. +0 −11 django/db/backends/mysql/compiler.py
  15. +4 −0 django/db/backends/mysql/validation.py
  16. +50 −35 django/db/backends/oracle/base.py
  17. +1 −17 django/db/backends/oracle/compiler.py
  18. +27 −18 django/db/backends/sqlite3/base.py
  19. +5 −0 django/db/models/fields/__init__.py
  20. +7 −0 django/db/models/fields/subclassing.py
  21. +4 −5 django/db/models/query.py
  22. +63 −58 django/db/models/sql/compiler.py
  23. +7 −19 django/db/models/sql/query.py
  24. +51 −102 docs/howto/custom-model-fields.txt
  25. +2 −0 docs/internals/deprecation.txt
  26. +37 −17 docs/ref/models/fields.txt
  27. +11 −0 docs/releases/1.8.txt
  28. +0 −12 tests/aggregation_regress/tests.py
  29. +0 −12 tests/backends/tests.py
  30. +6 −1 tests/custom_pk/fields.py
  31. +11 −0 tests/field_subclassing/models.py
  32. 0 tests/from_db_value/__init__.py
  33. +32 −0 tests/from_db_value/models.py
  34. +30 −0 tests/from_db_value/tests.py
  35. +4 −1 tests/serializers/models.py
@@ -1,41 +0,0 @@
-from django.contrib.gis.db.models.sql.compiler import GeoSQLCompiler as BaseGeoSQLCompiler
-from django.db.backends.mysql import compiler
-
-SQLCompiler = compiler.SQLCompiler
-
-
-class GeoSQLCompiler(BaseGeoSQLCompiler, SQLCompiler):
- def resolve_columns(self, row, fields=()):
- """
- Integrate the cases handled both by the base GeoSQLCompiler and the
- main MySQL compiler (converting 0/1 to True/False for boolean fields).
-
- Refs #15169.
-
- """
- row = BaseGeoSQLCompiler.resolve_columns(self, row, fields)
- return SQLCompiler.resolve_columns(self, row, fields)
-
-
-class SQLInsertCompiler(compiler.SQLInsertCompiler, GeoSQLCompiler):
- pass
-
-
-class SQLDeleteCompiler(compiler.SQLDeleteCompiler, GeoSQLCompiler):
- pass
-
-
-class SQLUpdateCompiler(compiler.SQLUpdateCompiler, GeoSQLCompiler):
- pass
-
-
-class SQLAggregateCompiler(compiler.SQLAggregateCompiler, GeoSQLCompiler):
- pass
-
-
-class SQLDateCompiler(compiler.SQLDateCompiler, GeoSQLCompiler):
- pass
-
-
-class SQLDateTimeCompiler(compiler.SQLDateTimeCompiler, GeoSQLCompiler):
- pass
@@ -6,7 +6,7 @@
class MySQLOperations(DatabaseOperations, BaseSpatialOperations):
- compiler_module = 'django.contrib.gis.db.backends.mysql.compiler'
+ compiler_module = 'django.contrib.gis.db.models.sql.compiler'
mysql = True
name = 'mysql'
select = 'AsText(%s)'
@@ -197,6 +197,11 @@ def get_prep_value(self, value):
else:
return geom
+ def from_db_value(self, value, connection):
+ if value is not None:
+ value = Geometry(value)
+ return value
+
def get_srid(self, geom):
"""
Returns the default SRID for the given geometry, taking into account
@@ -1,5 +1,5 @@
from django.db import connections
-from django.db.models.query import QuerySet, ValuesQuerySet, ValuesListQuerySet
+from django.db.models.query import QuerySet
from django.contrib.gis.db.models import aggregates
from django.contrib.gis.db.models.fields import get_srid_info, PointField, LineStringField
@@ -18,19 +18,6 @@ def __init__(self, model=None, query=None, using=None, hints=None):
super(GeoQuerySet, self).__init__(model=model, query=query, using=using, hints=hints)
self.query = query or GeoQuery(self.model)
- def values(self, *fields):
- return self._clone(klass=GeoValuesQuerySet, setup=True, _fields=fields)
-
- def values_list(self, *fields, **kwargs):
- flat = kwargs.pop('flat', False)
- if kwargs:
- raise TypeError('Unexpected keyword arguments to values_list: %s'
- % (list(kwargs),))
- if flat and len(fields) > 1:
- raise TypeError("'flat' is not valid when values_list is called with more than one field.")
- return self._clone(klass=GeoValuesListQuerySet, setup=True, flat=flat,
- _fields=fields)
-
### GeoQuerySet Methods ###
def area(self, tolerance=0.05, **kwargs):
"""
@@ -767,16 +754,3 @@ def _geocol_select(self, geo_field, field_name):
return self.query.get_compiler(self.db)._field_column(geo_field, parent_model._meta.db_table)
else:
return self.query.get_compiler(self.db)._field_column(geo_field)
-
-
-class GeoValuesQuerySet(ValuesQuerySet):
- def __init__(self, *args, **kwargs):
- super(GeoValuesQuerySet, self).__init__(*args, **kwargs)
- # This flag tells `resolve_columns` to run the values through
- # `convert_values`. This ensures that Geometry objects instead
- # of string values are returned with `values()` or `values_list()`.
- self.query.geo_values = True
-
-
-class GeoValuesListQuerySet(GeoValuesQuerySet, ValuesListQuerySet):
- pass
@@ -1,12 +1,6 @@
-import datetime
-
-from django.conf import settings
-from django.db.backends.utils import truncate_name, typecast_date, typecast_timestamp
+from django.db.backends.utils import truncate_name
from django.db.models.sql import compiler
-from django.db.models.sql.constants import MULTI
from django.utils import six
-from django.utils.six.moves import zip, zip_longest
-from django.utils import timezone
SQLCompiler = compiler.SQLCompiler
@@ -153,38 +147,13 @@ def get_default_columns(self, with_aliases=False, col_aliases=None,
col_aliases.add(field.column)
return result, aliases
- def resolve_columns(self, row, fields=()):
- """
- This routine is necessary so that distances and geometries returned
- from extra selection SQL get resolved appropriately into Python
- objects.
- """
- values = []
- aliases = list(self.query.extra_select)
-
- # Have to set a starting row number offset that is used for
- # determining the correct starting row index -- needed for
- # doing pagination with Oracle.
- rn_offset = 0
- if self.connection.ops.oracle:
- if self.query.high_mark is not None or self.query.low_mark:
- rn_offset = 1
- index_start = rn_offset + len(aliases)
-
- # Converting any extra selection values (e.g., geometries and
- # distance objects added by GeoQuerySet methods).
- values = [self.query.convert_values(v,
- self.query.extra_select_fields.get(a, None),
- self.connection)
- for v, a in zip(row[rn_offset:index_start], aliases)]
- if self.connection.ops.oracle or getattr(self.query, 'geo_values', False):
- # We resolve the rest of the columns if we're on Oracle or if
- # the `geo_values` attribute is defined.
- for value, field in zip_longest(row[index_start:], fields):
- values.append(self.query.convert_values(value, field, self.connection))
- else:
- values.extend(row[index_start:])
- return tuple(values)
+ def get_converters(self, fields):
+ converters = super(GeoSQLCompiler, self).get_converters(fields)
+ for i, alias in enumerate(self.query.extra_select):
+ field = self.query.extra_select_fields.get(alias)
+ if field:
+ converters[i] = ([], [field.from_db_value], field)
+ return converters
#### Routines unique to GeoQuery ####
def get_extra_select_format(self, alias):
@@ -268,55 +237,8 @@ class SQLAggregateCompiler(compiler.SQLAggregateCompiler, GeoSQLCompiler):
class SQLDateCompiler(compiler.SQLDateCompiler, GeoSQLCompiler):
- """
- This is overridden for GeoDjango to properly cast date columns, since
- `GeoQuery.resolve_columns` is used for spatial values.
- See #14648, #16757.
- """
- def results_iter(self):
- if self.connection.ops.oracle:
- from django.db.models.fields import DateTimeField
- fields = [DateTimeField()]
- else:
- needs_string_cast = self.connection.features.needs_datetime_string_cast
-
- offset = len(self.query.extra_select)
- for rows in self.execute_sql(MULTI):
- for row in rows:
- date = row[offset]
- if self.connection.ops.oracle:
- date = self.resolve_columns(row, fields)[offset]
- elif needs_string_cast:
- date = typecast_date(str(date))
- if isinstance(date, datetime.datetime):
- date = date.date()
- yield date
+ pass
class SQLDateTimeCompiler(compiler.SQLDateTimeCompiler, GeoSQLCompiler):
- """
- This is overridden for GeoDjango to properly cast date columns, since
- `GeoQuery.resolve_columns` is used for spatial values.
- See #14648, #16757.
- """
- def results_iter(self):
- if self.connection.ops.oracle:
- from django.db.models.fields import DateTimeField
- fields = [DateTimeField()]
- else:
- needs_string_cast = self.connection.features.needs_datetime_string_cast
-
- offset = len(self.query.extra_select)
- for rows in self.execute_sql(MULTI):
- for row in rows:
- datetime = row[offset]
- if self.connection.ops.oracle:
- datetime = self.resolve_columns(row, fields)[offset]
- elif needs_string_cast:
- datetime = typecast_timestamp(str(datetime))
- # Datetimes are artificially returned in UTC on databases that
- # don't support time zone. Restore the zone used in the query.
- if settings.USE_TZ:
- datetime = datetime.replace(tzinfo=None)
- datetime = timezone.make_aware(datetime, self.query.tzinfo)
- yield datetime
+ pass
@@ -1,32 +1,53 @@
"""
-This module holds simple classes used by GeoQuery.convert_values
-to convert geospatial values from the database.
+This module holds simple classes to convert geospatial values from the
+database.
"""
+from django.contrib.gis.geometry.backend import Geometry
+from django.contrib.gis.measure import Area, Distance
+
class BaseField(object):
empty_strings_allowed = True
- def get_internal_type(self):
- "Overloaded method so OracleQuery.convert_values doesn't balk."
- return None
-
class AreaField(BaseField):
"Wrapper for Area values."
def __init__(self, area_att):
self.area_att = area_att
+ def from_db_value(self, value, connection):
+ if value is not None:
+ value = Area(**{self.area_att: value})
+ return value
+
+ def get_internal_type(self):
+ return 'AreaField'
+
class DistanceField(BaseField):
"Wrapper for Distance values."
def __init__(self, distance_att):
self.distance_att = distance_att
+ def from_db_value(self, value, connection):
+ if value is not None:
+ value = Distance(**{self.distance_att: value})
+ return value
+
+ def get_internal_type(self):
+ return 'DistanceField'
+
class GeomField(BaseField):
"""
Wrapper for Geometry values. It is a lightweight alternative to
using GeometryField (which requires an SQL query upon instantiation).
"""
- pass
+ def from_db_value(self, value, connection):
+ if value is not None:
+ value = Geometry(value)
+ return value
+
+ def get_internal_type(self):
+ return 'GeometryField'
@@ -5,9 +5,7 @@
from django.contrib.gis.db.models.fields import GeometryField
from django.contrib.gis.db.models.lookups import GISLookup
from django.contrib.gis.db.models.sql import aggregates as gis_aggregates
-from django.contrib.gis.db.models.sql.conversion import AreaField, DistanceField, GeomField
-from django.contrib.gis.geometry.backend import Geometry
-from django.contrib.gis.measure import Area, Distance
+from django.contrib.gis.db.models.sql.conversion import GeomField
class GeoQuery(sql.Query):
@@ -38,32 +36,6 @@ def clone(self, *args, **kwargs):
obj.extra_select_fields = self.extra_select_fields.copy()
return obj
- def convert_values(self, value, field, connection):
- """
- Using the same routines that Oracle does we can convert our
- extra selection objects into Geometry and Distance objects.
- TODO: Make converted objects 'lazy' for less overhead.
- """
- if connection.ops.oracle:
- # Running through Oracle's first.
- value = super(GeoQuery, self).convert_values(value, field or GeomField(), connection)
-
- if value is None:
- # Output from spatial function is NULL (e.g., called
- # function on a geometry field with NULL value).
- pass
- elif isinstance(field, DistanceField):
- # Using the field's distance attribute, can instantiate
- # `Distance` with the right context.
- value = Distance(**{field.distance_att: value})
- elif isinstance(field, AreaField):
- value = Area(**{field.area_att: value})
- elif isinstance(field, (GeomField, GeometryField)) and value:
- value = Geometry(value)
- elif field is not None:
- return super(GeoQuery, self).convert_values(value, field, connection)
- return value
-
def get_aggregation(self, using, force_subq=False):
# Remove any aggregates marked for reduction from the subquery
# and move them to the outer AggregateQuery.
@@ -66,3 +66,18 @@ class MinusOneSRID(models.Model):
class Meta:
app_label = 'geoapp'
+
+
+class NonConcreteField(models.IntegerField):
+
+ def db_type(self, connection):
+ return None
+
+ def get_attname_column(self):
+ attname, column = super(NonConcreteField, self).get_attname_column()
+ return attname, None
+
+
+class NonConcreteModel(NamedModel):
+ non_concrete = NonConcreteField()
+ point = models.PointField(geography=True)
@@ -13,9 +13,7 @@
if HAS_GEOS:
from django.contrib.gis.geos import (fromstr, GEOSGeometry,
Point, LineString, LinearRing, Polygon, GeometryCollection)
-
- from .models import Country, City, PennsylvaniaCity, State, Track
- from .models import Feature, MinusOneSRID
+ from .models import Country, City, PennsylvaniaCity, State, Track, NonConcreteModel, Feature, MinusOneSRID
def postgis_bug_version():
@@ -754,10 +752,5 @@ def test_unionagg(self):
self.assertEqual(None, qs.unionagg(field_name='point'))
def test_non_concrete_field(self):
- pkfield = City._meta.get_field_by_name('id')[0]
- orig_pkfield_col = pkfield.column
- pkfield.column = None
- try:
- list(City.objects.all())
- finally:
- pkfield.column = orig_pkfield_col
+ NonConcreteModel.objects.create(point=Point(0, 0), name='name')
+ list(NonConcreteModel.objects.all())
@@ -71,3 +71,8 @@ class Article(SimpleModel):
class Book(SimpleModel):
title = models.CharField(max_length=100)
author = models.ForeignKey(Author, related_name='books', null=True)
+
+
+class Event(SimpleModel):
+ name = models.CharField(max_length=100)
+ when = models.DateTimeField()
Oops, something went wrong.

0 comments on commit e910340

Please sign in to comment.