Oracle float exprs #402

Closed
wants to merge 7 commits into
from
@@ -256,6 +256,10 @@ def quote_name(self, name):
if not name.startswith('"') and not name.endswith('"'):
name = '"%s"' % util.truncate_name(name.upper(),
self.max_name_length())
+ # This backend puts the query text into a (query % args) construct,
+ # so % signs in names need to be protected.
+ # Because of this, we are also not really making the name longer here.
+ name = name.replace('%','%%')
return name.upper()
def random_function_sql(self):
@@ -455,6 +459,7 @@ def __init__(self, *args, **kwargs):
self.features = DatabaseFeatures(self)
use_returning_into = self.settings_dict["OPTIONS"].get('use_returning_into', True)
self.features.can_return_id_from_insert = use_returning_into
+ self.float_is_enough = self.settings_dict["OPTIONS"].get('float_is_enough', False)
self.ops = DatabaseOperations(self)
self.client = DatabaseClient(self)
self.creation = DatabaseCreation(self)
@@ -492,8 +497,10 @@ def _cursor(self):
conn_params = self.settings_dict['OPTIONS'].copy()
if 'use_returning_into' in conn_params:
del conn_params['use_returning_into']
+ if 'float_is_enough' in conn_params:
+ del conn_params['float_is_enough']
self.connection = Database.connect(conn_string, **conn_params)
- cursor = FormatStylePlaceholderCursor(self.connection)
+ cursor = FormatStylePlaceholderCursor(self.connection, self.float_is_enough)
# Set the territory first. The territory overrides NLS_DATE_FORMAT
# and NLS_TIMESTAMP_FORMAT to the territory default. When all of
# these are set in single statement it isn't clear what is supposed
@@ -543,7 +550,7 @@ def _cursor(self):
pass
connection_created.send(sender=self.__class__, connection=self)
if not cursor:
- cursor = FormatStylePlaceholderCursor(self.connection)
+ cursor = FormatStylePlaceholderCursor(self.connection, self.float_is_enough)
return cursor
# Oracle doesn't support savepoint commits. Ignore them.
@@ -664,12 +671,17 @@ class FormatStylePlaceholderCursor(object):
"""
charset = 'utf-8'
- def __init__(self, connection):
+ def __init__(self, connection, float_is_enough):
self.cursor = connection.cursor()
- # Necessary to retrieve decimal values without rounding error.
- self.cursor.numbersAsStrings = True
# Default arraysize of 1 is highly sub-optimal.
self.cursor.arraysize = 100
+ if float_is_enough:
+ # Some conversions needed
+ self.cursor.outputtypehandler = _outputtypehandler_float
+ else:
+ # Besides the above, return some numbers as strings so they
+ # may be transformed to Decimals
+ self.cursor.outputtypehandler = _outputtypehandler
def _format_params(self, params):
return tuple([OracleParam(p, self, True) for p in params])
@@ -739,20 +751,15 @@ def executemany(self, query, params=None):
six.reraise(utils.DatabaseError, utils.DatabaseError(*tuple(e.args)), sys.exc_info()[2])
def fetchone(self):
- row = self.cursor.fetchone()
- if row is None:
- return row
- return _rowfactory(row, self.cursor)
+ return self.cursor.fetchone()
def fetchmany(self, size=None):
if size is None:
size = self.arraysize
- return tuple([_rowfactory(r, self.cursor)
- for r in self.cursor.fetchmany(size)])
+ return tuple(self.cursor.fetchmany(size))
def fetchall(self):
- return tuple([_rowfactory(r, self.cursor)
- for r in self.cursor.fetchall()])
+ return tuple(self.cursor.fetchall())
def var(self, *args):
return VariableWrapper(self.cursor.var(*args))
@@ -767,74 +774,70 @@ def __getattr__(self, attr):
return getattr(self.cursor, attr)
def __iter__(self):
- return CursorIterator(self.cursor)
-
-
-class CursorIterator(object):
-
- """Cursor iterator wrapper that invokes our custom row factory."""
-
- def __init__(self, cursor):
- self.cursor = cursor
- self.iter = iter(cursor)
-
- def __iter__(self):
- return self
-
- def __next__(self):
- return _rowfactory(next(self.iter), self.cursor)
-
- next = __next__ # Python 2 compatibility
-
-
-def _rowfactory(row, cursor):
- # Cast numeric values as the appropriate Python type based upon the
- # cursor description, and convert strings to unicode.
- casted = []
- for value, desc in zip(row, cursor.description):
- if value is not None and desc[1] is Database.NUMBER:
- precision, scale = desc[4:6]
- if scale == -127:
- if precision == 0:
- # NUMBER column: decimal-precision floating point
- # This will normally be an integer from a sequence,
- # but it could be a decimal value.
- if '.' in value:
- value = decimal.Decimal(value)
- else:
- value = int(value)
- else:
- # FLOAT column: binary-precision floating point.
- # This comes from FloatField columns.
- value = float(value)
- elif precision > 0:
- # NUMBER(p,s) column: decimal-precision fixed point.
- # This comes from IntField and DecimalField columns.
- if scale == 0:
- value = int(value)
- else:
- value = decimal.Decimal(value)
- elif '.' in value:
- # No type information. This normally comes from a
- # mathematical expression in the SELECT list. Guess int
- # or Decimal based on whether it has a decimal point.
- value = decimal.Decimal(value)
+ return iter(self.cursor)
+
+def _outputtypehandler(cursor, name, default_type, length, precision, scale):
+ if default_type is Database.NUMBER:
+ if scale == -127:
+ if precision == 0:
+ # NUMBER column: decimal-precision floating point
+ # This will normally be an integer from a sequence,
+ # but it could be a decimal value.
+ return cursor.var(str, 100, cursor.arraysize,
+ outconverter=_decimal_or_int)
else:
- value = int(value)
- # datetimes are returned as TIMESTAMP, except the results
- # of "dates" queries, which are returned as DATETIME.
- elif desc[1] in (Database.TIMESTAMP, Database.DATETIME):
- # Confirm that dt is naive before overwriting its tzinfo.
- if settings.USE_TZ and value is not None and timezone.is_naive(value):
- value = value.replace(tzinfo=timezone.utc)
- elif desc[1] in (Database.STRING, Database.FIXED_CHAR,
- Database.LONG_STRING):
- value = to_unicode(value)
- casted.append(value)
- return tuple(casted)
-
-
-def to_unicode(s):
+ # FLOAT column: binary-precision floating point.
+ # This comes from FloatField columns.
+ return cursor.var(default_type, arraysize=cursor.arraysize,
+ outconverter=float)
+ elif precision > 0:
+ # NUMBER(p,s) column: decimal-precision fixed point.
+ # This comes from IntField and DecimalField columns.
+ if scale == 0:
+ return cursor.var(default_type, arraysize=cursor.arraysize,
+ outconverter=int)
+ else:
+ return cursor.var(str, 100, cursor.arraysize,
+ outconverter=decimal.Decimal)
+ else:
+ # No type information. This normally comes from a
+ # mathematical expression in the SELECT list. Guess int
+ # or Decimal based on whether it has a decimal point.
+ return cursor.var(str, 100, cursor.arraysize,
+ outconverter=_decimal_or_int)
+ # datetimes are returned as TIMESTAMP, except the results
+ # of "dates" queries, which are returned as DATETIME.
+ elif default_type in (Database.TIMESTAMP, Database.DATETIME) and settings.USE_TZ:
+ return cursor.var(default_type, arraysize=cursor.arraysize,
+ outconverter=_add_tzinfo)
+ elif default_type in (Database.STRING, Database.FIXED_CHAR,
+ Database.LONG_STRING):
+ return cursor.var(default_type, length, cursor.arraysize,
+ outconverter=_to_unicode)
+
+def _outputtypehandler_float(cursor, name, default_type, length, precision, scale):
+ # This version only uses Decimal when it knows it is necessary,
+ # and float or int otherwise
+ if default_type is Database.NUMBER:
+ if precision>0 and scale!=0 and scale!=-127:
+ # This probably came from a DecimalField
+ return cursor.var(str, 100, cursor.arraysize,
+ outconverter=decimal.Decimal)
+ else:
+ # IntField, FloatField, sequences, expressions... Trust cx_Oracle
+ return None
+ # datetimes are returned as TIMESTAMP, except the results
+ # of "dates" queries, which are returned as DATETIME.
+ elif default_type in (Database.TIMESTAMP, Database.DATETIME) and settings.USE_TZ:
+ return cursor.var(default_type, arraysize=cursor.arraysize,
+ outconverter=_add_tzinfo)
+ elif default_type in (Database.STRING, Database.FIXED_CHAR,
+ Database.LONG_STRING):
+ return cursor.var(default_type, length, cursor.arraysize,
+ outconverter=_to_unicode)
+
+
+def _to_unicode(s):
"""
Convert strings to Unicode objects (and return all other data types
unchanged).
@@ -844,6 +847,20 @@ def to_unicode(s):
return s
+def _decimal_or_int(value):
+ if '.' in value:
+ return decimal.Decimal(value)
+ else:
+ return int(value)
+
+
+def _add_tzinfo(value):
+ # Confirm that dt is naive before overwriting its tzinfo.
+ if value is not None and timezone.is_naive(value):
+ value = value.replace(tzinfo=timezone.utc)
+ return value
+
+
def _get_sequence_reset_sql():
# TODO: colorize this SQL code with style.SQL_KEYWORD(), etc.
return """
View
@@ -562,20 +562,21 @@ Oracle notes
Django supports `Oracle Database Server`_ versions 9i and
higher. Oracle version 10g or later is required to use Django's
``regex`` and ``iregex`` query operators. You will also need at least
-version 4.3.1 of the `cx_Oracle`_ Python driver.
+version 5.0.1 of the `cx_Oracle`_ Python driver.
-Note that due to a Unicode-corruption bug in ``cx_Oracle`` 5.0, that
-version of the driver should **not** be used with Django;
-``cx_Oracle`` 5.0.1 resolved this issue, so if you'd like to use a
-more recent ``cx_Oracle``, use version 5.0.1.
-
-``cx_Oracle`` 5.0.1 or greater can optionally be compiled with the
+``cx_Oracle`` can optionally be compiled with the
``WITH_UNICODE`` environment variable. This is recommended but not
required.
+.. versionchanged:: 1.5
+
+Django 1.5 changed the required `cx_Oracle`_ version from 4.3.1 to 5.0.1.
+
.. _`Oracle Database Server`: http://www.oracle.com/
.. _`cx_Oracle`: http://cx-oracle.sourceforge.net/
+
+
In order for the ``python manage.py syncdb`` command to work, your Oracle
database user must have privileges to run the following commands:
@@ -658,6 +659,32 @@ The ``RETURNING INTO`` clause can be disabled by setting the
In this case, the Oracle backend will use a separate ``SELECT`` query to
retrieve AutoField values.
+Floats and Decimals
+-------------------
+
+The Oracle backend returns a number as a ``float``, ``int`` or
+``decimal.Decimal``, when there is enough information to know which type
+is expected. This is not the case with some expressions used in raw SQL
+(or ``extra(select={...})`` fields). In these cases, by default, the Oracle
+backend reads the value from the database as a string, and returns it as an
+``int`` or ``Decimal`` based on the presence of a decimal point in it. This
+behavior can be slow, and may cause incompatibility with other database
+backends (which use ``float`` in these cases), but it assures accurate
+number retrieval (since an Oracle ``NUMBER`` can have many more digits than
+a Python ``float``).
+
+If precision beyond ``float`` is not an issue, speed and compatibility can be
+regained by setting the ``float_is_enough`` option of the database
+configuration to True::
+
+ 'OPTIONS': {
+ 'float_is_enough': True,
+ },
+
+.. versionchanged:: 1.5
+
+The ``float_is_enough`` option is new in Django 1.5.
+
Naming issues
-------------
View
@@ -129,6 +129,12 @@ Django 1.5 also includes several smaller improvements worth noting:
configuration duplication. More information can be found in the
:func:`~django.contrib.auth.decorators.login_required` documentation.
+* The Oracle backend now supports a `float_is_enough` entry in the
+ `OPTIONS` dictionary, which makes it treat numbers as floats (rather
+ than slower, more precise data types) when it does not know which
+ number type to use. For more details, see the
+ :ref:`Oracle notes <oracle-notes>`.
+
Backwards incompatible changes in 1.5
=====================================
@@ -343,6 +349,8 @@ Miscellaneous
* GeoDjango dropped support for GDAL < 1.5
+* The Oracle database backend dropped support for cx_Oracle < 5.0.1
+
* :func:`~django.utils.http.int_to_base36` properly raises a :exc:`TypeError`
instead of :exc:`ValueError` for non-integer inputs.
@@ -355,6 +363,7 @@ Miscellaneous
needs. The new default value is `0666` (octal) and the current umask value
is first masked out.
+
Features deprecated in 1.5
==========================