From 3ea70a4ff8b2de300f449423d84ffa5b3a9ae45e Mon Sep 17 00:00:00 2001 From: Anthony Tuininga Date: Thu, 4 Jan 2024 15:44:05 -0700 Subject: [PATCH] Fixed regression from cx_Oracle which ignored the value of the "encoding_errors" parameter when creating variables by calling the method Cursor.var() (#279). --- doc/src/release_notes.rst | 4 ++++ src/oracledb/base_impl.pxd | 3 ++- src/oracledb/impl/base/cursor.pyx | 1 + src/oracledb/impl/thick/utils.pyx | 7 ++++--- src/oracledb/impl/thick/var.pyx | 12 +++++++++--- src/oracledb/impl/thin/buffer.pyx | 9 +++++---- src/oracledb/impl/thin/messages.pyx | 7 ++++++- tests/test_3800_typehandler.py | 22 +++++++++++++++++++++- 8 files changed, 52 insertions(+), 13 deletions(-) diff --git a/doc/src/release_notes.rst b/doc/src/release_notes.rst index 134a3d6..3e10262 100644 --- a/doc/src/release_notes.rst +++ b/doc/src/release_notes.rst @@ -32,6 +32,10 @@ Common Changes #) Fixed regression which prevented a null value from being set on DbObject attributes or used as elements of collections (`issue 273 `__). +#) Fixed regression from cx_Oracle which ignored the value of the + ``encoding_errors`` parameter when creating variables by calling the method + :meth:`Cursor.var()` + (`issue 279 `__). #) Corrected typing declarations. #) Bumped minimum requirement of Cython to 3.0. diff --git a/src/oracledb/base_impl.pxd b/src/oracledb/base_impl.pxd index b922ad4..0514108 100644 --- a/src/oracledb/base_impl.pxd +++ b/src/oracledb/base_impl.pxd @@ -1,5 +1,5 @@ #------------------------------------------------------------------------------ -# Copyright (c) 2020, 2023, Oracle and/or its affiliates. +# Copyright (c) 2020, 2024, Oracle and/or its affiliates. # # This software is dual-licensed to you under the Universal Permissive License # (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl and Apache License @@ -342,6 +342,7 @@ cdef class BaseVarImpl: readonly uint32_t size readonly uint32_t buffer_size readonly bint bypass_decode + readonly str encoding_errors readonly bint is_array readonly bint nulls_allowed readonly bint convert_nulls diff --git a/src/oracledb/impl/base/cursor.pyx b/src/oracledb/impl/base/cursor.pyx index 924ae6e..6d4f5c3 100644 --- a/src/oracledb/impl/base/cursor.pyx +++ b/src/oracledb/impl/base/cursor.pyx @@ -419,6 +419,7 @@ cdef class BaseCursorImpl: var_impl.inconverter = inconverter var_impl.outconverter = outconverter var_impl.bypass_decode = bypass_decode + var_impl.encoding_errors = encoding_errors var_impl.is_array = is_array var_impl.convert_nulls = convert_nulls var_impl._finalize_init() diff --git a/src/oracledb/impl/thick/utils.pyx b/src/oracledb/impl/thick/utils.pyx index c82e91b..a6d9064 100644 --- a/src/oracledb/impl/thick/utils.pyx +++ b/src/oracledb/impl/thick/utils.pyx @@ -1,5 +1,5 @@ #------------------------------------------------------------------------------ -# Copyright (c) 2020, 2022, Oracle and/or its affiliates. +# Copyright (c) 2020, 2024, Oracle and/or its affiliates. # # This software is dual-licensed to you under the Universal Permissive License # (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl and Apache License @@ -261,7 +261,8 @@ cdef object _convert_to_python(ThickConnImpl conn_impl, DbType dbtype, ThickDbObjectTypeImpl obj_type_impl, dpiDataBuffer *dbvalue, int preferred_num_type=NUM_TYPE_FLOAT, - bint bypass_decode=False): + bint bypass_decode=False, + const char* encoding_errors=NULL): cdef: uint32_t oracle_type = dbtype.num ThickDbObjectImpl obj_impl @@ -282,7 +283,7 @@ cdef object _convert_to_python(ThickConnImpl conn_impl, DbType dbtype, or oracle_type == DPI_ORACLE_TYPE_LONG_NVARCHAR \ or oracle_type == DPI_ORACLE_TYPE_XMLTYPE: as_bytes = &dbvalue.asBytes - return as_bytes.ptr[:as_bytes.length].decode() + return as_bytes.ptr[:as_bytes.length].decode("utf-8", encoding_errors) elif oracle_type == DPI_ORACLE_TYPE_NUMBER: as_bytes = &dbvalue.asBytes if preferred_num_type == NUM_TYPE_INT \ diff --git a/src/oracledb/impl/thick/var.pyx b/src/oracledb/impl/thick/var.pyx index 40b3a0c..18e44ef 100644 --- a/src/oracledb/impl/thick/var.pyx +++ b/src/oracledb/impl/thick/var.pyx @@ -1,5 +1,5 @@ #------------------------------------------------------------------------------ -# Copyright (c) 2020, 2023, Oracle and/or its affiliates. +# Copyright (c) 2020, 2024, Oracle and/or its affiliates. # # This software is dual-licensed to you under the Universal Permissive License # (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl and Apache License @@ -264,15 +264,21 @@ cdef class ThickVarImpl(BaseVarImpl): Transforms a single element from the value supplied by ODPI-C to its equivalent Python value. """ - cdef object value + cdef: + const char *encoding_errors = NULL + bytes encoding_errors_bytes + object value data = &data[pos] if not data.isNull: if self._native_type_num == DPI_NATIVE_TYPE_STMT: return self._get_cursor_value(&data.value) + if self.encoding_errors is not None: + encoding_errors_bytes = self.encoding_errors.encode() + encoding_errors = encoding_errors_bytes value = _convert_to_python(self._conn_impl, self.dbtype, self.objtype, &data.value, self._preferred_num_type, - self.bypass_decode) + self.bypass_decode, encoding_errors) if self.outconverter is not None: value = self.outconverter(value) return value diff --git a/src/oracledb/impl/thin/buffer.pyx b/src/oracledb/impl/thin/buffer.pyx index c5bf8cb..b95a26b 100644 --- a/src/oracledb/impl/thin/buffer.pyx +++ b/src/oracledb/impl/thin/buffer.pyx @@ -1,5 +1,5 @@ #------------------------------------------------------------------------------ -# Copyright (c) 2020, 2023, Oracle and/or its affiliates. +# Copyright (c) 2020, 2024, Oracle and/or its affiliates. # # This software is dual-licensed to you under the Universal Permissive License # (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl and Apache License @@ -765,7 +765,7 @@ cdef class Buffer: self._pos = end_pos + 1 return self._data[start_pos:self._pos] - cdef object read_str(self, int csfrm): + cdef object read_str(self, int csfrm, const char* encoding_errors=NULL): """ Reads bytes from the buffer and decodes them into a string following the supplied character set form. @@ -776,8 +776,9 @@ cdef class Buffer: self.read_raw_bytes_and_length(&ptr, &num_bytes) if ptr != NULL: if csfrm == TNS_CS_IMPLICIT: - return ptr[:num_bytes].decode() - return ptr[:num_bytes].decode(TNS_ENCODING_UTF16) + return ptr[:num_bytes].decode(TNS_ENCODING_UTF8, + encoding_errors) + return ptr[:num_bytes].decode(TNS_ENCODING_UTF16, encoding_errors) cdef int read_ub1(self, uint8_t *value) except -1: """ diff --git a/src/oracledb/impl/thin/messages.pyx b/src/oracledb/impl/thin/messages.pyx index b5a7011..d2bb621 100644 --- a/src/oracledb/impl/thin/messages.pyx +++ b/src/oracledb/impl/thin/messages.pyx @@ -496,6 +496,8 @@ cdef class MessageWithData(Message): ThinVarImpl var_impl, uint32_t pos): cdef: uint8_t num_bytes, ora_type_num, csfrm + const char* encoding_errors = NULL + bytes encoding_errors_bytes ThinDbObjectTypeImpl typ_impl BaseThinCursorImpl cursor_impl object column_value = None @@ -525,7 +527,10 @@ cdef class MessageWithData(Message): or ora_type_num == TNS_DATA_TYPE_LONG: if csfrm == TNS_CS_NCHAR: buf._caps._check_ncharset_id() - column_value = buf.read_str(csfrm) + if var_impl.encoding_errors is not None: + encoding_errors_bytes = var_impl.encoding_errors.encode() + encoding_errors = encoding_errors_bytes + column_value = buf.read_str(csfrm, encoding_errors) elif ora_type_num == TNS_DATA_TYPE_RAW \ or ora_type_num == TNS_DATA_TYPE_LONG_RAW: column_value = buf.read_bytes() diff --git a/tests/test_3800_typehandler.py b/tests/test_3800_typehandler.py index 917327b..fec5f46 100644 --- a/tests/test_3800_typehandler.py +++ b/tests/test_3800_typehandler.py @@ -1,5 +1,5 @@ # ----------------------------------------------------------------------------- -# Copyright (c) 2021, 2023, Oracle and/or its affiliates. +# Copyright (c) 2021, 2024, Oracle and/or its affiliates. # # This software is dual-licensed to you under the Universal Permissive License # (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl and Apache License @@ -281,6 +281,26 @@ def output_type_handler(cursor, metadata): self.cursor.execute("select * from TestJson") self.assertEqual(self.cursor.fetchall(), data_to_insert) + def test_3807(self): + "3807 - output type handler for encoding errors" + + def output_type_handler(cursor, metadata): + if metadata.type_code is oracledb.DB_TYPE_VARCHAR: + return cursor.var( + metadata.type_code, + arraysize=cursor.arraysize, + encoding_errors="replace", + ) + + self.cursor.outputtypehandler = output_type_handler + self.cursor.execute( + "select utl_raw.cast_to_varchar2('41ab42cd43ef') from dual" + ) + (result,) = self.cursor.fetchone() + rc = chr(0xFFFD) + expected_result = f"A{rc}B{rc}C{rc}" + self.assertEqual(result, expected_result) + if __name__ == "__main__": test_env.run_test_cases()