Fixed bug when attempting to convert an integer that cannot be

represented as a native C int value to an Arrow data frame.

Author: anthony-tuininga

doc/src/release_notes.rst

@@ -57,6 +57,8 @@ Common Changes
     (`issue 525 <https://github.com/oracle/python-oracledb/issues/525>`__).
 #)  Pin Cython to 3.1.x instead of 3.1.0 as requested
     (`issue 530 <https://github.com/oracle/python-oracledb/issues/530>`__).
+#)  Fixed bug when attempting to convert an integer that cannot be represented
+    as a native C ``int`` value to an Arrow data frame.
 #)  API documentation is now generated from the source code.
 #)  Internal change: typing_extensions is now a dependency.
 

src/oracledb/base_impl.pxd

@@ -958,7 +958,7 @@ cdef struct OracleNumber:
     bint is_integer
     bint is_max_negative_value
     uint8_t num_chars
-    char_type chars[172]
+    char_type chars[173]
 
 
 cdef struct OracleRawBytes:

src/oracledb/base_impl.pyx

@@ -35,10 +35,11 @@ cimport cython
 cimport cpython
 cimport cpython.datetime as cydatetime
 
+from libc cimport errno
 from libc.stdint cimport int8_t, int16_t, int32_t, int64_t
 from libc.stdint cimport uint8_t, uint16_t, uint32_t, uint64_t
 from libc.stdint cimport UINT8_MAX, UINT16_MAX, UINT32_MAX, UINT64_MAX
-from libc.stdlib cimport atoi, atof
+from libc.stdlib cimport strtod, strtoll
 from libc.string cimport memcpy
 from cpython cimport array
 from cpython.conversion cimport PyOS_snprintf

src/oracledb/errors.py

@@ -364,6 +364,8 @@ def _raise_not_supported(feature: str) -> None:
 ERR_EXCEEDED_IDLE_TIME = 4033
 ERR_INVALID_PASSWORD_TYPE = 4034
 ERR_INVALID_SERVER_RESPONSE = 4035
+ERR_CANNOT_CONVERT_TO_ARROW_INTEGER = 4036
+ERR_CANNOT_CONVERT_TO_ARROW_DOUBLE = 4037
 
 # error numbers that result in InternalError
 ERR_MESSAGE_TYPE_UNKNOWN = 5000
@@ -559,6 +561,12 @@ def _raise_not_supported(feature: str) -> None:
         "insufficient to hold {required_buffer_len} bytes"
     ),
     ERR_CALL_TIMEOUT_EXCEEDED: "call timeout of {timeout} ms exceeded",
+    ERR_CANNOT_CONVERT_TO_ARROW_DOUBLE: (
+        "{value} cannot be converted to an Arrow double"
+    ),
+    ERR_CANNOT_CONVERT_TO_ARROW_INTEGER: (
+        "{value} cannot be converted to an Arrow integer"
+    ),
     ERR_CANNOT_PARSE_CONNECT_STRING: 'cannot parse connect string "{data}"',
     ERR_COLUMN_TRUNCATED: (
         "column truncated to {col_value_len} {unit}. "

src/oracledb/impl/base/converters.pyx

@@ -231,20 +231,34 @@ cdef int convert_number_to_arrow_double(ArrowArrayImpl array_impl,
     """
     Converts a NUMBER value stored in the buffer to Arrow DOUBLE.
     """
-    cdef OracleNumber *value = &buffer.as_number
+    cdef:
+        OracleNumber *value = &buffer.as_number
+        double double_value
     if value.is_max_negative_value:
         array_impl.append_double(-1.0e126)
     else:
-        array_impl.append_double(atof(value.chars[:value.num_chars]))
+        errno.errno = 0
+        double_value = strtod((<const char*> value.chars), NULL)
+        if errno.errno != 0:
+            errors._raise_err(errors.ERR_CANNOT_CONVERT_TO_ARROW_DOUBLE,
+                              value=value.chars[:value.num_chars].decode())
+        array_impl.append_double(double_value)
 
 
 cdef int convert_number_to_arrow_int64(ArrowArrayImpl array_impl,
                                        OracleDataBuffer *buffer) except -1:
     """
     Converts a NUMBER value stored in the buffer to Arrow INT64.
     """
-    cdef OracleNumber *value = &buffer.as_number
-    array_impl.append_int64(atoi(value.chars[:value.num_chars]))
+    cdef:
+        OracleNumber *value = &buffer.as_number
+        int64_t int64_value
+    errno.errno = 0
+    int64_value = strtoll((<const char*> value.chars), NULL, 0)
+    if errno.errno != 0:
+        errors._raise_err(errors.ERR_CANNOT_CONVERT_TO_ARROW_INTEGER,
+                          value=value.chars[:value.num_chars].decode())
+    array_impl.append_int64(int64_value)
 
 
 cdef object convert_number_to_python_decimal(OracleDataBuffer *buffer):

src/oracledb/impl/base/decoders.pyx

@@ -196,6 +196,7 @@ cdef int decode_number(const uint8_t* ptr, ssize_t num_bytes,
         if is_positive:
             output.num_chars = 1
             output.chars[0] = 48                    # zero
+            output.chars[1] = 0                     # null terminator
         else:
             output.is_max_negative_value = True
         return 0
@@ -270,6 +271,9 @@ cdef int decode_number(const uint8_t* ptr, ssize_t num_bytes,
             output.chars[output.num_chars] = 48     # zero
             output.num_chars += 1
 
+    # include null terminator for use by strtoll() and strtoull()
+    output.chars[output.num_chars] = 0
+
 
 cdef inline uint16_t decode_uint16be(const char_type *buf):
     """

src/oracledb/impl/thick/var.pyx

@@ -490,7 +490,7 @@ cdef class ThickVarImpl(BaseVarImpl):
                 ora_data.buffer.as_number.is_integer = \
                         memchr(as_bytes.ptr, b'.', as_bytes.length) == NULL;
                 memcpy(ora_data.buffer.as_number.chars, as_bytes.ptr,
-                        as_bytes.length);
+                        as_bytes.length + 1);
                 ora_data.buffer.as_number.num_chars = as_bytes.length;
             elif ora_type_num == DPI_ORACLE_TYPE_VECTOR:
                 vector = _convert_vector_to_python(data.value.asVector)

tests/sql/create_schema.sql

@@ -410,7 +410,8 @@ create table &main_user..TestDataframe (
     CreditScore                         number(3, 0),
     LastUpdated                         timestamp,
     DecimalData                         number(15, 4),
-    IntegerData                         number(20),
+    IntegerData                         number(15),
+    LongIntegerData                     number(38),
     FloatData                           binary_float,
     DoubleData                          binary_double,
     RawData                             raw(100),

tests/test_8000_dataframe.py

@@ -1868,6 +1868,21 @@ def test_8076(self):
         ]
         self.__test_df_interop(data)
 
+    def test_8077(self):
+        "8077 - test fetching large integers"
+        data = (-(2**40), 2**41)
+        ora_df = self.conn.fetch_df_all(
+            """
+            select
+                cast(:1 as number(15)),
+                cast(:2 as number(15))
+            from dual
+            """,
+            data,
+        )
+        fetched_df = pyarrow.table(ora_df).to_pandas()
+        self.assertEqual([data], self.__get_data_from_df(fetched_df))
+
 
 if __name__ == "__main__":
     test_env.run_test_cases()

tests/test_8100_dataframe_async.py

@@ -1638,6 +1638,21 @@ async def test_8165(self):
         ]
         await self.__test_df_interop(data)
 
+    async def test_8166(self):
+        "8166 - test fetching large integers"
+        data = (-(2**40), 2**41)
+        ora_df = await self.conn.fetch_df_all(
+            """
+            select
+                cast(:1 as number(15)),
+                cast(:2 as number(15))
+            from dual
+            """,
+            data,
+        )
+        fetched_df = pyarrow.table(ora_df).to_pandas()
+        self.assertEqual([data], self.__get_data_from_df(fetched_df))
+
 
 if __name__ == "__main__":
     test_env.run_test_cases()