Support LOB types in the fetch data frame APIs.

Author: anthony-tuininga

doc/src/release_notes.rst

@@ -62,6 +62,8 @@ Common Changes
       data frame
     - eliminated small memory leak with production of all data frames
 
+#)  Added support for CLOB, BLOB and RAW data types in
+    :meth:`Connection.fetch_df_all()` and :meth:`Connection.fetch_df_batches()`
 #)  Fixed bug when NUMBER data is fetched with
     :meth:`Connection.fetch_df_all()` or :meth:`Connection.fetch_df_batches()`
     that does not have precision or scale specified and

doc/src/user_guide/sql_execution.rst

@@ -850,6 +850,12 @@ Oracle Database will result in an exception.
       - TIMESTAMP
     * - DB_TYPE_TIMESTAMP_TZ
       - TIMESTAMP
+    * - DB_TYPE_CLOB
+      - LARGE_STRING
+    * - DB_TYPE_BLOB
+      - LARGE_BINARY
+    * - DB_TYPE_RAW
+      - BINARY
 
 When converting Oracle Database NUMBERs:
 
@@ -864,6 +870,10 @@ When converting Oracle Database NUMBERs:
 
 - In all other cases, the Arrow data type is DOUBLE.
 
+When converting Oracle Database CLOBs and BLOBs:
+
+- The LOBs must be no more than 1 GB in length.
+
 When converting Oracle Database DATEs and TIMESTAMPs:
 
 - For Oracle Database DATE types, the Arrow TIMESTAMP will have a time unit of

src/oracledb/base_impl.pyx

@@ -48,10 +48,13 @@ from .interchange.nanoarrow_bridge cimport (
     NANOARROW_TIME_UNIT_MICRO,
     NANOARROW_TIME_UNIT_NANO,
     NANOARROW_TYPE_BOOL,
+    NANOARROW_TYPE_BINARY,
     NANOARROW_TYPE_DECIMAL128,
     NANOARROW_TYPE_DOUBLE,
     NANOARROW_TYPE_FLOAT,
     NANOARROW_TYPE_INT64,
+    NANOARROW_TYPE_LARGE_BINARY,
+    NANOARROW_TYPE_LARGE_STRING,
     NANOARROW_TYPE_STRING,
     NANOARROW_TYPE_TIMESTAMP,
 )

src/oracledb/impl/base/converters.pyx

@@ -232,7 +232,12 @@ cdef int convert_oracle_data_to_arrow(OracleMetadata from_metadata,
             arrow_array.append_double(data.buffer.as_double)
     elif arrow_type == NANOARROW_TYPE_FLOAT:
         arrow_array.append_float(data.buffer.as_float)
-    elif arrow_type == NANOARROW_TYPE_STRING:
+    elif arrow_type in (
+            NANOARROW_TYPE_BINARY,
+            NANOARROW_TYPE_STRING,
+            NANOARROW_TYPE_LARGE_BINARY,
+            NANOARROW_TYPE_LARGE_STRING
+    ):
         rb = &data.buffer.as_raw_bytes
         arrow_array.append_bytes(<void*> rb.ptr, rb.num_bytes)
     elif arrow_type == NANOARROW_TYPE_TIMESTAMP:

src/oracledb/impl/base/cursor.pyx

@@ -221,7 +221,7 @@ cdef class BaseCursorImpl:
             var_impl.outconverter = conn_impl.decode_oson
         elif metadata.is_json and db_type_num != DB_TYPE_NUM_JSON:
             var_impl.outconverter = self._build_json_converter_fn()
-        elif not C_DEFAULTS.fetch_lobs:
+        elif not C_DEFAULTS.fetch_lobs or self.fetching_arrow:
             if db_type_num == DB_TYPE_NUM_BLOB:
                 var_impl.metadata.dbtype = DB_TYPE_LONG_RAW
                 var_impl._fetch_metadata.dbtype = DB_TYPE_LONG_RAW

src/oracledb/impl/base/metadata.pyx

@@ -84,6 +84,12 @@ cdef class OracleMetadata:
                              DB_TYPE_NUM_TIMESTAMP_LTZ,
                              DB_TYPE_NUM_TIMESTAMP_TZ):
             self._arrow_type = NANOARROW_TYPE_TIMESTAMP
+        elif db_type_num == DB_TYPE_NUM_LONG_RAW:
+            self._arrow_type = NANOARROW_TYPE_LARGE_BINARY
+        elif db_type_num == DB_TYPE_NUM_LONG_VARCHAR:
+            self._arrow_type = NANOARROW_TYPE_LARGE_STRING
+        elif db_type_num == DB_TYPE_NUM_RAW:
+            self._arrow_type = NANOARROW_TYPE_BINARY
         else:
             errors._raise_err(errors.ERR_ARROW_UNSUPPORTED_DATA_TYPE,
                               db_type_name=self.dbtype.name)

src/oracledb/interchange/column.py

@@ -45,9 +45,12 @@
     NANOARROW_TIME_UNIT_MILLI,
     NANOARROW_TIME_UNIT_MICRO,
     NANOARROW_TIME_UNIT_NANO,
+    NANOARROW_TYPE_BINARY,
     NANOARROW_TYPE_DOUBLE,
     NANOARROW_TYPE_FLOAT,
     NANOARROW_TYPE_INT64,
+    NANOARROW_TYPE_LARGE_BINARY,
+    NANOARROW_TYPE_LARGE_STRING,
     NANOARROW_TYPE_STRING,
     NANOARROW_TYPE_TIMESTAMP,
     NANOARROW_TYPE_DECIMAL128,
@@ -88,7 +91,13 @@ def _offsets_buffer(self):
         offsets_buffer = OracleColumnBuffer(
             size_in_bytes=size_bytes, address=address, buffer_type="offsets"
         )
-        dtype = (DtypeKind.INT, 32, "i", "=")
+        if self.ora_arrow_array.arrow_type in (
+            NANOARROW_TYPE_LARGE_STRING,
+            NANOARROW_TYPE_LARGE_BINARY,
+        ):
+            dtype = (DtypeKind.INT, 64, "l", "=")
+        else:
+            dtype = (DtypeKind.INT, 32, "i", "=")
         return offsets_buffer, dtype
 
     def _validity_buffer(self):
@@ -149,6 +158,12 @@ def dtype(self) -> Dtype:
                 f"d:{array.precision}.{array.scale}",
                 "=",
             )
+        elif self.ora_arrow_array.arrow_type == NANOARROW_TYPE_BINARY:
+            return (DtypeKind.STRING, 8, "z", "=")
+        elif self.ora_arrow_array.arrow_type == NANOARROW_TYPE_LARGE_BINARY:
+            return (DtypeKind.STRING, 8, "Z", "=")
+        elif self.ora_arrow_array.arrow_type == NANOARROW_TYPE_LARGE_STRING:
+            return (DtypeKind.STRING, 8, "U", "=")
 
     def get_buffers(self) -> ColumnBuffers:
         """

src/oracledb/interchange/nanoarrow_bridge.pxd

@@ -48,10 +48,13 @@ cdef extern from "nanoarrow.h":
 
     cpdef enum ArrowType:
         NANOARROW_TYPE_BOOL
+        NANOARROW_TYPE_BINARY
         NANOARROW_TYPE_DECIMAL128
         NANOARROW_TYPE_DOUBLE
         NANOARROW_TYPE_FLOAT
         NANOARROW_TYPE_INT64
+        NANOARROW_TYPE_LARGE_BINARY
+        NANOARROW_TYPE_LARGE_STRING
         NANOARROW_TYPE_STRING
         NANOARROW_TYPE_TIMESTAMP
 

src/oracledb/interchange/nanoarrow_bridge.pyx

@@ -302,7 +302,10 @@ cdef class OracleArrowArray:
             ArrowDecimalSetBytes(&decimal, ptr)
             _check_nanoarrow(ArrowArrayAppendDecimal(self.arrow_array,
                                                      &decimal))
-        elif array.arrow_type == NANOARROW_TYPE_STRING:
+        elif array.arrow_type in (
+                NANOARROW_TYPE_BINARY,
+                NANOARROW_TYPE_STRING
+        ):
             offsets_buffer = ArrowArrayBuffer(array.arrow_array, 1)
             data_buffer = ArrowArrayBuffer(array.arrow_array, 2)
             as_int32 = <int32_t*> offsets_buffer.data
@@ -316,6 +319,23 @@ cdef class OracleArrowArray:
             finally:
                 cpython.PyMem_Free(temp)
 
+        elif array.arrow_type in (
+                NANOARROW_TYPE_LARGE_BINARY,
+                NANOARROW_TYPE_LARGE_STRING
+        ):
+            offsets_buffer = ArrowArrayBuffer(array.arrow_array, 1)
+            data_buffer = ArrowArrayBuffer(array.arrow_array, 2)
+            as_int64 = <int64_t*> offsets_buffer.data
+            start_offset = as_int64[index]
+            end_offset = as_int64[index + 1]
+            temp = cpython.PyMem_Malloc(end_offset - start_offset)
+            memcpy(temp, &data_buffer.data[start_offset],
+                   end_offset - start_offset)
+            try:
+                self.append_bytes(temp, end_offset - start_offset)
+            finally:
+                cpython.PyMem_Free(temp)
+
     cdef int append_null(self) except -1:
         """
         Append a null value to the array.

tests/test_8000_dataframe.py

@@ -466,8 +466,8 @@ def test_8015(self):
         self.assertEqual(col.null_count, 1)
 
     def test_8016(self):
-        "8016 - check unsupported error for LOBs"
-        statement = "select to_clob('test_8016') from dual"
+        "8016 - check unsupported error"
+        statement = "select cursor(select user from dual) from dual"
         with self.assertRaisesFullCode("DPY-3030"):
             self.conn.fetch_df_all(statement)
 
@@ -538,6 +538,48 @@ def test_8022(self):
             fetched_data = self.__get_data_from_df(fetched_df)
             self.assertEqual(fetched_data, data)
 
+    def test_8023(self):
+        "8023 - fetch clob"
+        data = [("test_8023",)]
+        self.__check_interop()
+        ora_df = self.conn.fetch_df_all(
+            "select to_clob('test_8023') from dual"
+        )
+        fetched_tab = pyarrow.Table.from_arrays(
+            ora_df.column_arrays(), names=ora_df.column_names()
+        )
+        fetched_df = fetched_tab.to_pandas()
+        fetched_data = self.__get_data_from_df(fetched_df)
+        self.assertEqual(fetched_data, data)
+
+    def test_8024(self):
+        "8024 - fetch blob"
+        data = [(b"test_8024",)]
+        self.__check_interop()
+        ora_df = self.conn.fetch_df_all(
+            "select to_blob(utl_raw.cast_to_raw('test_8024')) from dual"
+        )
+        fetched_tab = pyarrow.Table.from_arrays(
+            ora_df.column_arrays(), names=ora_df.column_names()
+        )
+        fetched_df = fetched_tab.to_pandas()
+        fetched_data = self.__get_data_from_df(fetched_df)
+        self.assertEqual(fetched_data, data)
+
+    def test_8025(self):
+        "8025 - fetch raw"
+        data = [(b"test_8025",)]
+        self.__check_interop()
+        ora_df = self.conn.fetch_df_all(
+            "select utl_raw.cast_to_raw('test_8025') from dual"
+        )
+        fetched_tab = pyarrow.Table.from_arrays(
+            ora_df.column_arrays(), names=ora_df.column_names()
+        )
+        fetched_df = fetched_tab.to_pandas()
+        fetched_data = self.__get_data_from_df(fetched_df)
+        self.assertEqual(fetched_data, data)
+
 
 if __name__ == "__main__":
     test_env.run_test_cases()