Merge pull request #155 from MonetDBSolutions/issue154_decimal_types

Add support for decimal types.

.github/workflows/osx.yml

@@ -81,7 +81,7 @@ jobs:
         run: python3 -m venv venv
       -
         name: Install Python dependencies
-        run: venv/bin/pip install --upgrade pip wheel delocate==0.8.2 pytest build
+        run: venv/bin/pip install --upgrade pip==21.2.4 wheel delocate==0.8.2 pytest build
       -
         name: Compile and install MonetDBe-python
         run: MONETDB_BRANCH=${{ matrix.branch }} CFLAGS="-I/usr/local/include -L/usr/local/lib" venv/bin/pip install -e ".[test]"

.github/workflows/windows.yml

@@ -18,8 +18,8 @@ jobs:
       matrix:
         include:
           - MONETDB_BRANCH: default
-            MONETDB_WIN_PREFIX: 82777
-            MONETDB_WIN_VERSION: f619cf62c92d
+            MONETDB_WIN_PREFIX: 83187
+            MONETDB_WIN_VERSION: 9b268809c28c
           - MONETDB_BRANCH: Jul2021
             MONETDB_WIN_PREFIX: 82786
             MONETDB_WIN_VERSION: e39e91cde6b5

monetdbe/_cffi/convert/__init__.py

@@ -1,3 +1,4 @@
+from decimal import Decimal
 from typing import List, Optional, Callable, Union, Any, Mapping
 from typing import NamedTuple
 
@@ -72,20 +73,43 @@ def numpy_monetdb_map(numpy_type: np.dtype):
     raise ProgrammingError("append() only support int and float family types")
 
 
-def extract(rcol: monetdbe_column, r: int, text_factory: Optional[Callable[[str], Any]] = None):
-    """
-    Extracts values from a monetdbe_column.
-
-    The text_factory is optional, and wraps the value with a custom user supplied text function.
-    """
-    type_info = monet_c_type_map[rcol.type]
-    col = ffi.cast(f"monetdbe_column_{type_info.c_string_type} *", rcol)
-    if col.is_null(col.data + r):
-        return None
-    else:
-        if type_info.py_converter:
-            result = type_info.py_converter(col.data[r])
-            if rcol.type == lib.monetdbe_str and text_factory:
-                return text_factory(result)
-            return result
-        return col.data[r]
+from monetdbe._cffi.branch import newer_then_jul2021
+if newer_then_jul2021:
+    def extract(rcol: monetdbe_column, r: int, text_factory: Optional[Callable[[str], Any]] = None):
+        """
+        Extracts values from a monetdbe_column.
+
+        The text_factory is optional, and wraps the value with a custom user supplied text function.
+        """
+
+        type_info = monet_c_type_map[rcol.type]
+        col = ffi.cast(f"monetdbe_column_{type_info.c_string_type} *", rcol)
+        if col.is_null(col.data + r):
+            return None
+        else:
+            col_data = col.data[r]
+            if rcol.sql_type.name != ffi.NULL and ffi.string(rcol.sql_type.name).decode() == 'decimal':
+                col_data = Decimal(col_data) / (Decimal(10) ** rcol.sql_type.scale)
+            if type_info.py_converter:
+                result = type_info.py_converter(col_data)
+                if rcol.type == lib.monetdbe_str and text_factory:
+                    return text_factory(result)
+                return result
+            return col_data
+else:
+    def extract(rcol: monetdbe_column, r: int, text_factory: Optional[Callable[[str], Any]] = None):
+        """
+        Extracts values from a monetdbe_column.
+        The text_factory is optional, and wraps the value with a custom user supplied text function.
+        """
+        type_info = monet_c_type_map[rcol.type]
+        col = ffi.cast(f"monetdbe_column_{type_info.c_string_type} *", rcol)
+        if col.is_null(col.data + r):
+            return None
+        else:
+            if type_info.py_converter:
+                result = type_info.py_converter(col.data[r])
+                if rcol.type == lib.monetdbe_str and text_factory:
+                    return text_factory(result)
+                return result
+            return col.data[r]

monetdbe/_cffi/convert/bind.py

@@ -1,8 +1,25 @@
 from typing import Any, Union
+from decimal import Decimal
 import datetime
 from monetdbe._lowlevel import ffi
 
 
+def monetdbe_decimal_to_bte(data: int) -> ffi.CData:
+    return ffi.new("int8_t *", data)
+
+
+def monetdbe_decimal_to_sht(data: int) -> ffi.CData:
+    return ffi.new("int16_t *", data)
+
+
+def monetdbe_decimal_to_int(data: int) -> ffi.CData:
+    return ffi.new("int *", data)
+
+
+def monetdbe_decimal_to_lng(data: int) -> ffi.CData:
+    return ffi.new("int64_t *", data)
+
+
 def monetdbe_int(data: int) -> ffi.CData:
     if data > 2 ** 32:
         return ffi.new("int64_t *", data)

monetdbe/_cffi/embed.h.j2

@@ -34,8 +34,19 @@ typedef enum {
  monetdbe_type_unknown
 } monetdbe_types;
 
+{% if newer_then_jul2021 %}
+typedef struct {
+ char* name;
+ unsigned int scale;
+ unsigned int digits;
+} monetdbe_sql_type;
+{% endif %}
+
 typedef struct {
  monetdbe_types type;
+ {% if newer_then_jul2021 %}
+ monetdbe_sql_type sql_type;
+ {% endif %}
  void *data;
  size_t count;
  char* name;
@@ -80,23 +91,27 @@ typedef struct {
  const char *trace_file;
 } monetdbe_options;
 
-typedef struct { monetdbe_types type; int8_t *data; size_t count; char *name; int8_t null_value; double scale; int (*is_null)(int8_t *value); } monetdbe_column_bool;
-typedef struct { monetdbe_types type; int8_t *data; size_t count; char *name; int8_t null_value; double scale; int (*is_null)(int8_t *value); } monetdbe_column_int8_t;
-typedef struct { monetdbe_types type; int16_t *data; size_t count; char *name; int16_t null_value; double scale; int (*is_null)(int16_t *value); } monetdbe_column_int16_t;
-typedef struct { monetdbe_types type; int32_t *data; size_t count; char *name; int32_t null_value; double scale; int (*is_null)(int32_t *value); } monetdbe_column_int32_t;
-typedef struct { monetdbe_types type; int64_t *data; size_t count; char *name; int64_t null_value; double scale; int (*is_null)(int64_t *value); } monetdbe_column_int64_t;
+{% if newer_then_jul2021 %}
+ {% set monetdbe_sql_type = 'monetdbe_sql_type sql_type;' %}
+{% endif %}
+
+typedef struct { monetdbe_types type; {{monetdbe_sql_type}} int8_t *data; size_t count; char *name; int8_t null_value; double scale; int (*is_null)(int8_t *value); } monetdbe_column_bool;
+typedef struct { monetdbe_types type; {{monetdbe_sql_type}} int8_t *data; size_t count; char *name; int8_t null_value; double scale; int (*is_null)(int8_t *value); } monetdbe_column_int8_t;
+typedef struct { monetdbe_types type; {{monetdbe_sql_type}} int16_t *data; size_t count; char *name; int16_t null_value; double scale; int (*is_null)(int16_t *value); } monetdbe_column_int16_t;
+typedef struct { monetdbe_types type; {{monetdbe_sql_type}} int32_t *data; size_t count; char *name; int32_t null_value; double scale; int (*is_null)(int32_t *value); } monetdbe_column_int32_t;
+typedef struct { monetdbe_types type; {{monetdbe_sql_type}} int64_t *data; size_t count; char *name; int64_t null_value; double scale; int (*is_null)(int64_t *value); } monetdbe_column_int64_t;
 
-typedef struct { monetdbe_types type; size_t *data; size_t count; char *name; size_t null_value; double scale; int (*is_null)(size_t *value); } monetdbe_column_size_t;
+typedef struct { monetdbe_types type; {{monetdbe_sql_type}} size_t *data; size_t count; char *name; size_t null_value; double scale; int (*is_null)(size_t *value); } monetdbe_column_size_t;
 
-typedef struct { monetdbe_types type; float *data; size_t count; char *name; float null_value; double scale; int (*is_null)(float *value); } monetdbe_column_float;
-typedef struct { monetdbe_types type; double *data; size_t count; char *name; double null_value; double scale; int (*is_null)(double *value); } monetdbe_column_double;
+typedef struct { monetdbe_types type; {{monetdbe_sql_type}} float *data; size_t count; char *name; float null_value; double scale; int (*is_null)(float *value); } monetdbe_column_float;
+typedef struct { monetdbe_types type; {{monetdbe_sql_type}} double *data; size_t count; char *name; double null_value; double scale; int (*is_null)(double *value); } monetdbe_column_double;
 
-typedef struct { monetdbe_types type; char * *data; size_t count; char *name; char * null_value; double scale; int (*is_null)(char * *value); } monetdbe_column_str;
-typedef struct { monetdbe_types type; monetdbe_data_blob *data; size_t count; char *name; monetdbe_data_blob null_value; double scale; int (*is_null)(monetdbe_data_blob *value); } monetdbe_column_blob;
+typedef struct { monetdbe_types type; {{monetdbe_sql_type}} char * *data; size_t count; char *name; char * null_value; double scale; int (*is_null)(char * *value); } monetdbe_column_str;
+typedef struct { monetdbe_types type; {{monetdbe_sql_type}} monetdbe_data_blob *data; size_t count; char *name; monetdbe_data_blob null_value; double scale; int (*is_null)(monetdbe_data_blob *value); } monetdbe_column_blob;
 
-typedef struct { monetdbe_types type; monetdbe_data_date *data; size_t count; char *name; monetdbe_data_date null_value; double scale; int (*is_null)(monetdbe_data_date *value); } monetdbe_column_date;
-typedef struct { monetdbe_types type; monetdbe_data_time *data; size_t count; char *name; monetdbe_data_time null_value; double scale; int (*is_null)(monetdbe_data_time *value); } monetdbe_column_time;
-typedef struct { monetdbe_types type; monetdbe_data_timestamp *data; size_t count; char *name; monetdbe_data_timestamp null_value; double scale; int (*is_null)(monetdbe_data_timestamp *value); } monetdbe_column_timestamp;
+typedef struct { monetdbe_types type; {{monetdbe_sql_type}} monetdbe_data_date *data; size_t count; char *name; monetdbe_data_date null_value; double scale; int (*is_null)(monetdbe_data_date *value); } monetdbe_column_date;
+typedef struct { monetdbe_types type; {{monetdbe_sql_type}} monetdbe_data_time *data; size_t count; char *name; monetdbe_data_time null_value; double scale; int (*is_null)(monetdbe_data_time *value); } monetdbe_column_time;
+typedef struct { monetdbe_types type; {{monetdbe_sql_type}} monetdbe_data_timestamp *data; size_t count; char *name; monetdbe_data_timestamp null_value; double scale; int (*is_null)(monetdbe_data_timestamp *value); } monetdbe_column_timestamp;
 
 extern const char *monetdbe_version(void);
 
@@ -126,7 +141,12 @@ extern char* monetdbe_cleanup_statement(monetdbe_database dbhdl, monetdbe_statem
 extern char* monetdbe_append(monetdbe_database dbhdl, const char* schema, const char* table, monetdbe_column **input, size_t column_count);
 extern const void* monetdbe_null(monetdbe_database dbhdl, monetdbe_types t);
 
+{% if newer_then_jul2021 %}
+extern char* monetdbe_get_columns(monetdbe_database dbhdl, const char* schema_name, const char *table_name, size_t *column_count, monetdbe_column **columns);
+{% else %}
 extern char* monetdbe_get_columns(monetdbe_database dbhdl, const char* schema_name, const char *table_name, size_t *column_count, char ***column_names, int **column_types);
+{% endif %}
+
 
 extern char* monetdbe_dump_database(monetdbe_database dbhdl, const char *backupfile);
 extern char* monetdbe_dump_table(monetdbe_database dbhdl, const char *schema_name, const char *table_name, const char *backupfile);

monetdbe/_cffi/internal.py

@@ -2,13 +2,17 @@
 from _warnings import warn
 from pathlib import Path
 from typing import Optional, Tuple, Any, Mapping, Iterator, Dict, TYPE_CHECKING
+from decimal import Decimal
+from collections import namedtuple
 
 import numpy as np
 
 from monetdbe._lowlevel import ffi, lib
 from monetdbe import exceptions
 from monetdbe._cffi.convert import make_string, monet_c_type_map, extract, numpy_monetdb_map
 from monetdbe._cffi.convert.bind import prepare_bind
+from monetdbe._cffi.convert.bind import monetdbe_decimal_to_bte, monetdbe_decimal_to_sht, monetdbe_decimal_to_int, monetdbe_decimal_to_lng
+
 from monetdbe._cffi.errors import check_error
 from monetdbe._cffi.types_ import monetdbe_result, monetdbe_database, monetdbe_column, monetdbe_statement
 
@@ -64,8 +68,29 @@ def get_autocommit() -> bool:
     return bool(value[0])
 
 
-def bind(statement: monetdbe_statement, data: Any, parameter_nr: int) -> None:
-    prepared = prepare_bind(data)
+TypeInfo = namedtuple('TypeInfo', ('impl_type', 'sql_type', 'scale'))
+
+
+def bind(statement: monetdbe_statement, data: Any, parameter_nr: int, type_info=None) -> None:
+    try:
+        _type_info = type_info[parameter_nr]
+        if (_type_info.sql_type == 'decimal'):
+            d = int(Decimal(data) * (Decimal(10) ** _type_info.scale))
+            if (_type_info.impl_type == 'bte'):
+                prepared = monetdbe_decimal_to_bte(d)
+            elif (_type_info.impl_type == 'sht'):
+                prepared = monetdbe_decimal_to_sht(d)
+            elif (_type_info.impl_type == 'int'):
+                prepared = monetdbe_decimal_to_int(d)
+            elif (_type_info.impl_type == 'lng'):
+                prepared = monetdbe_decimal_to_lng(d)
+            else:
+                raise NotImplementedError("Unknown decimal implementation type")
+        else:
+            prepared = prepare_bind(data)
+    except IndexError as e:
+        from monetdbe import exceptions
+        raise exceptions.ProgrammingError from e
     check_error(lib.monetdbe_bind(statement, prepared, parameter_nr))
 
 
@@ -256,9 +281,19 @@ def append(self, table: str, data: Mapping[str, np.ndarray], schema: str = 'sys'
 
     def prepare(self, query: str) -> monetdbe_statement:
         self._switch()
+
         stmt = ffi.new("monetdbe_statement **")
-        check_error(lib.monetdbe_prepare(self._monetdbe_database, str(query).encode(), stmt, ffi.NULL))
-        return stmt[0]
+        p_result = ffi.new("monetdbe_result **")
+        check_error(lib.monetdbe_prepare(self._monetdbe_database, str(query).encode(), stmt, p_result))
+
+        input_parameter_info = list()
+
+        for r in range(p_result[0].nrows):
+            if (extract(result_fetch(p_result[0], 3), r)) is None:
+                row = TypeInfo(impl_type=extract(result_fetch(p_result[0], 6), r), sql_type=extract(result_fetch(p_result[0], 0), r), scale=extract(result_fetch(p_result[0], 2), r))
+                input_parameter_info.append(row)
+
+        return stmt[0], input_parameter_info
 
     def cleanup_statement(self, statement: monetdbe_statement) -> None:
         self._switch()
@@ -275,24 +310,44 @@ def dump_table(self, schema_name: str, table_name: str, backupfile: Path):
 
     def get_columns(self, table: str, schema: str = 'sys') -> Iterator[Tuple[str, int]]:
         self._switch()
-        count_p = ffi.new('size_t *')
-        names_p = ffi.new('char ***')
-        types_p = ffi.new('int **')
+        count_p = ffi.new('size_t*')
+        columns_p = ffi.new('monetdbe_column**')
 
-        lib.monetdbe_get_columns(self._monetdbe_database, schema.encode(), table.encode(), count_p, names_p, types_p)
+        lib.monetdbe_get_columns(self._monetdbe_database, schema.encode(), table.encode(), count_p, columns_p)
 
         for i in range(count_p[0]):
-            name = ffi.string(names_p[0][i]).decode()
-            type_ = types_p[0][i]
+            name = ffi.string(columns_p[0][i].name).decode()
+            type_ = columns_p[0][i].type
             yield name, type_
 
 
 from monetdbe._cffi.branch import newer_then_jul2021
 if not newer_then_jul2021:
+    def bind(statement: monetdbe_statement, data: Any, parameter_nr: int, type_info=None) -> None:
+        prepared = prepare_bind(data)
+        check_error(lib.monetdbe_bind(statement, prepared, parameter_nr))
+
     def prepare(self, query: str) -> monetdbe_statement:
         self._switch()
         stmt = ffi.new("monetdbe_statement **")
         check_error(lib.monetdbe_prepare(self._monetdbe_database, str(query).encode(), stmt))
-        return stmt[0]
+        return stmt[0],
+
+    def get_columns(self, table: str, schema: str = 'sys') -> Iterator[Tuple[str, int]]:
+        self._switch()
+        count_p = ffi.new('size_t *')
+        names_p = ffi.new('char ***')
+        types_p = ffi.new('int **')
+
+        lib.monetdbe_get_columns(self._monetdbe_database, schema.encode(), table.encode(), count_p, names_p, types_p)
+
+        for i in range(count_p[0]):
+            name = ffi.string(names_p[0][i]).decode()
+            type_ = types_p[0][i]
+            yield name, type_
 
+    import sys
+    Module = sys.modules[__name__]
+    setattr(Module, 'bind', bind)
     setattr(Internal, 'prepare', prepare)
+    setattr(Internal, 'get_columns', get_columns)

monetdbe/cursors.py

@@ -126,10 +126,16 @@ def _execute_python(self, operation: str, parameters: parameters_type = None) ->
     def _execute_monetdbe(self, operation: str, parameters: parameters_type = None):
         from monetdbe._cffi.internal import bind, execute
         self._check_connection()
-        statement = self.connection.prepare(operation)
+        prepare_result = self.connection.prepare(operation)
+        statement = prepare_result[0]
+
+        self.connection.type_info = None
+        if len(prepare_result) == 2:
+            self.connection.type_info = prepare_result[1]
+
         if parameters:
             for index, parameter in enumerate(parameters):
-                bind(statement, parameter, index)
+                bind(statement, parameter, index, self.connection.type_info)
         self.connection.result, self.rowcount = execute(statement, make_result=True)
         self.connection.cleanup_statement(statement)
         self.connection.total_changes += self.rowcount

tests/test_dbapi.py

@@ -740,6 +740,28 @@ def test_Binary(self):
         b = monetdbe.Binary(b"\0'")
 
 
+from monetdbe._cffi.branch import newer_then_jul2021
+if newer_then_jul2021:
+    class DecimalTests(unittest.TestCase):
+        def setUp(self):
+            self.con = monetdbe.connect(":memory:")
+            self.cur = self.con.cursor()
+            # NOTE: (gijs) replaced binary type with blob
+            self.cur.execute("create table test(x DECIMAL(10,5))")
+
+        def tearDown(self):
+            self.cur.close()
+            self.con.close()
+
+        def test_decimals(self):
+            from decimal import Decimal
+            d = Decimal('12345.12345')
+            self.cur.execute("insert into test VALUES (?)", [d])
+            self.cur.execute("select x from test")
+            row = self.cur.fetchone()
+            assert(d == row[0])
+
+
 class ExtensionTests(unittest.TestCase):
     def test_ScriptStringSql(self):
         with monetdbe.connect(":memory:") as con: