From cfe1d10702df698305b5157dbe64c255573a50eb Mon Sep 17 00:00:00 2001 From: Georg Richter Date: Wed, 11 Oct 2023 14:50:23 +0200 Subject: [PATCH] CONPY-271: Added cursor.metadata property Similiar to description property, this property returns a dictionary with complete metadata. The dictionary contains the following keys: - catalog: catalog (always 'def') - schema: current schema - field: alias column name or if no alias was specified column name - org_field: original column name - table: alias table name or if no alias was specified table name - org_table: original table name - type: column type - charset: character set (utf8mb4 or binary) - length: The length of the column - max length: The maximum length of the column - decimals: The numer of decimals - flags: Flags (flags are defined in constants.FIELD_FLAG) - ext_type: Extended data type (types are defined in constants.EXT_FIELD_TYPE) This fixes also CONPY-270: Instead of checking BINARY_FLAG we now check character set for binary object types. --- include/docs/cursor.h | 25 ++++++- include/mariadb_python.h | 19 ++++- mariadb/constants/EXT_FIELD_TYPE.py | 20 ++++++ mariadb/mariadb_codecs.c | 63 ++++++++++++----- mariadb/mariadb_cursor.c | 92 +++++++++++++++++++++---- setup.py | 1 + testing/test/integration/test_cursor.py | 70 ++++++++++++++++++- 7 files changed, 256 insertions(+), 34 deletions(-) create mode 100644 mariadb/constants/EXT_FIELD_TYPE.py diff --git a/include/docs/cursor.h b/include/docs/cursor.h index cfbe684..e35660f 100644 --- a/include/docs/cursor.h +++ b/include/docs/cursor.h @@ -31,9 +31,30 @@ PyDoc_STRVAR( "- field_flags\n" "- table_name\n" "- original_column_name\n" - "- original_table_name\n\n" + "- original_table_name\n" "This attribute will be None for operations that do not return rows or if the cursor has\n" - "not had an operation invoked via the .execute*() method yet." + "not had an operation invoked via the .execute*() method yet.\n\n" + "extended field type information was added in MariaDB Connector/Python 1.1.8. It will be available\n" + "only, if the cursor or connection was created with optional parameter ext_field_type=True.\n" +); + +PyDoc_STRVAR( + cursor_metadata__doc__, + "Similiar to description property, this property returns a dictionary with complete metadata.\n\n" + "The dictionary contains the following keys:\n" + "- catalog: catalog (always 'def')\n" + "- schema: current schema\n" + "- field: alias column name or if no alias was specified column name\n" + "- org_field: original column name\n" + "- table: alias table name or if no alias was specified table name\n" + "- org_table: original table name\n" + "- type: column type\n" + "- charset: character set (utf8mb4 or binary)\n" + "- length: The length of the column\n" + "- max length: The maximum length of the column\n" + "- decimals: The numer of decimals\n" + "- flags: Flags (flags are defined in constants.FIELD_FLAG)\n" + "- ext_type: Extended data type (types are defined in constants.EXT_FIELD_TYPE)\n" ); PyDoc_STRVAR( diff --git a/include/mariadb_python.h b/include/mariadb_python.h index 1f1853e..9514085 100755 --- a/include/mariadb_python.h +++ b/include/mariadb_python.h @@ -127,7 +127,17 @@ enum enum_binary_command { enum enum_extended_field_type { EXT_TYPE_NONE=0, - EXT_TYPE_JSON=1 + EXT_TYPE_JSON, + EXT_TYPE_UUID, + EXT_TYPE_INET4, + EXT_TYPE_INET6, + EXT_TYPE_POINT, + EXT_TYPE_MULTIPOINT, + EXT_TYPE_LINESTRING, + EXT_TYPE_MULTILINESTRING, + EXT_TYPE_POLYGON, + EXT_TYPE_MULTIPOLYGON, + EXT_TYPE_GEOMETRYCOLLECTION }; enum enum_result_format @@ -162,6 +172,11 @@ enum enum_paramstyle PYFORMAT= 3 }; +typedef struct st_ext_field_type { + enum enum_extended_field_type ext_type; + MARIADB_CONST_STRING str; +} Mrdb_ExtFieldType; + typedef struct st_parser { MrdbString statement; MrdbString *keys; @@ -332,7 +347,7 @@ mariadb_throw_exception(void *handle, const char *message, ...); -enum enum_extended_field_type mariadb_extended_field_type(const MYSQL_FIELD *field); +Mrdb_ExtFieldType *mariadb_extended_field_type(const MYSQL_FIELD *field); PyObject * MrdbConnection_ping(MrdbConnection *self); diff --git a/mariadb/constants/EXT_FIELD_TYPE.py b/mariadb/constants/EXT_FIELD_TYPE.py new file mode 100644 index 0000000..ea09a79 --- /dev/null +++ b/mariadb/constants/EXT_FIELD_TYPE.py @@ -0,0 +1,20 @@ +""" +MariaDB EXT_FIELD_TYPE Constants + +These constants represent the extended field types supported by MariaDB. + +Extended field types are defined in module *mariadb.constants.EXT_FIELD_TYPE* +""" + +NONE =0 +JSON = 1 +UUID = 2 +INET4 = 3 +INET6 = 4 +POINT = 5 +MULTIPOINT = 6 +LINESTRING = 7 +MULTILINESTRING = 8 +POLYGON = 9 +MULTIPOLYGON = 10 +GEOMETRYCOLLECTION = 11 diff --git a/mariadb/mariadb_codecs.c b/mariadb/mariadb_codecs.c index 1b4e8e9..90a60ce 100644 --- a/mariadb/mariadb_codecs.c +++ b/mariadb/mariadb_codecs.c @@ -41,18 +41,46 @@ int codecs_datetime_init(void) return 0; } -enum enum_extended_field_type mariadb_extended_field_type(const MYSQL_FIELD *field) +Mrdb_ExtFieldType extended_field_types[] = { + {EXT_TYPE_JSON, {"json", 4}}, + {EXT_TYPE_UUID, {"uuid", 4}}, + {EXT_TYPE_INET4, {"inet4", 5}}, + {EXT_TYPE_INET6, {"inet6", 5}}, + {EXT_TYPE_POINT, {"point", 5}}, + {EXT_TYPE_MULTIPOINT, {"multipoint", 10}}, + {EXT_TYPE_LINESTRING, {"linestring", 10}}, + {EXT_TYPE_MULTILINESTRING, {"multilinestring", 15}}, + {EXT_TYPE_POLYGON, {"polygon", 7}}, + {EXT_TYPE_MULTIPOLYGON, {"multipolygon", 12}}, + {EXT_TYPE_GEOMETRYCOLLECTION, {"geometrycollection", 18}}, + {0, {NULL, 0}} +}; + +Mrdb_ExtFieldType *mariadb_extended_field_type(const MYSQL_FIELD *field) { #if MARIADB_PACKAGE_VERSION_ID > 30107 - MARIADB_CONST_STRING str; + MARIADB_CONST_STRING str= {0,0}; - if (!mariadb_field_attr(&str, field, MARIADB_FIELD_ATTR_FORMAT_NAME)) - { - if (str.length == 4 && !strncmp(str.str, "json", 4)) - return EXT_TYPE_JSON; + /* Extended field type has either format name or type name */ + if (mariadb_field_attr(&str, field, MARIADB_FIELD_ATTR_FORMAT_NAME)) + return NULL; + if (!str.length && mariadb_field_attr(&str, field, MARIADB_FIELD_ATTR_DATA_TYPE_NAME)) + return NULL; + if (str.length) { + uint8_t i= 0; + + while (extended_field_types[i].ext_type) + { + if (extended_field_types[i].str.length == str.length && + !strncmp(str.str, extended_field_types[i].str.str, str.length)) + { + return &extended_field_types[i]; + } + i++; + } } #endif - return EXT_TYPE_NONE; + return NULL; } /* @@ -430,9 +458,11 @@ field_fetch_fromtext(MrdbCursor *self, char *data, unsigned int column) { MYSQL_TIME tm; unsigned long *length; - enum enum_extended_field_type ext_type= mariadb_extended_field_type(&self->fields[column]); + Mrdb_ExtFieldType *ext_field_type; uint16_t type= self->fields[column].type; + ext_field_type= mariadb_extended_field_type(&self->fields[column]); + if (!data) type= MYSQL_TYPE_NULL; @@ -519,8 +549,7 @@ field_fetch_fromtext(MrdbCursor *self, char *data, unsigned int column) { self->fields[column].max_length= length[column]; } - if (self->fields[column].charsetnr== CHARSET_BINARY && - ext_type != EXT_TYPE_JSON) + if (self->fields[column].charsetnr== CHARSET_BINARY) { self->values[column]= PyBytes_FromStringAndSize((const char *)data, @@ -575,7 +604,7 @@ field_fetch_fromtext(MrdbCursor *self, char *data, unsigned int column) PyObject *val; enum enum_field_types type; - if (ext_type == EXT_TYPE_JSON) + if (ext_field_type && ext_field_type->ext_type == EXT_TYPE_JSON) type= MYSQL_TYPE_JSON; else type= self->fields[column].type; @@ -599,7 +628,9 @@ void field_fetch_callback(void *data, unsigned int column, unsigned char **row) { MrdbCursor *self= (MrdbCursor *)data; - enum enum_extended_field_type ext_type= mariadb_extended_field_type(&self->fields[column]); + Mrdb_ExtFieldType *ext_field_type; + + ext_field_type= mariadb_extended_field_type(&self->fields[column]); if (!row) { @@ -749,8 +780,7 @@ field_fetch_callback(void *data, unsigned int column, unsigned char **row) unsigned long length= mysql_net_field_length(row); if (length > self->fields[column].max_length) self->fields[column].max_length= length; - if (self->fields[column].charsetnr == CHARSET_BINARY && - ext_type != EXT_TYPE_JSON) + if (self->fields[column].charsetnr== CHARSET_BINARY) { self->values[column]= PyBytes_FromStringAndSize((const char *)*row, @@ -793,8 +823,7 @@ field_fetch_callback(void *data, unsigned int column, unsigned char **row) unsigned long utf8len; length= mysql_net_field_length(row); - if ((self->fields[column].flags & BINARY_FLAG || - self->fields[column].charsetnr == CHARSET_BINARY)) + if (self->fields[column].charsetnr== CHARSET_BINARY) { self->values[column]= PyBytes_FromStringAndSize((const char *)*row, @@ -820,7 +849,7 @@ field_fetch_callback(void *data, unsigned int column, unsigned char **row) PyObject *val; enum enum_field_types type; - if (ext_type == EXT_TYPE_JSON) + if (ext_field_type && ext_field_type->ext_type == EXT_TYPE_JSON) type= MYSQL_TYPE_JSON; else type= self->fields[column].type; diff --git a/mariadb/mariadb_cursor.c b/mariadb/mariadb_cursor.c index eb7b7fd..7c13014 100644 --- a/mariadb/mariadb_cursor.c +++ b/mariadb/mariadb_cursor.c @@ -1,4 +1,4 @@ -/***************************************************************************** + /***************************************************************************** Copyright (C) 2018-2020 Georg Richter and MariaDB Corporation AB This library is free software; you can redistribute it and/or @@ -106,12 +106,15 @@ static PyObject *Mariadb_row_count(MrdbCursor *self); static PyObject *Mariadb_row_number(MrdbCursor *self); static PyObject *MrdbCursor_warnings(MrdbCursor *self); static PyObject *MrdbCursor_closed(MrdbCursor *self); +static PyObject *MrdbCursor_metadata(MrdbCursor *self); static PyGetSetDef MrdbCursor_sets[]= { {"description", (getter)MrdbCursor_description, NULL, cursor_description__doc__, NULL}, + {"metadata", (getter)MrdbCursor_metadata, NULL, + cursor_metadata__doc__, NULL}, {"rowcount", (getter)Mariadb_row_count, NULL, NULL, NULL}, {"warnings", (getter)MrdbCursor_warnings, NULL, @@ -715,6 +718,70 @@ static int Mrdb_execute_direct(MrdbCursor *self, return rc; } +/* {{{ MrdbCursor_metadata */ +static PyObject *MrdbCursor_metadata(MrdbCursor *self) +{ + uint32_t i; + PyObject *dict; + const char *keys[14]= {"catalog", "schema", "field", "org_field", "table", + "org_table", "type", "charset", "length", + "max_length", "decimals", "flags", "ext_type_or_format"}; + PyObject *tuple[14]= {0}; + Mrdb_ExtFieldType *ext_field_type= NULL; + + if (!self->field_count) + Py_RETURN_NONE; + + if (PyErr_Occurred()) + return NULL; + + for (i=0; i < 13; i++) + if (!(tuple[i] = PyTuple_New(self->field_count))) + goto error; + + + for (i=0; i < self->field_count; i++) + { + PyTuple_SetItem(tuple[0], i, PyUnicode_FromString(self->fields[i].catalog)); + PyTuple_SetItem(tuple[1], i, PyUnicode_FromString(self->fields[i].db)); + PyTuple_SetItem(tuple[2], i, PyUnicode_FromString(self->fields[i].name)); + PyTuple_SetItem(tuple[3], i, PyUnicode_FromString(self->fields[i].org_name)); + PyTuple_SetItem(tuple[4], i, PyUnicode_FromString(self->fields[i].table)); + PyTuple_SetItem(tuple[5], i, PyUnicode_FromString(self->fields[i].org_table)); + PyTuple_SetItem(tuple[6], i, PyLong_FromLong((long)self->fields[i].type)); + PyTuple_SetItem(tuple[7], i, PyLong_FromLong((long)self->fields[i].charsetnr)); + PyTuple_SetItem(tuple[8], i, PyLong_FromLongLong((long long)self->fields[i].max_length)); + PyTuple_SetItem(tuple[9], i, PyLong_FromLongLong((long long)self->fields[i].length)); + PyTuple_SetItem(tuple[10], i, PyLong_FromLong((long)self->fields[i].decimals)); + PyTuple_SetItem(tuple[11], i, PyLong_FromLong((long)self->fields[i].flags)); + + if (ext_field_type= mariadb_extended_field_type(&self->fields[i])) + PyTuple_SetItem(tuple[12], i, PyLong_FromLong((long)ext_field_type->ext_type)); + else + PyTuple_SetItem(tuple[12], i, PyLong_FromLong((long)EXT_TYPE_NONE)); + } + + if (!(dict =PyDict_New())) + goto error; + + for (i=0; i < 13; i++) + { + if (PyDict_SetItem(dict, PyUnicode_FromString(keys[i]), tuple[i])) + goto error; + Py_DECREF(tuple[i]); + tuple[i]= NULL; + } + return dict; +error: + for (i=0; i < 13; i++) + if (tuple[i]) + Py_DECREF(tuple[i]); + if (dict) + Py_DECREF(dict); + return NULL; +} +/* }}}*/ + /* {{{ MrdbCursor_description PEP-249 description method() @@ -730,7 +797,6 @@ PyObject *MrdbCursor_description(MrdbCursor *self) if (PyErr_Occurred()) return NULL; - if (self->fields && field_count) { uint32_t i; @@ -742,22 +808,22 @@ PyObject *MrdbCursor_description(MrdbCursor *self) { uint32_t precision= 0; uint32_t decimals= 0; + uint8_t err= 0; MY_CHARSET_INFO cs; unsigned long display_length; long packed_len= 0; PyObject *desc; - enum enum_extended_field_type ext_type= mariadb_extended_field_type(&self->fields[i]); + Mrdb_ExtFieldType *ext_field_type= mariadb_extended_field_type(&self->fields[i]); display_length= self->fields[i].max_length > self->fields[i].length ? self->fields[i].max_length : self->fields[i].length; mysql_get_character_set_info(self->connection->mysql, &cs); if (cs.mbmaxlen > 1) { - packed_len= display_length; - display_length/= cs.mbmaxlen; - } - else { - packed_len= mysql_ps_fetch_functions[self->fields[i].type].pack_len; + packed_len= display_length; + display_length/= cs.mbmaxlen; + } else { + packed_len= mysql_ps_fetch_functions[self->fields[i].type].pack_len; } if (self->fields[i].decimals) @@ -770,9 +836,11 @@ PyObject *MrdbCursor_description(MrdbCursor *self) } } - if (ext_type == EXT_TYPE_JSON) - self->fields[i].type= MYSQL_TYPE_JSON; - + if (ext_field_type) + { + if (ext_field_type->ext_type == EXT_TYPE_JSON) + self->fields[i].type= MYSQL_TYPE_JSON; + } if (!(desc= Py_BuildValue("(sIIiIIOIsss)", self->fields[i].name, self->fields[i].type, @@ -788,7 +856,7 @@ PyObject *MrdbCursor_description(MrdbCursor *self) { Py_XDECREF(obj); mariadb_throw_exception(NULL, Mariadb_OperationalError, 0, - "Can't build descriptor record"); + "Can't build descriptor record"); return NULL; } PyTuple_SetItem(obj, i, desc); diff --git a/setup.py b/setup.py index 329e7a8..afed736 100644 --- a/setup.py +++ b/setup.py @@ -141,6 +141,7 @@ 'mariadb.constants.ERR', 'mariadb.constants.FIELD_FLAG', 'mariadb.constants.FIELD_TYPE', + 'mariadb.constants.EXT_FIELD_TYPE', 'mariadb.constants.INDICATOR', 'mariadb.constants.INFO', 'mariadb.constants.STATUS', diff --git a/testing/test/integration/test_cursor.py b/testing/test/integration/test_cursor.py index d1553fb..87a63fe 100644 --- a/testing/test/integration/test_cursor.py +++ b/testing/test/integration/test_cursor.py @@ -9,7 +9,7 @@ from decimal import Decimal import mariadb -from mariadb.constants import FIELD_TYPE, ERR, CURSOR, INDICATOR, CLIENT +from mariadb.constants import FIELD_TYPE, EXT_FIELD_TYPE, ERR, CURSOR, INDICATOR, CLIENT from test.base_test import create_connection, is_maxscale, is_mysql @@ -242,6 +242,52 @@ def test_buffered(self): self.assertEqual(row[0], 2) del cursor + def test_ext_field_types(self): + if is_mysql(): + self.skipTest("Skip (MySQL)") + cursor = self.connection.cursor() + cursor.execute("CREATE TEMPORARY TABLE t1 (a json, b uuid, c inet4, d inet6,"\ + "e point)") + cursor.execute("SELECT a,b,c,d,e FROM t1") + metadata= cursor.metadata + self.assertEqual(metadata["ext_type_or_format"][0], EXT_FIELD_TYPE.JSON) + self.assertEqual(metadata["type"][0], FIELD_TYPE.BLOB) + self.assertEqual(metadata["ext_type_or_format"][1], EXT_FIELD_TYPE.UUID) + self.assertEqual(metadata["type"][1], FIELD_TYPE.STRING) + self.assertEqual(metadata["ext_type_or_format"][2], EXT_FIELD_TYPE.INET4) + self.assertEqual(metadata["type"][2], FIELD_TYPE.STRING) + self.assertEqual(metadata["ext_type_or_format"][3], EXT_FIELD_TYPE.INET6) + self.assertEqual(metadata["type"][3], FIELD_TYPE.STRING) + self.assertEqual(metadata["ext_type_or_format"][4], EXT_FIELD_TYPE.POINT) + self.assertEqual(metadata["type"][4], FIELD_TYPE.GEOMETRY) + cursor.execute("SELECT a,b,c,d,e FROM t1 WHERE 1=?", (1,)) + metadata= cursor.metadata + self.assertEqual(metadata["ext_type_or_format"][0], EXT_FIELD_TYPE.JSON) + self.assertEqual(metadata["type"][0], FIELD_TYPE.BLOB) + self.assertEqual(metadata["ext_type_or_format"][1], EXT_FIELD_TYPE.UUID) + self.assertEqual(metadata["type"][1], FIELD_TYPE.STRING) + self.assertEqual(metadata["ext_type_or_format"][2], EXT_FIELD_TYPE.INET4) + self.assertEqual(metadata["type"][2], FIELD_TYPE.STRING) + self.assertEqual(metadata["ext_type_or_format"][3], EXT_FIELD_TYPE.INET6) + self.assertEqual(metadata["type"][3], FIELD_TYPE.STRING) + self.assertEqual(metadata["ext_type_or_format"][4], EXT_FIELD_TYPE.POINT) + self.assertEqual(metadata["type"][4], FIELD_TYPE.GEOMETRY) + + cursor.close() + + def test_xfield_types(self): + cursor = self.connection.cursor() + cursor.execute("SELECT a,b,c,d,e FROM t1") + description= cursor.description + self.assertEqual(12, len(description[0])) + self.assertEqual(description[0][11], EXT_FIELD_TYPE.JSON) + cursor.execute("SELECT a,b,c,d,e FROM t1 WHERE 1=?", (1,)) + description= cursor.description + self.assertEqual(12, len(description[0])) + self.assertEqual(description[0][11], EXT_FIELD_TYPE.JSON) + cursor.close() + connection.close() + def test_xfield_types(self): cursor = self.connection.cursor() fieldinfo = mariadb.fieldinfo() @@ -1490,7 +1536,29 @@ def test_conpy225(self): self.assertEqual(cursor.affected_rows, 1) self.assertEqual(cursor.rowcount, 1) + def test_conpy270(self): + connection = create_connection() + cursor = connection.cursor() + + cursor.execute("create or replace table t1 (a uuid)") + cursor.execute("insert into t1 values (uuid())") + + # text protocol + cursor.execute("select a from t1") + self.assertEqual(cursor.description[0][1], mariadb.STRING); + cursor.fetchall() + + # binary protcol + cursor.execute("select a from t1 WHERE 1=?", (1,)) + self.assertEqual(cursor.description[0][1], mariadb.STRING); + cursor.fetchall() + + cursor.close() + connection.close() + def test_conpy269(self): + if is_mysql(): + self.skipTest("Skip (MySQL)") connection = create_connection() cursor = connection.cursor() cursor.execute("SELECT 1 UNION SELECT 2")