From dc872780c6a24ea03e4e0be28e1faf5682458e7e Mon Sep 17 00:00:00 2001 From: Clinton James Date: Sun, 8 Sep 2019 18:16:46 -0500 Subject: [PATCH 1/9] Initial commit + Creates object in sqlite3 + row_factory works + Get by index + Get by dict + Case insensitive + Underscore acceptable replacement for space or dash + AttributeError works + IndexError works --- Modules/_sqlite/NamedRow/named_row.py | 49 +++ Modules/_sqlite/NamedRow/test.py | 100 +++++ Modules/_sqlite/NamedRow/timing.py | 51 +++ Modules/_sqlite/NamedRow/timing.sh | 36 ++ .../_sqlite/NamedRow/unit_test_named_row.py | 113 ++++++ Modules/_sqlite/module.c | 4 + Modules/_sqlite/named_row.c | 344 ++++++++++++++++++ Modules/_sqlite/named_row.h | 38 ++ setup.py | 1 + 9 files changed, 736 insertions(+) create mode 100644 Modules/_sqlite/NamedRow/named_row.py create mode 100644 Modules/_sqlite/NamedRow/test.py create mode 100644 Modules/_sqlite/NamedRow/timing.py create mode 100644 Modules/_sqlite/NamedRow/timing.sh create mode 100644 Modules/_sqlite/NamedRow/unit_test_named_row.py create mode 100644 Modules/_sqlite/named_row.c create mode 100644 Modules/_sqlite/named_row.h diff --git a/Modules/_sqlite/NamedRow/named_row.py b/Modules/_sqlite/NamedRow/named_row.py new file mode 100644 index 00000000000000..56447dc1a824b5 --- /dev/null +++ b/Modules/_sqlite/NamedRow/named_row.py @@ -0,0 +1,49 @@ +import sqlite3 + + +class NamedRow: + """Sqlite3.NamedRow as a namedtuple like object. + """ + + def __init__(self, cursor, values): + """Create a named row. + + The title_tuple comes from tuple(x[0] for x in defination. However, + there is no need to keep creating it anew. + """ + self._datum = dict(zip(("a", "b"), values)) + + # self._titles = ("a", "b") + # 1 msec for creating the tuple + # self._titles = tuple(x[0] for x in cursor.description) + # self._values = values + + def __len__(self): + return len(self._values) + + def __iter__(self): + """Dict.items() equivalent.""" + # print("__iter__") + return iter(zip(self._titles, self._values)) + + def __getitem__(self, key): + """Sequence or Mapping lookup.""" + # print(f"__getitem__[{repr(key)}]") + # return self._values[self._titles.index(key)] + return self._datum[key] + + def __getattr__(self, name): + try: + # print(f"__getattr__[{repr(name)}]") + # return self._values[self._titles.index(name)] + return self._datum[name] + except ValueError: + # print(f"__getattr__[{repr(name)}] raise AttributeError") + raise AttributeError(name) + + def __contains__(self, key): + """True if Row has the specified key, else False.""" + return key in self._titles + + def __repr__(self): + return "".format(repr(self._titles), id(self)) diff --git a/Modules/_sqlite/NamedRow/test.py b/Modules/_sqlite/NamedRow/test.py new file mode 100644 index 00000000000000..1dd9ac26e69b47 --- /dev/null +++ b/Modules/_sqlite/NamedRow/test.py @@ -0,0 +1,100 @@ +import sqlite3 +import pytest +from named_row import NamedRow + + +def create_database(): + """Database ('a', '-dash') + Filled with a 1000 of (1, 2) + """ + conn = sqlite3.connect(":memory:") + conn.execute('CREATE TABLE t(a, "-dash")') + for i in range(1000): + conn.execute("INSERT INTO t VALUES(?,?)", (1, 2)) + conn.row_factory = NamedRow + return conn + + +@pytest.fixture +def db(scope="session"): + yield create_database() + + +@pytest.fixture +def row(db): + """An instance of NamedRow.""" + cursor = db.execute("SELECT * FROM t LIMIT 1") + row = cursor.fetchone() + yield row + + +def test_01_database(db): + assert "NamedRow" in repr(db.row_factory) + cursor = db.execute("SELECT * FROM t LIMIT 1") + assert ("a", "-dash") == tuple(x[0] for x in cursor.description) + + +def test_02_is_NamedRow(row): + assert "NamedRow" in row.__class__.__name__ + assert "a" in row + repr_str = repr(row) + assert "NamedRow" in repr_str + assert "-dash" in repr_str + + +def test_11_iterable_keyValue_pairs(row): + assert [("a", 1), ("-dash", 2)] == [x for x in row] + + +def test_12_iterable_into_dict(row): + assert dict((("a", 1), ("-dash", 2))) == dict(row) + + +def test_30_index_by_number(row): + assert row[0] == 1 + assert row[1] == 2 + + +def test_31_index_by_string(row): + assert row["a"] == 1 + assert row["-dash"] == 2 + + +def test_32_access_by_attribute(row): + assert row.a == 1 + + +def test_50_IndexError(row): + try: + row[10] + raise RuntimeError("Expected IndexError") + except IndexError: + pass + + +def test_51_ValueError(row): + try: + row["alphabet"] + raise RuntimeError("Expected ValueError") + except ValueError: + pass + + +def test_52_AttributeError(row): + try: + row.alphabet + raise RuntimeError("Expected AttributeError") + except AttributeError: + pass + + +def test_53_not_assignable(row): + try: + row["a"] = 100 + raise RuntimeError("Expecting TypeError object does no support item assignment") + except TypeError: + pass + + +if __name__ == "__main__": + pass diff --git a/Modules/_sqlite/NamedRow/timing.py b/Modules/_sqlite/NamedRow/timing.py new file mode 100644 index 00000000000000..e11de18bb03b69 --- /dev/null +++ b/Modules/_sqlite/NamedRow/timing.py @@ -0,0 +1,51 @@ +import collections +import timeit +import sqlite3 +from named_row import NamedRow + + +SETUP = """\ +import sqlite3, collections +from named_row import NamedRow +conn = sqlite3.connect(":memory:") +conn.execute('CREATE TABLE t(a, b)') +for i in range(1000): + conn.execute("INSERT INTO t VALUES(?,?)", (i, i)) +cursor = conn.cursor() +""" +SQLITE3_ROW = SETUP + "cursor.row_factory=sqlite3.Row" +SQLITE3_NAMED = SETUP + "cursor.row_factory=sqlite3.NamedRow" +NAMED_TUPLE = SETUP + "cursor.row_factory=collections.namedtuple('Row', ['a','b'])" +NAMED_ROW = SETUP + "cursor.row_factory=NamedRow" + +NUMBER = 1000 +INDEX = """for x in cursor.execute("SELECT * FROM t").fetchall(): x[0]""" +DICTS = """for x in cursor.execute("SELECT * FROM t").fetchall(): x['a']""" +ATTRS = """for x in cursor.execute("SELECT * FROM t").fetchall(): x.a""" + +tests = ( + (SETUP, INDEX, "sqlite3 indx"), + (SQLITE3_ROW, INDEX, "sqlite3.Row indx"), + (SQLITE3_ROW, DICTS, "sqlite3.Row dict"), + (SQLITE3_NAMED, INDEX, "sqlite3.NamedRow indx"), + (SQLITE3_NAMED, DICTS, "sqlite3.NamedRow dict"), + (SQLITE3_NAMED, ATTRS, "sqlite3.NamedRow attr"), + (NAMED_TUPLE, INDEX, "namedtuple indx"), + (NAMED_TUPLE, ATTRS, "namedtuple attr"), + (NAMED_ROW, DICTS, "NamedRow dict"), + (NAMED_ROW, ATTRS, "NamedRow attr"), +) + +base = 0 + +for setup, test, title in tests: + try: + v = timeit.timeit(test, setup=setup, number=1000) + except AttributeError: + print("AttributeError", title) + continue + + if setup == SETUP: + base = v + percent = int(100 * ((v / base)) - 100) + print(f"+{percent}%", round(1000 * v), "usec", title) diff --git a/Modules/_sqlite/NamedRow/timing.sh b/Modules/_sqlite/NamedRow/timing.sh new file mode 100644 index 00000000000000..3c27f331cb4add --- /dev/null +++ b/Modules/_sqlite/NamedRow/timing.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# https://bugs.python.org/issue13299 +# https://bugs.python.org/file37673/sqlite_namedtuplerow.patch +# https://github.com/python/cpython/tree/master/Modules/_sqlite + + +IMPORTS="import sqlite3, collections, named_row; con=sqlite3.connect(':memory:')" +CREATE="con.execute('CREATE TABLE t(a,b)')" +POPULATE="for i in range(1000): con.execute('INSERT INTO t VALUES (?,?)', (i, 10000-i))" +TEST="con.execute('SELECT * FROM t').fetchall()" + +echo "tuple" +python -m timeit -s "$IMPORTS" -s "$CREATE" -s "$POPULATE" -- "for x in $TEST: x[0]" + +factory="con.row_factory=sqlite3.Row" +echo -e "\nsqlite3.row # $factory" +python -m timeit -s "$IMPORTS" -s "$factory" -s"$CREATE" -s "$POPULATE" -- "for x in $TEST: x['a']" + +factory="con.row_factory=collections.namedtuple('Row', ['a','b'])" +echo -e "\nnamedtuple # $factory" +python -m timeit -s "$IMPORTS" -s "$factory" -s"$CREATE" -s "$POPULATE" -- "for x in $TEST: x.a" + +factory="con.row_factory=named_row.NamedRow" +echo -e "\nNamedRow # $factory [0]" +python -m timeit -s "$IMPORTS" -s "$factory" -s"$CREATE" -s "$POPULATE" -- "for x in $TEST: x[0]" + +factory="con.row_factory=named_row.NamedRow" +echo -e "\nNamedRow # $factory" +python -m timeit -s "$IMPORTS" -s "$factory" -s"$CREATE" -s "$POPULATE" -- "for x in $TEST: x.a" + +# tuple 804 540 +# sqlite3.row 983 673 +# namedtuple 1.38 923 +# NamedRow[0] 1.34 1.03 +# NamedRow.a 2.33 1.5 diff --git a/Modules/_sqlite/NamedRow/unit_test_named_row.py b/Modules/_sqlite/NamedRow/unit_test_named_row.py new file mode 100644 index 00000000000000..ff195e2cc53e08 --- /dev/null +++ b/Modules/_sqlite/NamedRow/unit_test_named_row.py @@ -0,0 +1,113 @@ +import sqlite3 +import unittest + + +class NamedRowSetup(unittest.TestCase): + + expected_list = [("a", 1), ("dash-name", 2), ("space name", 3)] + + def setUp(self): + """Database ('a', 'dash-name', 'space name') + Filled with a 100 of (1, 2) + """ + conn = sqlite3.connect(":memory:") + conn.execute('CREATE TABLE t(a, "dash-name", "space name")') + for i in range(100): + conn.execute("INSERT INTO t VALUES(?,?,?)", (1, 2, 3)) + conn.row_factory = sqlite3.NamedRow + self.conn = conn + + def row(self): + """An instance of NamedRow.""" + return self.conn.execute("SELECT * FROM t LIMIT 1").fetchone() + + +class Test01_sqlite3_connection(NamedRowSetup): + def test_database(self): + self.assertIn("NamedRow", repr(self.conn.row_factory)) + cursor = self.conn.execute("SELECT * FROM t LIMIT 1") + self.assertSequenceEqual( + ("a", "dash-name", "space name"), tuple(x[0] for x in cursor.description) + ) + + +class Test02_NamedRow_class(NamedRowSetup): + def test1_is_NamedRow(self): + row = self.row() + self.assertIn("NamedRow", row.__class__.__name__) + self.assertIn("NamedRow", repr(row)) + + def test2_len(self): + self.assertEqual(3, len(self.row())) + + +class Test03_NamedRow_access(NamedRowSetup): + def test1_getitem_by_number(self): + row = self.row() + self.assertEqual(1, row[0]) + self.assertEqual(2, row[1]) + self.assertEqual(3, row[2]) + + def test2_getitem_by_string(self): + row = self.row() + self.assertEqual(1, row["a"]) + self.assertEqual(2, row["dash-name"]) + self.assertEqual(3, row["space name"]) + + def test3_getitem_by_string_acceptable_replacement(self): + row = self.row() + # underscore is acceptable for space and dash + self.assertEqual(2, row["dash_name"]) + self.assertEqual(3, row["space_name"]) + + def test4_getitem_by_string_case_insensitive(self): + row = self.row() + self.assertEqual(1, row["A"]) + self.assertEqual(2, row["DaSh-nAmE"]) + self.assertEqual(3, row["Space Name"]) + + self.assertEqual(2, row["dAsH_NaMe"]) + self.assertEqual(3, row["Space_Name"]) + + def test5_access_by_attribute(self): + row = self.row() + self.assertEqual(1, row.a) + self.assertEqual(2, row.dash_name) + self.assertEqual(3, row.SPACE_Name) + + +class Test_NamedRow_iterator: + def test1_iterable_keyValue_pairs(self): + row = self.row() + self.assertSequenceEqual(self.expected_list, [x for x in row]) + self.assertSequenceEqual(self.expected_list, tuple(row)) + self.assertSequenceEqual(self.expected_list, list(row)) + + def test2_iterable_into_dict(self): + row = self.row() + self.assertDictEqual(dict(self.expected_list), dict(row)) + + +class Test_NamedRow_exceptions(NamedRowSetup): + def test_IndexError(self): + row = self.row() + with self.assertRaises(IndexError): + row[9] + with self.assertRaises(IndexError): + row["no_such_field"] + + def test_AttributeError(self): + row = self.row() + with self.assertRaises(AttributeError): + row.alphabet + + def test_not_assignable(self): + row = self.row() + with self.assertRaises(TypeError): + row["a"] = 100 + with self.assertRaises(TypeError): + row.a = 100 + + +if __name__ == "__main__": + unittest.main() diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index d5c353ea7bee83..cf6d9df34daa39 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -28,6 +28,7 @@ #include "prepare_protocol.h" #include "microprotocols.h" #include "row.h" +#include "named_row.h" #if SQLITE_VERSION_NUMBER >= 3003003 #define HAVE_SHARED_CACHE @@ -355,6 +356,7 @@ PyMODINIT_FUNC PyInit__sqlite3(void) module = PyModule_Create(&_sqlite3module); if (!module || + (named_row_setup_types() < 0) || (pysqlite_row_setup_types() < 0) || (pysqlite_cursor_setup_types() < 0) || (pysqlite_connection_setup_types() < 0) || @@ -374,6 +376,8 @@ PyMODINIT_FUNC PyInit__sqlite3(void) PyModule_AddObject(module, "PrepareProtocol", (PyObject*) &pysqlite_PrepareProtocolType); Py_INCREF(&pysqlite_RowType); PyModule_AddObject(module, "Row", (PyObject*) &pysqlite_RowType); + Py_INCREF(&pysqlite_NamedRowType); + PyModule_AddObject(module, "NamedRow", (PyObject*) &pysqlite_NamedRowType); if (!(dict = PyModule_GetDict(module))) { goto error; diff --git a/Modules/_sqlite/named_row.c b/Modules/_sqlite/named_row.c new file mode 100644 index 00000000000000..6dca33fe03f685 --- /dev/null +++ b/Modules/_sqlite/named_row.c @@ -0,0 +1,344 @@ +/* named_row.c - an enhansed namedtuple for database rows + * + * Copyright (C) 2019 Clinton James + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +#include "named_row.h" +#include "cursor.h" +/* + * Space efficient. Identical to Row + * No fancy repr. Just fields? + * Is hashable hash(row) + * Is unpackable + * Much faster than NamedTuple + * Allows for fields that would be invalid in Python, a['-dash'] + * Is sliceable, row[1:3] + * Iterator like Objects/dictobject.c:3435 dictiter_new + * conversion to dictionary with dict(row) + * for key, value in row: + * queries like count(*) will return ('count(*)', #) so use 'count(*) AS qty' + */ + +void named_row_dealloc(pysqlite_NamedRow* self) +{ + Py_XDECREF(self->data); + Py_XDECREF(self->description); + + Py_TYPE(self)->tp_free((PyObject*)self); +} + +static PyObject * +named_row_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + pysqlite_NamedRow *self; + PyObject* data; + pysqlite_Cursor* cursor; + + assert(type != NULL && type->tp_alloc != NULL); + + if (!_PyArg_NoKeywords("NamedRow", kwargs)) + return NULL; + if (!PyArg_ParseTuple(args, "OO", &cursor, &data)) + return NULL; + + if (!PyObject_TypeCheck((PyObject*)cursor, &pysqlite_CursorType)) { + PyErr_SetString(PyExc_TypeError, "instance of cursor required for first argument"); + return NULL; + } + + if (!PyTuple_Check(data)) { + PyErr_SetString(PyExc_TypeError, "tuple required for second argument"); + return NULL; + } + + self = (pysqlite_NamedRow *) type->tp_alloc(type, 0); + if (self == NULL) + return NULL; + + Py_INCREF(data); + self->data = data; + + Py_INCREF(cursor->description); + self->description = cursor->description; + + return (PyObject *) self; +} + +PyObject* named_row_item(pysqlite_NamedRow* self, Py_ssize_t idx) +{ + PyObject* item = PyTuple_GetItem(self->data, idx); + Py_XINCREF(item); + return item; +} + +Py_ssize_t named_row_find_index(pysqlite_NamedRow* self, const char* name) +{ + // Return the index into cursor.description matching name + Py_ssize_t nitems = PyTuple_Size(self->description); + + for (Py_ssize_t i = 0; i < nitems; i++) { + PyObject* obj = PyTuple_GET_ITEM(PyTuple_GET_ITEM(self->description, i), 0); + const char *cursor_field = PyUnicode_AsUTF8(obj); + if (!cursor_field) { + continue; + } + + const char* p1 = name; + const char* p2 = cursor_field; + + while (1) { + // Done checking if at the end of either string + if ((*p1 == (char)0) || (*p2 == (char)0)) { + break; + } + + // case insensitive comparison and accept '_' instead of space or dash + if ((*p1 | 0x20) != (*p2 | 0x20) && *p1 != '_' && *p2 !=' ' && *p2 !='-') { + break; + } + + p1++; + p2++; + } + + // Found it if at the end of both strings + if ((*p1 == (char)0) && (*p2 == (char)0)) { + return i; + } + } + return -1; +} + +static PyObject* named_row_getattro(pysqlite_NamedRow* self, char* attr_name) +{ + Py_ssize_t idx = named_row_find_index(self, attr_name); + if (idx < 0) { + PyTypeObject *tp = Py_TYPE(self); + /* TODO change 'FIELD' to 'field'*/ + PyErr_Format(PyExc_AttributeError, "'%.50s' object has no FIELD %d '%U'", + tp->tp_name, idx, attr_name); + return NULL; + } + PyObject *value = PyTuple_GET_ITEM(self->data, idx); + Py_INCREF(value); + return value; +} + +PyObject* named_row_subscript(pysqlite_NamedRow* self, PyObject* idx) +{ + PyObject* item; + + // Process integer index + if (PyLong_Check(idx)) { + Py_ssize_t _idx = PyNumber_AsSsize_t(idx, PyExc_IndexError); + if (_idx == -1 && PyErr_Occurred()) + return NULL; + if (_idx < 0) + _idx += PyTuple_GET_SIZE(self->data); + item = PyTuple_GetItem(self->data, _idx); + Py_XINCREF(item); + return item; + } + // Process string, dict like + else if (PyUnicode_Check(idx)) { + const char* key = PyUnicode_AsUTF8(idx); + if (key == NULL) + return NULL; + + Py_ssize_t idx = named_row_find_index(self, key); + if (idx < 0) { + PyErr_SetString(PyExc_IndexError, "No item with that key"); + return NULL; + } + item = PyTuple_GetItem(self->data, idx); + Py_INCREF(item); + return item; + } + else if (PySlice_Check(idx)) { + return PyObject_GetItem(self->data, idx); + } + else { + PyErr_SetString(PyExc_IndexError, "Index must be int or string"); + return NULL; + } +} + +static Py_ssize_t named_row_length(pysqlite_NamedRow* self) +{ + return PyTuple_GET_SIZE(self->data); +} + +static PyObject* named_row_iter(pysqlite_NamedRow* self) +{ + // Return a tuple of key/value tuples + Py_ssize_t item_count = PyTuple_Size(self->data); + PyObject* result = PyTuple_New(item_count); + if (!result) { + return NULL; + } + + for (Py_ssize_t i = 0; i < item_count; i++) { + PyObject *item = PyTuple_New(2); + if (item == NULL) { + Py_DECREF(result); + return NULL; + } + + // Get the field title + PyObject *key = PyTuple_GET_ITEM(PyTuple_GET_ITEM(self->description, i), 0); + Py_INCREF(key); + PyTuple_SET_ITEM(item, 0, key); + + // Get the value + PyObject *value = PyTuple_GET_ITEM(self->data, i); + Py_INCREF(value); + PyTuple_SET_ITEM(item, 1, value); + + // Add to result tuple + PyTuple_SET_ITEM(result, i, item); + } + return result; +} + +static int named_row_contains(pysqlite_NamedRow* self, PyObject* name) +{ + if (PyUnicode_Check(name)) { + const char* key = PyUnicode_AsUTF8(name); + if (key == NULL) + return -1; + + Py_ssize_t idx = named_row_find_index(self, key); + return idx >= 0 ? 1 : 0; + } + return -1; +} + +static Py_hash_t named_row_hash(pysqlite_NamedRow* self) +{ + return PyObject_Hash(self->description) ^ PyObject_Hash(self->data); +} + +static PyObject* named_row_richcompare(pysqlite_NamedRow *self, PyObject *_other, int opid) +{ + if (opid != Py_EQ && opid != Py_NE) + Py_RETURN_NOTIMPLEMENTED; + + if (PyType_IsSubtype(Py_TYPE(_other), &pysqlite_NamedRowType)) { + pysqlite_NamedRow *other = (pysqlite_NamedRow *)_other; + PyObject *res = PyObject_RichCompare(self->description, other->description, opid); + if ((opid == Py_EQ && res == Py_True) + || (opid == Py_NE && res == Py_False)) { + Py_DECREF(res); + return PyObject_RichCompare(self->data, other->data, opid); + } + } + Py_RETURN_NOTIMPLEMENTED; +} + +PyMappingMethods named_row_as_mapping = { + /* mp_length */ (lenfunc)named_row_length, + /* mp_subscript */ (binaryfunc)named_row_subscript, + /* mp_ass_subscript */ (objobjargproc)0, +}; + +static PySequenceMethods named_row_as_sequence = { + /* sq_length */ (lenfunc)named_row_length, + /* sq_concat */ 0, + /* sq_repeat */ 0, + /* sq_item */ (ssizeargfunc)named_row_item, + /* sq_slice */ (ssizeargfunc)named_row_item, + /* sq_ass_item */ 0, + /* sq_ass_slice */ 0, + /* sq_sq_contains */ (objobjproc)named_row_contains, +}; + + +PyTypeObject pysqlite_NamedRowType = { + PyVarObject_HEAD_INIT(NULL, 0) + /* tp_name */ MODULE_NAME ".NamedRow", + /* tp_basicsize */ sizeof(pysqlite_NamedRow), + /* tp_itemsize */ 0, + /* tp_dealloc */ (destructor)named_row_dealloc, + /* tp_vectorcall_offset */ 0, + /* tp_getattr */ 0, + /* tp_setattr */ 0, + /* tp_as_async */ 0, + /* tp_repr */ 0, + /* tp_as_number */ 0, + /* tp_as_sequence */ &named_row_as_sequence, + /* tp_as_mapping */ 0, + /* tp_hash */ (hashfunc)named_row_hash, + /* tp_call */ 0, + /* tp_str */ 0, + /* tp_getattro */ 0, //(getattrofunc)named_row_getattro, + /* tp_setattro */ 0, + /* tp_as_buffer */ 0, + /* tp_flags */ Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, + /* tp_doc */ 0, /* TODO */ + /* tp_traverse */ (traverseproc)0, + /* tp_clear */ 0, + /* tp_richcompare */ (richcmpfunc)named_row_richcompare, + /* tp_weaklistoffset */ 0, + /* tp_iter */ (getiterfunc)named_row_iter, + /* tp_iternext */ 0, + /* tp_methods */ 0, + /* tp_members */ 0, + /* tp_getset */ 0, + /* tp_base */ 0, + /* tp_dict */ 0, + /* tp_descr_get */ 0, + /* tp_descr_set */ 0, + /* tp_dictoffset */ 0, + /* tp_init */ 0, + /* tp_alloc */ 0, + /* tp_new */ 0, + /* tp_free */ 0 +}; + +extern int named_row_setup_types(void) +{ + pysqlite_NamedRowType.tp_new = named_row_new; + pysqlite_NamedRowType.tp_as_mapping = &named_row_as_mapping; + pysqlite_NamedRowType.tp_as_sequence = &named_row_as_sequence; + return PyType_Ready(&pysqlite_NamedRowType); +} + +/***************** Iterator *****************/ +// Objects/tupleobject.c 1006 +/* +typedef struct { + PyObject_HEAD + Py_ssize_t it_index; + //named_row *it_seq; // Set to NULL when iterator is exhausted +} named_row_iter_object; + + +static void named_row_iter_dealloc(named_row_iter_object *it) +{ + _PyObject_GC_UNTRACK(it); + Py_XDECREF(it->it_seq); + PyObject_GC_Del(it); +} + +static PyObject* named_row_iter_traverse(named_row_iter_object *it, visitproc visit, void* arg) +{ + Py_Visit(it->it_seq); + return 0; +} +*/ diff --git a/Modules/_sqlite/named_row.h b/Modules/_sqlite/named_row.h new file mode 100644 index 00000000000000..244d486f564085 --- /dev/null +++ b/Modules/_sqlite/named_row.h @@ -0,0 +1,38 @@ +/* named_row.h - an enhanced namedtuple for database rows + * + * Copyright (C) 2019 Clinton James + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +#ifndef PYSQLITE_NAMED_ROW_H +#define PYSQLITE_NAMED_ROW_H +#define PY_SSIZE_T_CLEAN +#include "Python.h" + +typedef struct _NamedRow +{ + PyObject_HEAD + PyObject* data; + PyObject* description; +} pysqlite_NamedRow; + +extern PyTypeObject pysqlite_NamedRowType; + +int named_row_setup_types(void); + +#endif diff --git a/setup.py b/setup.py index 02f523c42d355f..a79d8704648bac 100644 --- a/setup.py +++ b/setup.py @@ -1418,6 +1418,7 @@ def detect_sqlite(self): '_sqlite/module.c', '_sqlite/prepare_protocol.c', '_sqlite/row.c', + '_sqlite/named_row.c', '_sqlite/statement.c', '_sqlite/util.c', ] From 1e4e907c90a52a12371334f0f31dab00cee78fdd Mon Sep 17 00:00:00 2001 From: Clinton James Date: Wed, 18 Sep 2019 00:17:32 -0500 Subject: [PATCH 2/9] Working version passing tests --- Lib/sqlite3/dbapi2.py | 2 + .../_sqlite/NamedRow/unit_test_named_row.py | 201 +++++----- Modules/_sqlite/module.c | 2 +- Modules/_sqlite/named_row.c | 344 +++++++++--------- Modules/_sqlite/named_row.h | 4 +- PCbuild/_sqlite3.vcxproj | 4 +- PCbuild/_sqlite3.vcxproj.filters | 8 +- 7 files changed, 315 insertions(+), 250 deletions(-) diff --git a/Lib/sqlite3/dbapi2.py b/Lib/sqlite3/dbapi2.py index 991682ce9ef3b7..5b5ed8fa7da738 100644 --- a/Lib/sqlite3/dbapi2.py +++ b/Lib/sqlite3/dbapi2.py @@ -52,6 +52,8 @@ def TimestampFromTicks(ticks): Binary = memoryview collections.abc.Sequence.register(Row) +collections.abc.Sequence.register(NamedRow) + def register_adapters_and_converters(): def adapt_date(val): diff --git a/Modules/_sqlite/NamedRow/unit_test_named_row.py b/Modules/_sqlite/NamedRow/unit_test_named_row.py index ff195e2cc53e08..b2622f2f7dd938 100644 --- a/Modules/_sqlite/NamedRow/unit_test_named_row.py +++ b/Modules/_sqlite/NamedRow/unit_test_named_row.py @@ -2,111 +2,146 @@ import unittest -class NamedRowSetup(unittest.TestCase): - - expected_list = [("a", 1), ("dash-name", 2), ("space name", 3)] - +class NamedRowFactoryTests(unittest.TestCase): def setUp(self): - """Database ('a', 'dash-name', 'space name') - Filled with a 100 of (1, 2) - """ - conn = sqlite3.connect(":memory:") - conn.execute('CREATE TABLE t(a, "dash-name", "space name")') - for i in range(100): - conn.execute("INSERT INTO t VALUES(?,?,?)", (1, 2, 3)) - conn.row_factory = sqlite3.NamedRow - self.conn = conn - - def row(self): - """An instance of NamedRow.""" - return self.conn.execute("SELECT * FROM t LIMIT 1").fetchone() - - -class Test01_sqlite3_connection(NamedRowSetup): - def test_database(self): - self.assertIn("NamedRow", repr(self.conn.row_factory)) - cursor = self.conn.execute("SELECT * FROM t LIMIT 1") - self.assertSequenceEqual( - ("a", "dash-name", "space name"), tuple(x[0] for x in cursor.description) - ) - - -class Test02_NamedRow_class(NamedRowSetup): - def test1_is_NamedRow(self): - row = self.row() + self.conn = sqlite3.connect(":memory:") + self.conn.row_factory = sqlite3.NamedRow + + def test_instance(self): + """row_factory creates NamedRow instances""" + row = self.conn.execute("SELECT 1, 2").fetchone() + self.assertIsInstance(row, sqlite3.NamedRow) self.assertIn("NamedRow", row.__class__.__name__) self.assertIn("NamedRow", repr(row)) + self.assertIn("optimized row_factory with namedtuple", sqlite3.NamedRow.__doc__) - def test2_len(self): - self.assertEqual(3, len(self.row())) + def test_by_number_index(self): + """Get values by numbered index [0]""" + row = self.conn.execute("SELECT 1, 2").fetchone() + # len(row) + self.assertEqual(2, len(row)) + self.assertEqual(row[0], 1) + self.assertEqual(row[1], 2) + self.assertEqual(row[-1], 2) + self.assertEqual(row[-2], 1) -class Test03_NamedRow_access(NamedRowSetup): - def test1_getitem_by_number(self): - row = self.row() - self.assertEqual(1, row[0]) - self.assertEqual(2, row[1]) - self.assertEqual(3, row[2]) + # Check past boundary + with self.assertRaises(IndexError): + row[len(row) + 1] + with self.assertRaises(IndexError): + row[-1 * (len(row) + 1)] - def test2_getitem_by_string(self): - row = self.row() + def test_by_string_index(self): + """Get values by string index ['field']""" + row = self.conn.execute( + 'SELECT 1 AS a, 2 AS "dash-name", ' '3 AS "space name", 4 AS "四"' + ).fetchone() self.assertEqual(1, row["a"]) self.assertEqual(2, row["dash-name"]) self.assertEqual(3, row["space name"]) + self.assertEqual(4, row["四"]) - def test3_getitem_by_string_acceptable_replacement(self): - row = self.row() - # underscore is acceptable for space and dash - self.assertEqual(2, row["dash_name"]) - self.assertEqual(3, row["space_name"]) - - def test4_getitem_by_string_case_insensitive(self): - row = self.row() + # Case insensitive self.assertEqual(1, row["A"]) - self.assertEqual(2, row["DaSh-nAmE"]) - self.assertEqual(3, row["Space Name"]) + self.assertEqual(2, row["DaSh-NaMe"]) + self.assertEqual(3, row["SPACE name"]) - self.assertEqual(2, row["dAsH_NaMe"]) - self.assertEqual(3, row["Space_Name"]) + # Underscore as acceptable replacement for dash and space + self.assertEqual(2, row["dash_NAME"]) + self.assertEqual(3, row["space_name"]) - def test5_access_by_attribute(self): - row = self.row() + def test_by_attribute(self): + """Get values by attribute row.field""" + row = self.conn.execute( + 'SELECT 1 AS a, 2 AS "dash-name", ' '3 AS "space name", 4 AS "四"' + ).fetchone() self.assertEqual(1, row.a) self.assertEqual(2, row.dash_name) - self.assertEqual(3, row.SPACE_Name) - - -class Test_NamedRow_iterator: - def test1_iterable_keyValue_pairs(self): - row = self.row() - self.assertSequenceEqual(self.expected_list, [x for x in row]) - self.assertSequenceEqual(self.expected_list, tuple(row)) - self.assertSequenceEqual(self.expected_list, list(row)) - - def test2_iterable_into_dict(self): - row = self.row() - self.assertDictEqual(dict(self.expected_list), dict(row)) - - -class Test_NamedRow_exceptions(NamedRowSetup): - def test_IndexError(self): - row = self.row() - with self.assertRaises(IndexError): - row[9] + self.assertEqual(3, row.space_name) + self.assertEqual(4, row.四) + self.assertEqual(2, row.DASH_NAME) + + def test_contains_field(self): + row = self.conn.execute("SELECT 1 AS a, 2 AS b").fetchone() + self.assertIn("a", row) + self.assertIn("A", row) + + def test_slice(self): + """Does NamedRow slice like a list.""" + row = self.conn.execute("SELECT 1, 2, 3, 4").fetchone() + self.assertEqual(row[0:0], ()) + self.assertEqual(row[0:1], (1,)) + self.assertEqual(row[1:3], (2, 3)) + self.assertEqual(row[3:1], ()) + # Explicit bounds are optional. + self.assertEqual(row[1:], (2, 3, 4)) + self.assertEqual(row[:3], (1, 2, 3)) + # Slices can use negative indices. + self.assertEqual(row[-2:-1], (3,)) + self.assertEqual(row[-2:], (3, 4)) + # Slicing supports steps. + self.assertEqual(row[0:4:2], (1, 3)) + self.assertEqual(row[3:0:-2], (4, 2)) + + def test_hash_comparison(self): + row1 = self.conn.execute("SELECT 1 AS a, 2 AS b").fetchone() + row2 = self.conn.execute("SELECT 1 AS a, 2 AS b").fetchone() + row3 = self.conn.execute("SELECT 1 AS a, 3 AS b").fetchone() + + self.assertEqual(row1, row1) + self.assertEqual(row1, row2) + self.assertTrue(row2 != row3) + + self.assertFalse(row1 != row1) + self.assertFalse(row1 != row2) + self.assertFalse(row2 == row3) + + self.assertEqual(row1, row2) + self.assertEqual(hash(row1), hash(row2)) + self.assertNotEqual(row1, row3) + self.assertNotEqual(hash(row1), hash(row3)) + + def test_fake_cursor_class(self): + # Issue #24257: Incorrect use of PyObject_IsInstance() caused + # segmentation fault. + # Issue #27861: Also applies for cursor factory. + class FakeCursor(str): + __class__ = sqlite3.Cursor + + self.assertRaises(TypeError, self.conn.cursor, FakeCursor) + self.assertRaises(TypeError, sqlite3.NamedRow, FakeCursor(), ()) + + def test_iterable(self): + """Is NamedRow iterable.""" + row = self.conn.execute("SELECT 1 AS a, 2 AS b").fetchone() + for col in row: + pass + + def test_iterable_pairs(self): + expected = (("a", 1), ("b", 2)) + row = self.conn.execute("SELECT 1 AS a, 2 AS b").fetchone() + self.assertSequenceEqual(expected, [x for x in row]) + self.assertSequenceEqual(expected, tuple(row)) + self.assertSequenceEqual(expected, list(row)) + self.assertSequenceEqual(dict(expected), dict(row)) + + def test_expected_exceptions(self): + row = self.conn.execute('SELECT 1 AS a, 2 AS "dash-name"').fetchone() with self.assertRaises(IndexError): row["no_such_field"] - - def test_AttributeError(self): - row = self.row() with self.assertRaises(AttributeError): - row.alphabet - - def test_not_assignable(self): - row = self.row() + row.no_such_field with self.assertRaises(TypeError): - row["a"] = 100 + row[0] = 100 with self.assertRaises(TypeError): + row["a"] = 100 + with self.assertRaises(TypeError) as err: row.a = 100 + self.assertIn("does not support item assignment", str(err.exception)) + + with self.assertRaises(TypeError): + setattr(row, "a", 100) if __name__ == "__main__": diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index cf6d9df34daa39..4c19fd4de0091b 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -356,8 +356,8 @@ PyMODINIT_FUNC PyInit__sqlite3(void) module = PyModule_Create(&_sqlite3module); if (!module || - (named_row_setup_types() < 0) || (pysqlite_row_setup_types() < 0) || + (pysqlite_named_row_setup_types() < 0) || (pysqlite_cursor_setup_types() < 0) || (pysqlite_connection_setup_types() < 0) || (pysqlite_cache_setup_types() < 0) || diff --git a/Modules/_sqlite/named_row.c b/Modules/_sqlite/named_row.c index 6dca33fe03f685..268056c266695a 100644 --- a/Modules/_sqlite/named_row.c +++ b/Modules/_sqlite/named_row.c @@ -1,4 +1,4 @@ -/* named_row.c - an enhansed namedtuple for database rows +/* named_row.c - an row_factory with attribute access for database rows * * Copyright (C) 2019 Clinton James * @@ -21,32 +21,31 @@ #include "named_row.h" #include "cursor.h" -/* - * Space efficient. Identical to Row - * No fancy repr. Just fields? - * Is hashable hash(row) - * Is unpackable - * Much faster than NamedTuple - * Allows for fields that would be invalid in Python, a['-dash'] - * Is sliceable, row[1:3] - * Iterator like Objects/dictobject.c:3435 dictiter_new - * conversion to dictionary with dict(row) - * for key, value in row: - * queries like count(*) will return ('count(*)', #) so use 'count(*) AS qty' - */ -void named_row_dealloc(pysqlite_NamedRow* self) +PyDoc_STRVAR(named_row__doc__, +"NamedRow an optimized row_factory with column name access to row fields\n\n\ +NamedRow extends Row to support mapping access by attribute, column name, and \ +index. Column names access is case insensitive.\n\ +row.field == row.FIELD\n\ +For attribute names that would be illegal due to dashes or spaces, an \ +underscore is an acceptable replacement.\n\ +row['dash-name'] == row.dash_name \n\ +For functions in SQL remember to use the 'AS' statement to name the output \ +field.\n\ +connection.execute('SELECT count(*) AS count').fetchone()\n\n\ +Iteration gives field/value pairs similar to dict.items()."); + +static void +named_row_dealloc(pysqlite_NamedRow* self) { Py_XDECREF(self->data); - Py_XDECREF(self->description); - + Py_XDECREF(self->fields); Py_TYPE(self)->tp_free((PyObject*)self); } -static PyObject * +static PyObject* named_row_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { - pysqlite_NamedRow *self; PyObject* data; pysqlite_Cursor* cursor; @@ -58,7 +57,8 @@ named_row_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) return NULL; if (!PyObject_TypeCheck((PyObject*)cursor, &pysqlite_CursorType)) { - PyErr_SetString(PyExc_TypeError, "instance of cursor required for first argument"); + PyErr_SetString(PyExc_TypeError, + "instance of cursor required for first argument"); return NULL; } @@ -67,7 +67,7 @@ named_row_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) return NULL; } - self = (pysqlite_NamedRow *) type->tp_alloc(type, 0); + pysqlite_NamedRow *self = (pysqlite_NamedRow *) type->tp_alloc(type, 0); if (self == NULL) return NULL; @@ -75,26 +75,30 @@ named_row_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) self->data = data; Py_INCREF(cursor->description); - self->description = cursor->description; + self->fields = cursor->description; return (PyObject *) self; } -PyObject* named_row_item(pysqlite_NamedRow* self, Py_ssize_t idx) +// Get data value by index +static PyObject* +named_row_item(pysqlite_NamedRow* self, Py_ssize_t idx) { PyObject* item = PyTuple_GetItem(self->data, idx); Py_XINCREF(item); return item; } -Py_ssize_t named_row_find_index(pysqlite_NamedRow* self, const char* name) +// Find index from field name +static Py_ssize_t +named_row_find_index(pysqlite_NamedRow* self, const char* name) { - // Return the index into cursor.description matching name - Py_ssize_t nitems = PyTuple_Size(self->description); + Py_ssize_t nitems = PyTuple_Size(self->fields); for (Py_ssize_t i = 0; i < nitems; i++) { - PyObject* obj = PyTuple_GET_ITEM(PyTuple_GET_ITEM(self->description, i), 0); + PyObject* obj = PyTuple_GET_ITEM(PyTuple_GET_ITEM(self->fields, i), 0); const char *cursor_field = PyUnicode_AsUTF8(obj); + if (!cursor_field) { continue; } @@ -108,15 +112,16 @@ Py_ssize_t named_row_find_index(pysqlite_NamedRow* self, const char* name) break; } - // case insensitive comparison and accept '_' instead of space or dash - if ((*p1 | 0x20) != (*p2 | 0x20) && *p1 != '_' && *p2 !=' ' && *p2 !='-') { + // case insensitive comparison + if ((*p1 | 0x20) != (*p2 | 0x20) && + // accept '_' instead of space or dash + *p1 != '_' && *p2 !=' ' && *p2 !='-') + { break; } - p1++; p2++; } - // Found it if at the end of both strings if ((*p1 == (char)0) && (*p2 == (char)0)) { return i; @@ -125,43 +130,63 @@ Py_ssize_t named_row_find_index(pysqlite_NamedRow* self, const char* name) return -1; } -static PyObject* named_row_getattro(pysqlite_NamedRow* self, char* attr_name) +static PyObject* +named_row_getattro(pysqlite_NamedRow* self, PyObject* attr_obj) { + // Calling function checked `name` is PyUnicode, no need to check here. + Py_INCREF(attr_obj); + const char* attr_name = PyUnicode_AsUTF8(attr_obj); Py_ssize_t idx = named_row_find_index(self, attr_name); - if (idx < 0) { - PyTypeObject *tp = Py_TYPE(self); - /* TODO change 'FIELD' to 'field'*/ - PyErr_Format(PyExc_AttributeError, "'%.50s' object has no FIELD %d '%U'", - tp->tp_name, idx, attr_name); - return NULL; + Py_DECREF(attr_obj); + + if (idx >= 0) { + PyObject* value = PyTuple_GET_ITEM(self->data, idx); + Py_INCREF(value); + return value; } - PyObject *value = PyTuple_GET_ITEM(self->data, idx); - Py_INCREF(value); - return value; + + return PyObject_GenericGetAttr((PyObject *)self, attr_obj); +} + +static int +named_row_setattro(PyObject* self, PyObject* name, PyObject* value) +{ + PyErr_Format(PyExc_TypeError, + "'%.50s' object does not support item assignment", + Py_TYPE(self)->tp_name); + return -1; } -PyObject* named_row_subscript(pysqlite_NamedRow* self, PyObject* idx) +// Find the data value by either number or string +static PyObject* +named_row_index(pysqlite_NamedRow* self, PyObject* index) { PyObject* item; - // Process integer index - if (PyLong_Check(idx)) { - Py_ssize_t _idx = PyNumber_AsSsize_t(idx, PyExc_IndexError); - if (_idx == -1 && PyErr_Occurred()) + if (PyLong_Check(index)) { + // Process integer index + Py_ssize_t idx = PyNumber_AsSsize_t(index, PyExc_IndexError); + if (idx == -1 && PyErr_Occurred()) { return NULL; - if (_idx < 0) - _idx += PyTuple_GET_SIZE(self->data); - item = PyTuple_GetItem(self->data, _idx); + } + if (idx < 0) { + idx += PyTuple_GET_SIZE(self->data); + } + item = PyTuple_GetItem(self->data, idx); Py_XINCREF(item); return item; } - // Process string, dict like - else if (PyUnicode_Check(idx)) { - const char* key = PyUnicode_AsUTF8(idx); - if (key == NULL) + else if (PyUnicode_Check(index)) { + // Process string index, dict like + Py_ssize_t idx = 0; + const char* key = PyUnicode_AsUTF8(index); + + if (key == NULL) { return NULL; + } + + idx = named_row_find_index(self, key); - Py_ssize_t idx = named_row_find_index(self, key); if (idx < 0) { PyErr_SetString(PyExc_IndexError, "No item with that key"); return NULL; @@ -170,53 +195,24 @@ PyObject* named_row_subscript(pysqlite_NamedRow* self, PyObject* idx) Py_INCREF(item); return item; } - else if (PySlice_Check(idx)) { - return PyObject_GetItem(self->data, idx); + else if (PySlice_Check(index)) { + return PyObject_GetItem(self->data, index); } else { - PyErr_SetString(PyExc_IndexError, "Index must be int or string"); + PyErr_SetString(PyExc_IndexError, "Index must be int or str"); return NULL; } } -static Py_ssize_t named_row_length(pysqlite_NamedRow* self) +static Py_ssize_t +named_row_length(pysqlite_NamedRow* self) { return PyTuple_GET_SIZE(self->data); } -static PyObject* named_row_iter(pysqlite_NamedRow* self) -{ - // Return a tuple of key/value tuples - Py_ssize_t item_count = PyTuple_Size(self->data); - PyObject* result = PyTuple_New(item_count); - if (!result) { - return NULL; - } - - for (Py_ssize_t i = 0; i < item_count; i++) { - PyObject *item = PyTuple_New(2); - if (item == NULL) { - Py_DECREF(result); - return NULL; - } - - // Get the field title - PyObject *key = PyTuple_GET_ITEM(PyTuple_GET_ITEM(self->description, i), 0); - Py_INCREF(key); - PyTuple_SET_ITEM(item, 0, key); - - // Get the value - PyObject *value = PyTuple_GET_ITEM(self->data, i); - Py_INCREF(value); - PyTuple_SET_ITEM(item, 1, value); - - // Add to result tuple - PyTuple_SET_ITEM(result, i, item); - } - return result; -} - -static int named_row_contains(pysqlite_NamedRow* self, PyObject* name) +// Check for a field name in NamedRow +static int +named_row_contains(pysqlite_NamedRow* self, PyObject* name) { if (PyUnicode_Check(name)) { const char* key = PyUnicode_AsUTF8(name); @@ -229,89 +225,64 @@ static int named_row_contains(pysqlite_NamedRow* self, PyObject* name) return -1; } -static Py_hash_t named_row_hash(pysqlite_NamedRow* self) +static Py_hash_t +named_row_hash(pysqlite_NamedRow* self) { - return PyObject_Hash(self->description) ^ PyObject_Hash(self->data); + return PyObject_Hash(self->fields) ^ PyObject_Hash(self->data); } -static PyObject* named_row_richcompare(pysqlite_NamedRow *self, PyObject *_other, int opid) +static PyObject* +named_row_richcompare(pysqlite_NamedRow *self, PyObject *_other, int opid) { - if (opid != Py_EQ && opid != Py_NE) + if (opid != Py_EQ && opid != Py_NE) { Py_RETURN_NOTIMPLEMENTED; + } if (PyType_IsSubtype(Py_TYPE(_other), &pysqlite_NamedRowType)) { pysqlite_NamedRow *other = (pysqlite_NamedRow *)_other; - PyObject *res = PyObject_RichCompare(self->description, other->description, opid); - if ((opid == Py_EQ && res == Py_True) - || (opid == Py_NE && res == Py_False)) { + PyObject *res = PyObject_RichCompare(self->fields, + other->fields, opid); + if ((opid == Py_EQ && res == Py_True) || + (opid == Py_NE && res == Py_False)) + { Py_DECREF(res); return PyObject_RichCompare(self->data, other->data, opid); } + Py_XDECREF(res); } Py_RETURN_NOTIMPLEMENTED; } PyMappingMethods named_row_as_mapping = { - /* mp_length */ (lenfunc)named_row_length, - /* mp_subscript */ (binaryfunc)named_row_subscript, - /* mp_ass_subscript */ (objobjargproc)0, + .mp_length = (lenfunc)named_row_length, + .mp_subscript = (binaryfunc)named_row_index, }; static PySequenceMethods named_row_as_sequence = { - /* sq_length */ (lenfunc)named_row_length, - /* sq_concat */ 0, - /* sq_repeat */ 0, - /* sq_item */ (ssizeargfunc)named_row_item, - /* sq_slice */ (ssizeargfunc)named_row_item, - /* sq_ass_item */ 0, - /* sq_ass_slice */ 0, - /* sq_sq_contains */ (objobjproc)named_row_contains, + .sq_length = (lenfunc)named_row_length, + .sq_item = (ssizeargfunc)named_row_item, + .was_sq_slice = (ssizeargfunc)named_row_item, + .sq_contains = (objobjproc)named_row_contains, }; +static PyObject* named_row_iterator(pysqlite_NamedRow* self); PyTypeObject pysqlite_NamedRowType = { PyVarObject_HEAD_INIT(NULL, 0) - /* tp_name */ MODULE_NAME ".NamedRow", - /* tp_basicsize */ sizeof(pysqlite_NamedRow), - /* tp_itemsize */ 0, - /* tp_dealloc */ (destructor)named_row_dealloc, - /* tp_vectorcall_offset */ 0, - /* tp_getattr */ 0, - /* tp_setattr */ 0, - /* tp_as_async */ 0, - /* tp_repr */ 0, - /* tp_as_number */ 0, - /* tp_as_sequence */ &named_row_as_sequence, - /* tp_as_mapping */ 0, - /* tp_hash */ (hashfunc)named_row_hash, - /* tp_call */ 0, - /* tp_str */ 0, - /* tp_getattro */ 0, //(getattrofunc)named_row_getattro, - /* tp_setattro */ 0, - /* tp_as_buffer */ 0, - /* tp_flags */ Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, - /* tp_doc */ 0, /* TODO */ - /* tp_traverse */ (traverseproc)0, - /* tp_clear */ 0, - /* tp_richcompare */ (richcmpfunc)named_row_richcompare, - /* tp_weaklistoffset */ 0, - /* tp_iter */ (getiterfunc)named_row_iter, - /* tp_iternext */ 0, - /* tp_methods */ 0, - /* tp_members */ 0, - /* tp_getset */ 0, - /* tp_base */ 0, - /* tp_dict */ 0, - /* tp_descr_get */ 0, - /* tp_descr_set */ 0, - /* tp_dictoffset */ 0, - /* tp_init */ 0, - /* tp_alloc */ 0, - /* tp_new */ 0, - /* tp_free */ 0 + .tp_name = MODULE_NAME ".NamedRow", + .tp_doc = named_row__doc__, + .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, + .tp_basicsize = sizeof(pysqlite_NamedRow), + .tp_dealloc = (destructor)named_row_dealloc, + .tp_as_sequence = &named_row_as_sequence, + .tp_hash = (hashfunc)named_row_hash, + .tp_getattro = (getattrofunc)named_row_getattro, + .tp_setattro = (setattrofunc)named_row_setattro, + .tp_richcompare = (richcmpfunc)named_row_richcompare, + .tp_iter = (getiterfunc)named_row_iterator, }; -extern int named_row_setup_types(void) +extern int pysqlite_named_row_setup_types(void) { pysqlite_NamedRowType.tp_new = named_row_new; pysqlite_NamedRowType.tp_as_mapping = &named_row_as_mapping; @@ -319,26 +290,75 @@ extern int named_row_setup_types(void) return PyType_Ready(&pysqlite_NamedRowType); } + /***************** Iterator *****************/ -// Objects/tupleobject.c 1006 -/* -typedef struct { + +typedef struct _NamedRowIter{ PyObject_HEAD - Py_ssize_t it_index; - //named_row *it_seq; // Set to NULL when iterator is exhausted -} named_row_iter_object; + Py_ssize_t idx; + Py_ssize_t len; + pysqlite_NamedRow *row; +} NamedRowIter; +static void +named_row_iter_dealloc(NamedRowIter *it) +{ + Py_XDECREF(it->row); +} -static void named_row_iter_dealloc(named_row_iter_object *it) +static PyObject* +named_row_iter_iter(PyObject *self) { - _PyObject_GC_UNTRACK(it); - Py_XDECREF(it->it_seq); - PyObject_GC_Del(it); + Py_INCREF(self); + return self; } -static PyObject* named_row_iter_traverse(named_row_iter_object *it, visitproc visit, void* arg) +static PyObject* +named_row_iter_next(PyObject *self) { - Py_Visit(it->it_seq); - return 0; + NamedRowIter *p = (NamedRowIter *)self; + if (p->idx >= p->len) + { + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } + PyObject *item = PyTuple_New(2); + + // Get the field name + PyObject *cursor_descript_field = PyTuple_GET_ITEM(p->row->fields, p->idx); + PyObject *key = PyTuple_GET_ITEM(cursor_descript_field, 0); + Py_INCREF(key); + PyTuple_SET_ITEM(item, 0, key); + + // Get the value + PyObject *value = PyTuple_GET_ITEM(p->row->data, p->idx); + Py_INCREF(value); + PyTuple_SET_ITEM(item, 1, value); + (p->idx)++; + return item; +} + +PyTypeObject NamedRowIterType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = MODULE_NAME ".NamedRowIter", + .tp_doc = "Internal NamedRow iterator", + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_basicsize = sizeof(NamedRowIter), + .tp_dealloc = (destructor)named_row_iter_dealloc, + .tp_iter = named_row_iter_iter, + .tp_iternext = named_row_iter_next, +}; + +static PyObject* +named_row_iterator(pysqlite_NamedRow* self) +{ + NamedRowIter *iter = PyObject_New(NamedRowIter, &NamedRowIterType); + if (!iter) { + return NULL; + } + Py_INCREF(self); + iter->idx = 0; + iter->len = PyTuple_GET_SIZE(self->data); + iter->row = self; + return (PyObject *)iter; } -*/ diff --git a/Modules/_sqlite/named_row.h b/Modules/_sqlite/named_row.h index 244d486f564085..aaa44f322cc8d2 100644 --- a/Modules/_sqlite/named_row.h +++ b/Modules/_sqlite/named_row.h @@ -28,11 +28,11 @@ typedef struct _NamedRow { PyObject_HEAD PyObject* data; - PyObject* description; + PyObject* fields; } pysqlite_NamedRow; extern PyTypeObject pysqlite_NamedRowType; -int named_row_setup_types(void); +int pysqlite_named_row_setup_types(void); #endif diff --git a/PCbuild/_sqlite3.vcxproj b/PCbuild/_sqlite3.vcxproj index 7e0062692b8f83..d53f174fc62792 100644 --- a/PCbuild/_sqlite3.vcxproj +++ b/PCbuild/_sqlite3.vcxproj @@ -105,6 +105,7 @@ + @@ -116,6 +117,7 @@ + @@ -135,4 +137,4 @@ - \ No newline at end of file + diff --git a/PCbuild/_sqlite3.vcxproj.filters b/PCbuild/_sqlite3.vcxproj.filters index dce77c96a80599..140b0ba1b4ee0f 100644 --- a/PCbuild/_sqlite3.vcxproj.filters +++ b/PCbuild/_sqlite3.vcxproj.filters @@ -30,6 +30,9 @@ Header Files + + Header Files + Header Files @@ -59,6 +62,9 @@ Source Files + + Source Files + Source Files @@ -66,4 +72,4 @@ Source Files - \ No newline at end of file + From 901307358a29ff63118eca395e06659f3705110b Mon Sep 17 00:00:00 2001 From: Clinton James Date: Sun, 6 Oct 2019 16:40:13 -0500 Subject: [PATCH 3/9] Fix Issue38175 Test both the cursor descriptor and data. --- Modules/_sqlite/named_row.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/Modules/_sqlite/named_row.c b/Modules/_sqlite/named_row.c index 268056c266695a..f9cdeb059b0843 100644 --- a/Modules/_sqlite/named_row.c +++ b/Modules/_sqlite/named_row.c @@ -238,17 +238,16 @@ named_row_richcompare(pysqlite_NamedRow *self, PyObject *_other, int opid) Py_RETURN_NOTIMPLEMENTED; } - if (PyType_IsSubtype(Py_TYPE(_other), &pysqlite_NamedRowType)) { + if (PyObject_TypeCheck(_other, &pysqlite_NamedRowType)) { pysqlite_NamedRow *other = (pysqlite_NamedRow *)_other; - PyObject *res = PyObject_RichCompare(self->fields, - other->fields, opid); - if ((opid == Py_EQ && res == Py_True) || - (opid == Py_NE && res == Py_False)) - { - Py_DECREF(res); + int eq = PyObject_RichCompareBool(self->fields, other->fields, Py_EQ); + if (eq < 0) { + return NULL; + } + if (eq) { return PyObject_RichCompare(self->data, other->data, opid); } - Py_XDECREF(res); + return PyBool_FromLong(opid != Py_EQ); } Py_RETURN_NOTIMPLEMENTED; } From 958e579d9563c9ee992185740551e3822be79702 Mon Sep 17 00:00:00 2001 From: Clinton James Date: Thu, 10 Oct 2019 00:32:57 -0500 Subject: [PATCH 4/9] Add improved timing test --- Modules/_sqlite/NamedRow/timing.py | 114 ++++++++++++++++++----------- Modules/_sqlite/NamedRow/timing.sh | 36 --------- 2 files changed, 73 insertions(+), 77 deletions(-) delete mode 100644 Modules/_sqlite/NamedRow/timing.sh diff --git a/Modules/_sqlite/NamedRow/timing.py b/Modules/_sqlite/NamedRow/timing.py index e11de18bb03b69..b12e60552af6b8 100644 --- a/Modules/_sqlite/NamedRow/timing.py +++ b/Modules/_sqlite/NamedRow/timing.py @@ -1,51 +1,83 @@ import collections import timeit +import random import sqlite3 from named_row import NamedRow -SETUP = """\ -import sqlite3, collections -from named_row import NamedRow -conn = sqlite3.connect(":memory:") -conn.execute('CREATE TABLE t(a, b)') -for i in range(1000): - conn.execute("INSERT INTO t VALUES(?,?)", (i, i)) -cursor = conn.cursor() -""" -SQLITE3_ROW = SETUP + "cursor.row_factory=sqlite3.Row" -SQLITE3_NAMED = SETUP + "cursor.row_factory=sqlite3.NamedRow" -NAMED_TUPLE = SETUP + "cursor.row_factory=collections.namedtuple('Row', ['a','b'])" -NAMED_ROW = SETUP + "cursor.row_factory=NamedRow" - -NUMBER = 1000 -INDEX = """for x in cursor.execute("SELECT * FROM t").fetchall(): x[0]""" -DICTS = """for x in cursor.execute("SELECT * FROM t").fetchall(): x['a']""" -ATTRS = """for x in cursor.execute("SELECT * FROM t").fetchall(): x.a""" - -tests = ( - (SETUP, INDEX, "sqlite3 indx"), - (SQLITE3_ROW, INDEX, "sqlite3.Row indx"), - (SQLITE3_ROW, DICTS, "sqlite3.Row dict"), - (SQLITE3_NAMED, INDEX, "sqlite3.NamedRow indx"), - (SQLITE3_NAMED, DICTS, "sqlite3.NamedRow dict"), - (SQLITE3_NAMED, ATTRS, "sqlite3.NamedRow attr"), - (NAMED_TUPLE, INDEX, "namedtuple indx"), - (NAMED_TUPLE, ATTRS, "namedtuple attr"), - (NAMED_ROW, DICTS, "NamedRow dict"), - (NAMED_ROW, ATTRS, "NamedRow attr"), +FIELD_MAP = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" +FIELD_NAME_LENGTH = 12 +TESTS = ( + ("Native", "index"), + ("Row", "index"), + ("Row", "dict"), + ("NamedRow", "index"), + ("NamedRow", "dict"), + ("NamedRow", "attr"), ) -base = 0 -for setup, test, title in tests: - try: - v = timeit.timeit(test, setup=setup, number=1000) - except AttributeError: - print("AttributeError", title) - continue +connection = sqlite3.Connection("timing.db") +connection.executescript( + """CREATE TABLE IF NOT EXISTS t ( + title char(15), + fields int, + timeit real +); +CREATE UNIQUE INDEX IF NOT EXISTS onlyone ON t (title, fields);""" +) + + +def make_field(idx: int) -> str: + return FIELD_MAP[idx] * random.randrange(4, 15) + + +for number_of_fields in range(2, 41): + # Each round will have identical fields to the proceeding. Show off the + # speed of NamedRow over Row + random.seed(101038) + fields = [make_field(i) for i in range(number_of_fields)] + select = ", ".join([f"{i} AS {fields[i]}" for i in range(number_of_fields)]) + print(f"Fields {number_of_fields}") + + # Process all tests for the same number of fields. Does this help with + # fluctuations in background processes? + for factory, test_method in TESTS: + factory_row = "" + if factory != "Native": + factory_row = f"conn.row_factory=sqlite3.{factory}" + + setup = "\n".join( + [ + "import sqlite3", + "conn = sqlite3.connect(':memory:')", + factory_row, + "cursor = conn.cursor()", + ] + ) + string, formatter = { + "index": ("row[{0}]", int), + "dict": ("row['{0}']", lambda i: fields[i]), + "attr": ("row.{0}", lambda i: fields[i]), + }[test_method] + + timing = [f'row = cursor.execute("SELECT {select}").fetchone()'] + timing.extend([string.format(formatter(i)) for i in range(number_of_fields)]) + timing = "\n".join(timing) + + title = f"{factory} {test_method}" + try: + v = timeit.timeit(timing, setup=setup, number=10000) + except AttributeError: + print("AttributeError", title) + continue - if setup == SETUP: - base = v - percent = int(100 * ((v / base)) - 100) - print(f"+{percent}%", round(1000 * v), "usec", title) + connection.execute( + "INSERT INTO t (title, fields, timeit) " + "VALUES(?,?,?) ON CONFLICT (title, fields) " + "DO UPDATE SET timeit=excluded.timeit " + "WHERE excluded.timeit < t.timeit;", + (title, number_of_fields, v), + ) + connection.commit() +connection.close() diff --git a/Modules/_sqlite/NamedRow/timing.sh b/Modules/_sqlite/NamedRow/timing.sh deleted file mode 100644 index 3c27f331cb4add..00000000000000 --- a/Modules/_sqlite/NamedRow/timing.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -# https://bugs.python.org/issue13299 -# https://bugs.python.org/file37673/sqlite_namedtuplerow.patch -# https://github.com/python/cpython/tree/master/Modules/_sqlite - - -IMPORTS="import sqlite3, collections, named_row; con=sqlite3.connect(':memory:')" -CREATE="con.execute('CREATE TABLE t(a,b)')" -POPULATE="for i in range(1000): con.execute('INSERT INTO t VALUES (?,?)', (i, 10000-i))" -TEST="con.execute('SELECT * FROM t').fetchall()" - -echo "tuple" -python -m timeit -s "$IMPORTS" -s "$CREATE" -s "$POPULATE" -- "for x in $TEST: x[0]" - -factory="con.row_factory=sqlite3.Row" -echo -e "\nsqlite3.row # $factory" -python -m timeit -s "$IMPORTS" -s "$factory" -s"$CREATE" -s "$POPULATE" -- "for x in $TEST: x['a']" - -factory="con.row_factory=collections.namedtuple('Row', ['a','b'])" -echo -e "\nnamedtuple # $factory" -python -m timeit -s "$IMPORTS" -s "$factory" -s"$CREATE" -s "$POPULATE" -- "for x in $TEST: x.a" - -factory="con.row_factory=named_row.NamedRow" -echo -e "\nNamedRow # $factory [0]" -python -m timeit -s "$IMPORTS" -s "$factory" -s"$CREATE" -s "$POPULATE" -- "for x in $TEST: x[0]" - -factory="con.row_factory=named_row.NamedRow" -echo -e "\nNamedRow # $factory" -python -m timeit -s "$IMPORTS" -s "$factory" -s"$CREATE" -s "$POPULATE" -- "for x in $TEST: x.a" - -# tuple 804 540 -# sqlite3.row 983 673 -# namedtuple 1.38 923 -# NamedRow[0] 1.34 1.03 -# NamedRow.a 2.33 1.5 From b4ea904cf49c1e6e066dd301df68c1a5f588d052 Mon Sep 17 00:00:00 2001 From: Clinton James Date: Fri, 11 Oct 2019 18:01:29 -0500 Subject: [PATCH 5/9] Add row creating timing Does the cursor.execute().fetchone() effect the general shape of the data? Add testing which only covers object creation and access. --- Modules/_sqlite/NamedRow/create_csv.py | 30 +++++ Modules/_sqlite/NamedRow/timing.py | 172 ++++++++++++++++--------- 2 files changed, 141 insertions(+), 61 deletions(-) create mode 100644 Modules/_sqlite/NamedRow/create_csv.py diff --git a/Modules/_sqlite/NamedRow/create_csv.py b/Modules/_sqlite/NamedRow/create_csv.py new file mode 100644 index 00000000000000..f50c2847232bf2 --- /dev/null +++ b/Modules/_sqlite/NamedRow/create_csv.py @@ -0,0 +1,30 @@ +import csv +import sqlite3 + +METHODS = [ + "Native index", + "Row index", + "Row dict", + "NamedRow index", + "NamedRow dict", + "NamedRow attr", +] + +if __name__ == "__main__": + db_filename = "instance.db" + connection = sqlite3.Connection(db_filename) + cursor = connection.cursor() + LENGTHS = [ + x[0] + for x in cursor.execute( + "SELECT distinct fields FROM t ORDER BY fields" + ).fetchall() + ] + with open(db_filename.split(".")[0] + ".csv", "wt") as csv_file: + writer = csv.writer(csv_file) + writer.writerow(["Method"] + LENGTHS) + for method in METHODS: + values = cursor.execute( + "SELECT timeit FROM t WHERE title=? ORDER BY fields", (method,) + ).fetchall() + writer.writerow([method] + [x[0] for x in values]) diff --git a/Modules/_sqlite/NamedRow/timing.py b/Modules/_sqlite/NamedRow/timing.py index b12e60552af6b8..88a4532ae86b32 100644 --- a/Modules/_sqlite/NamedRow/timing.py +++ b/Modules/_sqlite/NamedRow/timing.py @@ -6,7 +6,6 @@ FIELD_MAP = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" -FIELD_NAME_LENGTH = 12 TESTS = ( ("Native", "index"), ("Row", "index"), @@ -17,67 +16,118 @@ ) -connection = sqlite3.Connection("timing.db") -connection.executescript( - """CREATE TABLE IF NOT EXISTS t ( - title char(15), - fields int, - timeit real -); -CREATE UNIQUE INDEX IF NOT EXISTS onlyone ON t (title, fields);""" -) - - def make_field(idx: int) -> str: return FIELD_MAP[idx] * random.randrange(4, 15) -for number_of_fields in range(2, 41): - # Each round will have identical fields to the proceeding. Show off the - # speed of NamedRow over Row - random.seed(101038) - fields = [make_field(i) for i in range(number_of_fields)] - select = ", ".join([f"{i} AS {fields[i]}" for i in range(number_of_fields)]) - print(f"Fields {number_of_fields}") - - # Process all tests for the same number of fields. Does this help with - # fluctuations in background processes? - for factory, test_method in TESTS: - factory_row = "" - if factory != "Native": - factory_row = f"conn.row_factory=sqlite3.{factory}" - - setup = "\n".join( - [ - "import sqlite3", - "conn = sqlite3.connect(':memory:')", - factory_row, - "cursor = conn.cursor()", - ] - ) - string, formatter = { - "index": ("row[{0}]", int), - "dict": ("row['{0}']", lambda i: fields[i]), - "attr": ("row.{0}", lambda i: fields[i]), - }[test_method] - - timing = [f'row = cursor.execute("SELECT {select}").fetchone()'] - timing.extend([string.format(formatter(i)) for i in range(number_of_fields)]) - timing = "\n".join(timing) - - title = f"{factory} {test_method}" - try: - v = timeit.timeit(timing, setup=setup, number=10000) - except AttributeError: - print("AttributeError", title) - continue - - connection.execute( - "INSERT INTO t (title, fields, timeit) " - "VALUES(?,?,?) ON CONFLICT (title, fields) " - "DO UPDATE SET timeit=excluded.timeit " - "WHERE excluded.timeit < t.timeit;", - (title, number_of_fields, v), - ) - connection.commit() -connection.close() +def setup_test_instantiation(factory, method, fields): + """ + factory: 'Native', 'Row', 'NamedRow' + method: 'index', 'dict', or 'attr' + fields: all possible fields + """ + + select = ", ".join([f"{i} AS {field}" for i, field in enumerate(fields)]) + factory_row = "" if factory == "Native" else f"cursor.row_factory=sqlite3.{factory}" + setup = [ + "import sqlite3", + "cursor = sqlite3.Connection(':memory:').cursor()", + f"data = tuple(1 for x in range({len(fields)}))", + factory_row, + f'cursor.execute("SELECT {select}")', + ] + + string, formatter = { + "index": ("row[{0}]", int), + "dict": ("row['{0}']", lambda i: fields[i]), + "attr": ("row.{0}", lambda i: fields[i]), + }[method] + + test = [] + if factory == "Native": + test.append("row = data") + else: + test.append("row = cursor.row_factory(cursor, data)") + test.extend([string.format(formatter(i)) for i in range(len(fields))]) + + return "\n".join(setup), "\n".join(test) + + +def setup_test_fetchone(factory, method, fields): + """ + factory: 'Native', 'Row', 'NamedRow' + method: 'index', 'dict', or 'attr' + fields: all possible fields + """ + select = ", ".join([f"{i} AS {field}" for i, field in enumerate(fields)]) + + factory_row = "" + if factory != "Native": + factory_row = f"conn.row_factory=sqlite3.{factory}" + + timing_setup = [ + "import sqlite3", + "conn = sqlite3.connect(':memory:')", + factory_row, + "cursor = conn.cursor()", + ] + + string, formatter = { + "index": ("row[{0}]", int), + "dict": ("row['{0}']", lambda i: fields[i]), + "attr": ("row.{0}", lambda i: fields[i]), + }[method] + + test = [f'row = cursor.execute("SELECT {select}").fetchone()'] + test.extend([string.format(formatter(i)) for i in range(len(fields))]) + + return "\n".join(timing_setup), "\n".join(test) + + +def run_test(db_name, function): + connection = sqlite3.Connection(db_name) + connection.executescript( + """CREATE TABLE IF NOT EXISTS t ( + title char(15), + fields int, + timeit real + ); + CREATE UNIQUE INDEX IF NOT EXISTS onlyone ON t (title, fields);""" + ) + for number_of_fields in range(2, 41): + + # Each round will have identical fields to the proceeding. Show off the + # speed of NamedRow over Row + random.seed(101038) + fields = [make_field(i) for i in range(number_of_fields)] + + print(f"Fields {number_of_fields}") + + # Process all tests for the same number of fields. Does this help with + + for factory, test_method in TESTS: + title = f"{factory} {test_method}" + setup, test = function(factory, test_method, fields) + + # print(setup) + # print(test) + try: + v = timeit.timeit(test, setup=setup, number=100000) + except AttributeError: + print("AttributeError", title) + continue + + connection.execute( + "INSERT INTO t (title, fields, timeit) " + "VALUES(?,?,?) ON CONFLICT (title, fields) " + "DO UPDATE SET timeit=excluded.timeit " + "WHERE excluded.timeit < t.timeit;", + (title, number_of_fields, v), + ) + connection.commit() + connection.close() + + +if __name__ == "__main__": + # run_test("fetchone.db", setup_test_fetchone) + run_test("instance.db", setup_test_instantiation) From 7d28522d2a6fabb3977f4d1bdc3a4be9859b2116 Mon Sep 17 00:00:00 2001 From: Clinton James Date: Sat, 28 Dec 2019 21:54:32 -0600 Subject: [PATCH 6/9] Fix column access + Faster + Remove case insensitive --- Modules/_sqlite/named_row.c | 102 +++++++++++------------------------- 1 file changed, 31 insertions(+), 71 deletions(-) diff --git a/Modules/_sqlite/named_row.c b/Modules/_sqlite/named_row.c index f9cdeb059b0843..a3bc96812d7558 100644 --- a/Modules/_sqlite/named_row.c +++ b/Modules/_sqlite/named_row.c @@ -5,35 +5,20 @@ * This software is provided 'as-is', without any express or implied * warranty. In no event will the authors be held liable for any damages * arising from the use of this software. - * - * Permission is granted to anyone to use this software for any purpose, - * including commercial applications, and to alter it and redistribute it - * freely, subject to the following restrictions: - * - * 1. The origin of this software must not be misrepresented; you must not - * claim that you wrote the original software. If you use this software - * in a product, an acknowledgment in the product documentation would be - * appreciated but is not required. - * 2. Altered source versions must be plainly marked as such, and must not be - * misrepresented as being the original software. - * 3. This notice may not be removed or altered from any source distribution. */ #include "named_row.h" #include "cursor.h" PyDoc_STRVAR(named_row__doc__, -"NamedRow an optimized row_factory with column name access to row fields\n\n\ -NamedRow extends Row to support mapping access by attribute, column name, and \ -index. Column names access is case insensitive.\n\ -row.field == row.FIELD\n\ -For attribute names that would be illegal due to dashes or spaces, an \ -underscore is an acceptable replacement.\n\ -row['dash-name'] == row.dash_name \n\ -For functions in SQL remember to use the 'AS' statement to name the output \ -field.\n\ -connection.execute('SELECT count(*) AS count').fetchone()\n\n\ -Iteration gives field/value pairs similar to dict.items()."); + "NamedRow is an optimized row_factory for column name access.\n" + "\n" + "NamedRow supports mapping by attribute, column name, and index.\n" + "Iteration gives field/value pairs similar to dict.items().\n" + "For attribute names that would be illegal, either alter the SQL\n" + "using the 'AS' statement (SELECT count(*) AS count FROM ...) \n" + "or index by name as in `row['count(*)']`.\n" + ); static void named_row_dealloc(pysqlite_NamedRow* self) @@ -51,10 +36,12 @@ named_row_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) assert(type != NULL && type->tp_alloc != NULL); - if (!_PyArg_NoKeywords("NamedRow", kwargs)) + if (!_PyArg_NoKeywords("NamedRow", kwargs)) { return NULL; - if (!PyArg_ParseTuple(args, "OO", &cursor, &data)) + } + if (!PyArg_ParseTuple(args, "OO", &cursor, &data)) { return NULL; + } if (!PyObject_TypeCheck((PyObject*)cursor, &pysqlite_CursorType)) { PyErr_SetString(PyExc_TypeError, @@ -68,8 +55,9 @@ named_row_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } pysqlite_NamedRow *self = (pysqlite_NamedRow *) type->tp_alloc(type, 0); - if (self == NULL) + if (self == NULL) { return NULL; + } Py_INCREF(data); self->data = data; @@ -91,40 +79,25 @@ named_row_item(pysqlite_NamedRow* self, Py_ssize_t idx) // Find index from field name static Py_ssize_t -named_row_find_index(pysqlite_NamedRow* self, const char* name) +named_row_find_index(pysqlite_NamedRow* self, PyObject* name) { + Py_ssize_t name_len = PyUnicode_GET_LENGTH(name); Py_ssize_t nitems = PyTuple_Size(self->fields); for (Py_ssize_t i = 0; i < nitems; i++) { PyObject* obj = PyTuple_GET_ITEM(PyTuple_GET_ITEM(self->fields, i), 0); - const char *cursor_field = PyUnicode_AsUTF8(obj); - if (!cursor_field) { + if (name_len != PyUnicode_GET_LENGTH(obj)) { continue; } - const char* p1 = name; - const char* p2 = cursor_field; - - while (1) { - // Done checking if at the end of either string - if ((*p1 == (char)0) || (*p2 == (char)0)) { - break; - } - - // case insensitive comparison - if ((*p1 | 0x20) != (*p2 | 0x20) && - // accept '_' instead of space or dash - *p1 != '_' && *p2 !=' ' && *p2 !='-') - { - break; + Py_ssize_t len = name_len; + const Py_UCS1 *p1 = PyUnicode_1BYTE_DATA(name); + const Py_UCS1 *p2 = PyUnicode_1BYTE_DATA(obj); + for (; len; len--, p1++, p2++) { + if (*p1 == *p2) { + return i; } - p1++; - p2++; - } - // Found it if at the end of both strings - if ((*p1 == (char)0) && (*p2 == (char)0)) { - return i; } } return -1; @@ -135,17 +108,16 @@ named_row_getattro(pysqlite_NamedRow* self, PyObject* attr_obj) { // Calling function checked `name` is PyUnicode, no need to check here. Py_INCREF(attr_obj); - const char* attr_name = PyUnicode_AsUTF8(attr_obj); - Py_ssize_t idx = named_row_find_index(self, attr_name); + Py_ssize_t idx = named_row_find_index(self, attr_obj); Py_DECREF(attr_obj); - if (idx >= 0) { - PyObject* value = PyTuple_GET_ITEM(self->data, idx); - Py_INCREF(value); - return value; + if (idx < 0) { + return PyObject_GenericGetAttr((PyObject *)self, attr_obj); } - return PyObject_GenericGetAttr((PyObject *)self, attr_obj); + PyObject* value = PyTuple_GET_ITEM(self->data, idx); + Py_INCREF(value); + return value; } static int @@ -178,14 +150,7 @@ named_row_index(pysqlite_NamedRow* self, PyObject* index) } else if (PyUnicode_Check(index)) { // Process string index, dict like - Py_ssize_t idx = 0; - const char* key = PyUnicode_AsUTF8(index); - - if (key == NULL) { - return NULL; - } - - idx = named_row_find_index(self, key); + Py_ssize_t idx = named_row_find_index(self, index); if (idx < 0) { PyErr_SetString(PyExc_IndexError, "No item with that key"); @@ -215,12 +180,7 @@ static int named_row_contains(pysqlite_NamedRow* self, PyObject* name) { if (PyUnicode_Check(name)) { - const char* key = PyUnicode_AsUTF8(name); - if (key == NULL) - return -1; - - Py_ssize_t idx = named_row_find_index(self, key); - return idx >= 0 ? 1 : 0; + return named_row_find_index(self, name) < 0 ? 0 : 1; } return -1; } From bd34ea6aa29f5d61f7e08deff99c2cc26fc472f8 Mon Sep 17 00:00:00 2001 From: Clinton James Date: Mon, 30 Dec 2019 23:34:10 -0600 Subject: [PATCH 7/9] Add documentation --- Doc/library/sqlite3.rst | 83 +++++++++- Misc/ACKS | 1 + Modules/_sqlite/NamedRow/create_csv.py | 30 ---- Modules/_sqlite/NamedRow/named_row.py | 49 ------ Modules/_sqlite/NamedRow/test.py | 100 ------------ Modules/_sqlite/NamedRow/timing.py | 133 ---------------- .../_sqlite/NamedRow/unit_test_named_row.py | 148 ------------------ 7 files changed, 79 insertions(+), 465 deletions(-) delete mode 100644 Modules/_sqlite/NamedRow/create_csv.py delete mode 100644 Modules/_sqlite/NamedRow/named_row.py delete mode 100644 Modules/_sqlite/NamedRow/test.py delete mode 100644 Modules/_sqlite/NamedRow/timing.py delete mode 100644 Modules/_sqlite/NamedRow/unit_test_named_row.py diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 67ea2b1d776638..57f4dc646ac5d3 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -493,14 +493,21 @@ Connection Objects .. literalinclude:: ../includes/sqlite3/row_factory.py If returning a tuple doesn't suffice and you want name-based access to - columns, you should consider setting :attr:`row_factory` to the - highly-optimized :class:`sqlite3.Row` type. :class:`Row` provides both - index-based and case-insensitive name-based access to columns with almost no - memory overhead. It will probably be better than your own custom - dictionary-based approach or even a db_row based solution. + columns, you should consider setting :attr:`row_factory` one of the + highly optimized row_factory types :class:`sqlite3.Row` or + :class:`sqlite3.NamedRow`. + + :class:`Row` provides both index-based and case-insensitive name-based + access to columns with almost no memory overhead. It will probably be + better than your own custom dictionary-based approach or even a db_row + based solution. .. XXX what's a db_row-based solution? + :class:`NamedRow` provides both attribute and index by column name or number + to columns with the same low memory requirements of :class:`sqlite3.Row`. + As a compiled class, it will out perform a custom python row_factory solution. + .. attribute:: text_factory @@ -807,6 +814,72 @@ Now we plug :class:`Row` in:: 100.0 35.14 +.. class:: NamedRow + + A :class:`NamedRow` instance is an optimized :attr:`~Connection.row_factory` + for :class:`Connection` objects. + It tries to mimic a namedtuple in most of its features. + + It supports mapping access by attribute, column name, and index. + It also supports iteration, slicing, equality testing, :func:`contains`, + and :func:`len`. + +Here is a simple example, with a Japanese column name:: + + >>> db_connection = sqlite3.connect(":memory:") + >>> cursor = db_connection.cursor() + >>> cursor.row_factory = sqlite3.NamedRow + >>> row = cursor.execute("SELECT 'seven' AS english, '七' as 日本").fetchone() + >>> type(row) + + >>> len(row) + 2 + >>> '日本' in row + True + >>> row.english, row.日本 + ('seven', '七') + >>> row[0], row['english'] + ('seven', 'seven') + >>> tuple(row) + (('english', 'seven'), ('日本', '七')) + >>> dict(row) + {'english': 'seven', '日本': '七'} + >>> [x[0] for x in row] # Get column names + ['english', '日本'] + >>> [x[1] for x in row] # Get column values + ['seven', '七'] + +For columns names that are invalid attribute names, such as "count(*)" +or python reserved words, create a valid attribute name using the SQL +"AS" keyword. +For example, assume we have an address table and want the number of rows +where the city is Kahlotus.:: + + >>> cursor = db_connection.cursor() + >>> cursor.row_factory = sqlite3.NamedRow + >>> row = cursor.execute("SELECT city, count(*) FROM address WHERE city=?", + ... ('Kahlotus')).fetchone() + >>> row.count + Traceback (most recent call last): + File "", line 1, in + AttributeError: 'sqlite3.NamedRow' object has no attribute 'count' + >>> tuple(row) + (('city', 'Kahlotus'), ('count(*)', 2)) + >>> row = cursor.execute("SELECT city, count(*) AS cnt FROM address WHERE city=?", + ... ('Kahlotus')).fetchone() + (('city', 'Kahlotus'), ('count', 2)) + >>> row.count + 2 + +If you are unable to change the query, index by name or number.:: + + >>> tuple(row) + (('city', 'Kahlotus'), ('count(*)', 2)) + >>> row['count(*)'] + 2 + >>> row[1] + 2 + .. _sqlite3-exceptions: diff --git a/Misc/ACKS b/Misc/ACKS index 8ab7a6cee5a3fc..ae68e3e8480b91 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -775,6 +775,7 @@ Manuel Jacob David Jacobs Kevin Jacobs Kjetil Jacobsen +Clinton James Bertrand Janin Geert Jansen Jack Jansen diff --git a/Modules/_sqlite/NamedRow/create_csv.py b/Modules/_sqlite/NamedRow/create_csv.py deleted file mode 100644 index f50c2847232bf2..00000000000000 --- a/Modules/_sqlite/NamedRow/create_csv.py +++ /dev/null @@ -1,30 +0,0 @@ -import csv -import sqlite3 - -METHODS = [ - "Native index", - "Row index", - "Row dict", - "NamedRow index", - "NamedRow dict", - "NamedRow attr", -] - -if __name__ == "__main__": - db_filename = "instance.db" - connection = sqlite3.Connection(db_filename) - cursor = connection.cursor() - LENGTHS = [ - x[0] - for x in cursor.execute( - "SELECT distinct fields FROM t ORDER BY fields" - ).fetchall() - ] - with open(db_filename.split(".")[0] + ".csv", "wt") as csv_file: - writer = csv.writer(csv_file) - writer.writerow(["Method"] + LENGTHS) - for method in METHODS: - values = cursor.execute( - "SELECT timeit FROM t WHERE title=? ORDER BY fields", (method,) - ).fetchall() - writer.writerow([method] + [x[0] for x in values]) diff --git a/Modules/_sqlite/NamedRow/named_row.py b/Modules/_sqlite/NamedRow/named_row.py deleted file mode 100644 index 56447dc1a824b5..00000000000000 --- a/Modules/_sqlite/NamedRow/named_row.py +++ /dev/null @@ -1,49 +0,0 @@ -import sqlite3 - - -class NamedRow: - """Sqlite3.NamedRow as a namedtuple like object. - """ - - def __init__(self, cursor, values): - """Create a named row. - - The title_tuple comes from tuple(x[0] for x in defination. However, - there is no need to keep creating it anew. - """ - self._datum = dict(zip(("a", "b"), values)) - - # self._titles = ("a", "b") - # 1 msec for creating the tuple - # self._titles = tuple(x[0] for x in cursor.description) - # self._values = values - - def __len__(self): - return len(self._values) - - def __iter__(self): - """Dict.items() equivalent.""" - # print("__iter__") - return iter(zip(self._titles, self._values)) - - def __getitem__(self, key): - """Sequence or Mapping lookup.""" - # print(f"__getitem__[{repr(key)}]") - # return self._values[self._titles.index(key)] - return self._datum[key] - - def __getattr__(self, name): - try: - # print(f"__getattr__[{repr(name)}]") - # return self._values[self._titles.index(name)] - return self._datum[name] - except ValueError: - # print(f"__getattr__[{repr(name)}] raise AttributeError") - raise AttributeError(name) - - def __contains__(self, key): - """True if Row has the specified key, else False.""" - return key in self._titles - - def __repr__(self): - return "".format(repr(self._titles), id(self)) diff --git a/Modules/_sqlite/NamedRow/test.py b/Modules/_sqlite/NamedRow/test.py deleted file mode 100644 index 1dd9ac26e69b47..00000000000000 --- a/Modules/_sqlite/NamedRow/test.py +++ /dev/null @@ -1,100 +0,0 @@ -import sqlite3 -import pytest -from named_row import NamedRow - - -def create_database(): - """Database ('a', '-dash') - Filled with a 1000 of (1, 2) - """ - conn = sqlite3.connect(":memory:") - conn.execute('CREATE TABLE t(a, "-dash")') - for i in range(1000): - conn.execute("INSERT INTO t VALUES(?,?)", (1, 2)) - conn.row_factory = NamedRow - return conn - - -@pytest.fixture -def db(scope="session"): - yield create_database() - - -@pytest.fixture -def row(db): - """An instance of NamedRow.""" - cursor = db.execute("SELECT * FROM t LIMIT 1") - row = cursor.fetchone() - yield row - - -def test_01_database(db): - assert "NamedRow" in repr(db.row_factory) - cursor = db.execute("SELECT * FROM t LIMIT 1") - assert ("a", "-dash") == tuple(x[0] for x in cursor.description) - - -def test_02_is_NamedRow(row): - assert "NamedRow" in row.__class__.__name__ - assert "a" in row - repr_str = repr(row) - assert "NamedRow" in repr_str - assert "-dash" in repr_str - - -def test_11_iterable_keyValue_pairs(row): - assert [("a", 1), ("-dash", 2)] == [x for x in row] - - -def test_12_iterable_into_dict(row): - assert dict((("a", 1), ("-dash", 2))) == dict(row) - - -def test_30_index_by_number(row): - assert row[0] == 1 - assert row[1] == 2 - - -def test_31_index_by_string(row): - assert row["a"] == 1 - assert row["-dash"] == 2 - - -def test_32_access_by_attribute(row): - assert row.a == 1 - - -def test_50_IndexError(row): - try: - row[10] - raise RuntimeError("Expected IndexError") - except IndexError: - pass - - -def test_51_ValueError(row): - try: - row["alphabet"] - raise RuntimeError("Expected ValueError") - except ValueError: - pass - - -def test_52_AttributeError(row): - try: - row.alphabet - raise RuntimeError("Expected AttributeError") - except AttributeError: - pass - - -def test_53_not_assignable(row): - try: - row["a"] = 100 - raise RuntimeError("Expecting TypeError object does no support item assignment") - except TypeError: - pass - - -if __name__ == "__main__": - pass diff --git a/Modules/_sqlite/NamedRow/timing.py b/Modules/_sqlite/NamedRow/timing.py deleted file mode 100644 index 88a4532ae86b32..00000000000000 --- a/Modules/_sqlite/NamedRow/timing.py +++ /dev/null @@ -1,133 +0,0 @@ -import collections -import timeit -import random -import sqlite3 -from named_row import NamedRow - - -FIELD_MAP = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" -TESTS = ( - ("Native", "index"), - ("Row", "index"), - ("Row", "dict"), - ("NamedRow", "index"), - ("NamedRow", "dict"), - ("NamedRow", "attr"), -) - - -def make_field(idx: int) -> str: - return FIELD_MAP[idx] * random.randrange(4, 15) - - -def setup_test_instantiation(factory, method, fields): - """ - factory: 'Native', 'Row', 'NamedRow' - method: 'index', 'dict', or 'attr' - fields: all possible fields - """ - - select = ", ".join([f"{i} AS {field}" for i, field in enumerate(fields)]) - factory_row = "" if factory == "Native" else f"cursor.row_factory=sqlite3.{factory}" - setup = [ - "import sqlite3", - "cursor = sqlite3.Connection(':memory:').cursor()", - f"data = tuple(1 for x in range({len(fields)}))", - factory_row, - f'cursor.execute("SELECT {select}")', - ] - - string, formatter = { - "index": ("row[{0}]", int), - "dict": ("row['{0}']", lambda i: fields[i]), - "attr": ("row.{0}", lambda i: fields[i]), - }[method] - - test = [] - if factory == "Native": - test.append("row = data") - else: - test.append("row = cursor.row_factory(cursor, data)") - test.extend([string.format(formatter(i)) for i in range(len(fields))]) - - return "\n".join(setup), "\n".join(test) - - -def setup_test_fetchone(factory, method, fields): - """ - factory: 'Native', 'Row', 'NamedRow' - method: 'index', 'dict', or 'attr' - fields: all possible fields - """ - select = ", ".join([f"{i} AS {field}" for i, field in enumerate(fields)]) - - factory_row = "" - if factory != "Native": - factory_row = f"conn.row_factory=sqlite3.{factory}" - - timing_setup = [ - "import sqlite3", - "conn = sqlite3.connect(':memory:')", - factory_row, - "cursor = conn.cursor()", - ] - - string, formatter = { - "index": ("row[{0}]", int), - "dict": ("row['{0}']", lambda i: fields[i]), - "attr": ("row.{0}", lambda i: fields[i]), - }[method] - - test = [f'row = cursor.execute("SELECT {select}").fetchone()'] - test.extend([string.format(formatter(i)) for i in range(len(fields))]) - - return "\n".join(timing_setup), "\n".join(test) - - -def run_test(db_name, function): - connection = sqlite3.Connection(db_name) - connection.executescript( - """CREATE TABLE IF NOT EXISTS t ( - title char(15), - fields int, - timeit real - ); - CREATE UNIQUE INDEX IF NOT EXISTS onlyone ON t (title, fields);""" - ) - for number_of_fields in range(2, 41): - - # Each round will have identical fields to the proceeding. Show off the - # speed of NamedRow over Row - random.seed(101038) - fields = [make_field(i) for i in range(number_of_fields)] - - print(f"Fields {number_of_fields}") - - # Process all tests for the same number of fields. Does this help with - - for factory, test_method in TESTS: - title = f"{factory} {test_method}" - setup, test = function(factory, test_method, fields) - - # print(setup) - # print(test) - try: - v = timeit.timeit(test, setup=setup, number=100000) - except AttributeError: - print("AttributeError", title) - continue - - connection.execute( - "INSERT INTO t (title, fields, timeit) " - "VALUES(?,?,?) ON CONFLICT (title, fields) " - "DO UPDATE SET timeit=excluded.timeit " - "WHERE excluded.timeit < t.timeit;", - (title, number_of_fields, v), - ) - connection.commit() - connection.close() - - -if __name__ == "__main__": - # run_test("fetchone.db", setup_test_fetchone) - run_test("instance.db", setup_test_instantiation) diff --git a/Modules/_sqlite/NamedRow/unit_test_named_row.py b/Modules/_sqlite/NamedRow/unit_test_named_row.py deleted file mode 100644 index b2622f2f7dd938..00000000000000 --- a/Modules/_sqlite/NamedRow/unit_test_named_row.py +++ /dev/null @@ -1,148 +0,0 @@ -import sqlite3 -import unittest - - -class NamedRowFactoryTests(unittest.TestCase): - def setUp(self): - self.conn = sqlite3.connect(":memory:") - self.conn.row_factory = sqlite3.NamedRow - - def test_instance(self): - """row_factory creates NamedRow instances""" - row = self.conn.execute("SELECT 1, 2").fetchone() - self.assertIsInstance(row, sqlite3.NamedRow) - self.assertIn("NamedRow", row.__class__.__name__) - self.assertIn("NamedRow", repr(row)) - self.assertIn("optimized row_factory with namedtuple", sqlite3.NamedRow.__doc__) - - def test_by_number_index(self): - """Get values by numbered index [0]""" - row = self.conn.execute("SELECT 1, 2").fetchone() - # len(row) - self.assertEqual(2, len(row)) - - self.assertEqual(row[0], 1) - self.assertEqual(row[1], 2) - self.assertEqual(row[-1], 2) - self.assertEqual(row[-2], 1) - - # Check past boundary - with self.assertRaises(IndexError): - row[len(row) + 1] - with self.assertRaises(IndexError): - row[-1 * (len(row) + 1)] - - def test_by_string_index(self): - """Get values by string index ['field']""" - row = self.conn.execute( - 'SELECT 1 AS a, 2 AS "dash-name", ' '3 AS "space name", 4 AS "四"' - ).fetchone() - self.assertEqual(1, row["a"]) - self.assertEqual(2, row["dash-name"]) - self.assertEqual(3, row["space name"]) - self.assertEqual(4, row["四"]) - - # Case insensitive - self.assertEqual(1, row["A"]) - self.assertEqual(2, row["DaSh-NaMe"]) - self.assertEqual(3, row["SPACE name"]) - - # Underscore as acceptable replacement for dash and space - self.assertEqual(2, row["dash_NAME"]) - self.assertEqual(3, row["space_name"]) - - def test_by_attribute(self): - """Get values by attribute row.field""" - row = self.conn.execute( - 'SELECT 1 AS a, 2 AS "dash-name", ' '3 AS "space name", 4 AS "四"' - ).fetchone() - self.assertEqual(1, row.a) - self.assertEqual(2, row.dash_name) - self.assertEqual(3, row.space_name) - self.assertEqual(4, row.四) - self.assertEqual(2, row.DASH_NAME) - - def test_contains_field(self): - row = self.conn.execute("SELECT 1 AS a, 2 AS b").fetchone() - self.assertIn("a", row) - self.assertIn("A", row) - - def test_slice(self): - """Does NamedRow slice like a list.""" - row = self.conn.execute("SELECT 1, 2, 3, 4").fetchone() - self.assertEqual(row[0:0], ()) - self.assertEqual(row[0:1], (1,)) - self.assertEqual(row[1:3], (2, 3)) - self.assertEqual(row[3:1], ()) - # Explicit bounds are optional. - self.assertEqual(row[1:], (2, 3, 4)) - self.assertEqual(row[:3], (1, 2, 3)) - # Slices can use negative indices. - self.assertEqual(row[-2:-1], (3,)) - self.assertEqual(row[-2:], (3, 4)) - # Slicing supports steps. - self.assertEqual(row[0:4:2], (1, 3)) - self.assertEqual(row[3:0:-2], (4, 2)) - - def test_hash_comparison(self): - row1 = self.conn.execute("SELECT 1 AS a, 2 AS b").fetchone() - row2 = self.conn.execute("SELECT 1 AS a, 2 AS b").fetchone() - row3 = self.conn.execute("SELECT 1 AS a, 3 AS b").fetchone() - - self.assertEqual(row1, row1) - self.assertEqual(row1, row2) - self.assertTrue(row2 != row3) - - self.assertFalse(row1 != row1) - self.assertFalse(row1 != row2) - self.assertFalse(row2 == row3) - - self.assertEqual(row1, row2) - self.assertEqual(hash(row1), hash(row2)) - self.assertNotEqual(row1, row3) - self.assertNotEqual(hash(row1), hash(row3)) - - def test_fake_cursor_class(self): - # Issue #24257: Incorrect use of PyObject_IsInstance() caused - # segmentation fault. - # Issue #27861: Also applies for cursor factory. - class FakeCursor(str): - __class__ = sqlite3.Cursor - - self.assertRaises(TypeError, self.conn.cursor, FakeCursor) - self.assertRaises(TypeError, sqlite3.NamedRow, FakeCursor(), ()) - - def test_iterable(self): - """Is NamedRow iterable.""" - row = self.conn.execute("SELECT 1 AS a, 2 AS b").fetchone() - for col in row: - pass - - def test_iterable_pairs(self): - expected = (("a", 1), ("b", 2)) - row = self.conn.execute("SELECT 1 AS a, 2 AS b").fetchone() - self.assertSequenceEqual(expected, [x for x in row]) - self.assertSequenceEqual(expected, tuple(row)) - self.assertSequenceEqual(expected, list(row)) - self.assertSequenceEqual(dict(expected), dict(row)) - - def test_expected_exceptions(self): - row = self.conn.execute('SELECT 1 AS a, 2 AS "dash-name"').fetchone() - with self.assertRaises(IndexError): - row["no_such_field"] - with self.assertRaises(AttributeError): - row.no_such_field - with self.assertRaises(TypeError): - row[0] = 100 - with self.assertRaises(TypeError): - row["a"] = 100 - with self.assertRaises(TypeError) as err: - row.a = 100 - self.assertIn("does not support item assignment", str(err.exception)) - - with self.assertRaises(TypeError): - setattr(row, "a", 100) - - -if __name__ == "__main__": - unittest.main() From f05be4f373c2399bdf916eea8a9c33206c69d13a Mon Sep 17 00:00:00 2001 From: Clinton James Date: Tue, 31 Dec 2019 00:32:21 -0600 Subject: [PATCH 8/9] Add tests --- Lib/sqlite3/test/test_namedrow.py | 178 ++++++++++++++++++++++++++++++ Lib/test/test_sqlite.py | 46 +++++--- 2 files changed, 211 insertions(+), 13 deletions(-) create mode 100644 Lib/sqlite3/test/test_namedrow.py diff --git a/Lib/sqlite3/test/test_namedrow.py b/Lib/sqlite3/test/test_namedrow.py new file mode 100644 index 00000000000000..e22b9c530ae023 --- /dev/null +++ b/Lib/sqlite3/test/test_namedrow.py @@ -0,0 +1,178 @@ +# -*- coding: utf-8 -*- +# Tests the NamedRow row_factory with uft-8 for non ascii attribute names +# +# Copyright (C) 2019 Clinton James "JIDN" +# +# This software is provided 'as-is', without any express or implied +# warranty. In no even will the authors be held liable for any damages +# arising from the use of this software. +# +# Permission is granted to anyone to use this software for any purpose, +# including commercial applications, and to alter it and redistribute it +# freely, subject to the following restrictions: +# +# 1. Ths origin of this software must to be misrepresented; you must not +# claim that you wrote the original software. If you use this software +# in a product, an acknowledgment in the product documentation would be +# appreciated but is not required. +# 2. Altered source version must be plainly marked as such, and must not be +# misrepresented as being the original software. +# 3. This notice may not be removed or altered from any source distribution. + +import unittest +import sqlite3 + + +class NamedRowFactoryTests(unittest.TestCase): + def setUp(self): + self.conn = sqlite3.connect(":memory:") + self.conn.row_factory = sqlite3.NamedRow + + def tearDown(self): + self.conn.close() + + def CheckInstance(self): + """row_factory creates NamedRow instances""" + row = self.conn.execute("SELECT 1, 2").fetchone() + self.assertIsInstance(row, sqlite3.NamedRow) + self.assertIn("NamedRow", row.__class__.__name__) + self.assertIn("NamedRow", repr(row)) + self.assertIn( + "optimized row_factory for column name access", sqlite3.NamedRow.__doc__ + ) + + def CheckByNumberIndex(self): + """Get values by numbered index [0]""" + row = self.conn.execute("SELECT 1, 2").fetchone() + # len(row) + self.assertEqual(2, len(row)) + + self.assertEqual(row[0], 1) + self.assertEqual(row[1], 2) + self.assertEqual(row[-1], 2) + self.assertEqual(row[-2], 1) + + # Check past boundary + with self.assertRaises(IndexError): + row[len(row) + 1] + with self.assertRaises(IndexError): + row[-1 * (len(row) + 1)] + + def CheckByStringIndex(self): + """Get values by string index ['field']""" + row = self.conn.execute('SELECT 1 AS a, 2 AS abcdefg, 4 AS "四"').fetchone() + self.assertEqual(1, row["a"]) + self.assertEqual(2, row["abcdefg"]) + self.assertEqual(4, row["四"]) + + def CheckByAttribute(self): + """Get values by attribute row.field""" + row = self.conn.execute('SELECT 1 AS a, 2 AS abcdefg, 4 AS "四"').fetchone() + self.assertEqual(1, row.a) + self.assertEqual(2, row.abcdefg) + self.assertEqual(4, row.四) + + def CheckContainsField(self): + row = self.conn.execute("SELECT 1 AS a, 2 AS b").fetchone() + self.assertIn("a", row) + self.assertNotIn("A", row) + + def CheckSlice(self): + """Does NamedRow slice like a list.""" + row = self.conn.execute("SELECT 1, 2, 3, 4").fetchone() + self.assertEqual(row[0:0], ()) + self.assertEqual(row[0:1], (1,)) + self.assertEqual(row[1:3], (2, 3)) + self.assertEqual(row[3:1], ()) + # Explicit bounds are optional. + self.assertEqual(row[1:], (2, 3, 4)) + self.assertEqual(row[:3], (1, 2, 3)) + # Slices can use negative indices. + self.assertEqual(row[-2:-1], (3,)) + self.assertEqual(row[-2:], (3, 4)) + # Slicing supports steps. + self.assertEqual(row[0:4:2], (1, 3)) + self.assertEqual(row[3:0:-2], (4, 2)) + + def CheckHashComparison(self): + row1 = self.conn.execute("SELECT 1 AS a, 2 AS b").fetchone() + row2 = self.conn.execute("SELECT 1 AS a, 2 AS b").fetchone() + row3 = self.conn.execute("SELECT 1 AS a, 3 AS b").fetchone() + + self.assertEqual(row1, row1) + self.assertEqual(row1, row2) + self.assertTrue(row2 != row3) + + self.assertFalse(row1 != row1) + self.assertFalse(row1 != row2) + self.assertFalse(row2 == row3) + + self.assertEqual(row1, row2) + self.assertEqual(hash(row1), hash(row2)) + self.assertNotEqual(row1, row3) + self.assertNotEqual(hash(row1), hash(row3)) + + with self.assertRaises(TypeError): + row1 > row2 + with self.assertRaises(TypeError): + row1 < row2 + with self.assertRaises(TypeError): + row1 >= row2 + with self.assertRaises(TypeError): + row1 <= row1 + + def CheckFakeCursorClass(self): + # Issue #24257: Incorrect use of PyObject_IsInstance() caused + # segmentation fault. + # Issue #27861: Also applies for cursor factory. + class FakeCursor(str): + __class__ = sqlite3.Cursor + + self.assertRaises(TypeError, self.conn.cursor, FakeCursor) + self.assertRaises(TypeError, sqlite3.NamedRow, FakeCursor(), ()) + + def CheckIterable(self): + """Is NamedRow iterable.""" + row = self.conn.execute("SELECT 1 AS a, 2 AS abcdefg").fetchone() + self.assertEqual(2, len(row)) + for col in row: + # Its is a key/value pair + self.assertEqual(2, len(col)) + + def CheckIterablePairs(self): + expected = (("a", 1), ("b", 2)) + row = self.conn.execute("SELECT 1 AS a, 2 AS b").fetchone() + self.assertSequenceEqual(expected, [x for x in row]) + self.assertSequenceEqual(expected, tuple(row)) + self.assertSequenceEqual(expected, list(row)) + self.assertSequenceEqual(dict(expected), dict(row)) + + def CheckExpectedExceptions(self): + row = self.conn.execute("SELECT 1 AS a").fetchone() + with self.assertRaises(IndexError): + row["no_such_field"] + with self.assertRaises(AttributeError): + row.no_such_field + with self.assertRaises(TypeError): + row[0] = 100 + with self.assertRaises(TypeError): + row["a"] = 100 + with self.assertRaises(TypeError) as err: + row.a = 100 + self.assertIn("does not support item assignment", str(err.exception)) + + with self.assertRaises(TypeError): + setattr(row, "a", 100) + + +def suite(): + return unittest.makeSuite(NamedRowFactoryTests, "Check") + + +def test(): + runner = unittest.TextTestRunner() + runner.run(suite()) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_sqlite.py b/Lib/test/test_sqlite.py index 9564da35193f1f..16b54a1744a030 100644 --- a/Lib/test/test_sqlite.py +++ b/Lib/test/test_sqlite.py @@ -1,25 +1,45 @@ import test.support # Skip test if _sqlite3 module not installed -test.support.import_module('_sqlite3') +test.support.import_module("_sqlite3") import unittest import sqlite3 -from sqlite3.test import (dbapi, types, userfunctions, - factory, transactions, hooks, regression, - dump, backup) +from sqlite3.test import ( + dbapi, + types, + userfunctions, + factory, + transactions, + hooks, + regression, + dump, + backup, + test_namedrow, +) + def load_tests(*args): if test.support.verbose: - print("test_sqlite: testing with version", - "{!r}, sqlite_version {!r}".format(sqlite3.version, - sqlite3.sqlite_version)) - return unittest.TestSuite([dbapi.suite(), types.suite(), - userfunctions.suite(), - factory.suite(), transactions.suite(), - hooks.suite(), regression.suite(), - dump.suite(), - backup.suite()]) + print( + "test_sqlite: testing with version", + "{!r}, sqlite_version {!r}".format(sqlite3.version, sqlite3.sqlite_version), + ) + return unittest.TestSuite( + [ + dbapi.suite(), + types.suite(), + userfunctions.suite(), + factory.suite(), + transactions.suite(), + hooks.suite(), + regression.suite(), + dump.suite(), + backup.suite(), + test_namedrow.suite(), + ] + ) + if __name__ == "__main__": unittest.main() From f6f5e33c00d5c640d89f7ddcf19ae7b13af2de85 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Tue, 31 Dec 2019 14:45:28 +0000 Subject: [PATCH 9/9] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../NEWS.d/next/Library/2019-12-31-14-45-27.bpo-39170.bh3u0Q.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2019-12-31-14-45-27.bpo-39170.bh3u0Q.rst diff --git a/Misc/NEWS.d/next/Library/2019-12-31-14-45-27.bpo-39170.bh3u0Q.rst b/Misc/NEWS.d/next/Library/2019-12-31-14-45-27.bpo-39170.bh3u0Q.rst new file mode 100644 index 00000000000000..7f9b37ff0da159 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-12-31-14-45-27.bpo-39170.bh3u0Q.rst @@ -0,0 +1 @@ ++ Add sqlite3 row_factory for attribute access with the same memory requirements and speed of sqlite3.Row \ No newline at end of file