Support sqlite3_vtab_nochange / sqlite3_value_nochange

Fixes #402

apsw/tests.py

@@ -1124,7 +1124,7 @@ def testVTableStuff(self):
         if sys.version_info < (3, 7):
             # it works on 3.6 but apsw.ext doesn't because it uses dataclasses
             return
-        # we use apsw.ext.index_info_to_dict as part of the testing
+        # we also test apsw.ext
         import apsw.ext
 
         columns = [f"c{ n }" for n in range(30)]
@@ -1604,6 +1604,64 @@ def make_shadow():
         },
     ))
 
+    def testVTableNoChange(self):
+        "Test virtual table no change values on update"
+
+        class Source:
+            data = [(i, 2, i, 4, i) for i in range(10)]
+
+            def Create(self, *args):
+                return "create table ignored(c0, c1, c2, c3, c4)", Source.Table()
+
+            Connect = Create
+
+            class Table:
+
+                def BestIndex(self, *args):
+                    return None
+
+                def Open(self):
+                    return Source.Cursor()
+
+                def Disconnect(self):
+                    pass
+
+                Destroy = Disconnect
+
+                def UpdateChangeRow(tself, rowid, newrowid, fields):
+                    d=Source.data
+                    expected=(apsw.no_change, d[rowid][2]+1, apsw.no_change, 4, apsw.no_change)
+                    self.assertEqual(fields, expected)
+
+            class Cursor:
+
+                def Filter(self, *args):
+                    self.pos = 0
+
+                def Eof(self):
+                    return self.pos >= len(Source.data)
+
+                def Next(self):
+                    self.pos += 1
+
+                def Column(self, n):
+                    return Source.data[self.pos][n]
+
+                def ColumnNoChange(self, n):
+                    if n % 2:
+                        return self.Column(n)
+                    return apsw.no_change
+
+                def Close(self):
+                    pass
+
+                def Rowid(self):
+                    return self.pos
+
+        self.db.createmodule("testing", Source(), eponymous=True, use_no_change=True)
+        self.db.execute("update testing set c1=c2+1")
+
+
     def testWAL(self):
         "Test WAL functions"
         # note that it is harmless calling wal functions on a db not in wal mode
@@ -4813,7 +4871,8 @@ def testWikipedia(self):
                                                 "|declare_vtab|backup_remaining|backup_pagecount|mutex_enter|mutex_leave|sourceid|uri_.+"
                                                 "|column_name|column_decltype|column_database_name|column_table_name|column_origin_name"
                                                 "|stmt_isexplain|stmt_readonly|filename_journal|filename_wal|stmt_status|sql|log|vtab_collation"
-                                                "|vtab_rhs_value|vtab_distinct|vtab_config|vtab_on_conflict|vtab_in_first|vtab_in_next|vtab_in)$"),
+                                                "|vtab_rhs_value|vtab_distinct|vtab_config|vtab_on_conflict|vtab_in_first|vtab_in_next|vtab_in"
+                                                "|vtab_nochange)$"),
                         # error message
                         'desc': "sqlite3_ calls must wrap with PYSQLITE_CALL",
                         },

doc/changes.rst

@@ -30,6 +30,12 @@ Virtual table updates:
   <https://sqlite.org/vtab.html#eponymous_only_virtual_tables>`__, and
   read_only modules.
 
+* Virtual table updates can avoid having to provide all column
+  values when only a subset are changing.  See :attr:`apsw.no_change`,
+  :meth:`Connection.createmodule` *use_no_change* parameter,
+  :meth:`VTCursor.ColumnNoChange` and :meth:`VTTable.UpdateChangeRow`
+  (:issue:`402`)
+
 * :meth:`apsw.ext.make_virtual_module` makes it very easy to turn
   a Python function into a virtual table module.
 

src/apsw.c

@@ -135,6 +135,13 @@ static PyObject *apswmodule;
 /* root exception class */
 static PyObject *APSWException;
 
+/* no change sentinel for vtable updates */
+static PyTypeObject apsw_no_change_object = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+        .tp_name = "apsw.no_change",
+    .tp_doc = Apsw_no_change_DOC,
+};
+
 typedef struct
 {
   PyObject_HEAD long long blobsize;
@@ -1620,16 +1627,7 @@ PyInit_apsw(void)
     goto fail;
   }
 
-  if (PyType_Ready(&ConnectionType) < 0
-      || PyType_Ready(&APSWCursorType) < 0
-      || PyType_Ready(&ZeroBlobBindType) < 0
-      || PyType_Ready(&APSWBlobType) < 0
-      || PyType_Ready(&APSWVFSType) < 0
-      || PyType_Ready(&APSWVFSFileType) < 0
-      || PyType_Ready(&APSWURIFilenameType) < 0
-      || PyType_Ready(&FunctionCBInfoType) < 0
-      || PyType_Ready(&APSWBackupType) < 0
-      || PyType_Ready(&SqliteIndexInfoType) < 0)
+  if (PyType_Ready(&ConnectionType) < 0 || PyType_Ready(&APSWCursorType) < 0 || PyType_Ready(&ZeroBlobBindType) < 0 || PyType_Ready(&APSWBlobType) < 0 || PyType_Ready(&APSWVFSType) < 0 || PyType_Ready(&APSWVFSFileType) < 0 || PyType_Ready(&APSWURIFilenameType) < 0 || PyType_Ready(&FunctionCBInfoType) < 0 || PyType_Ready(&APSWBackupType) < 0 || PyType_Ready(&SqliteIndexInfoType) < 0 || PyType_Ready(&apsw_no_change_object) < 0)
     goto fail;
 
   m = apswmodule = PyModule_Create(&apswmoduledef);
@@ -1720,6 +1718,16 @@ PyInit_apsw(void)
   PyModule_AddObject(m, "using_amalgamation", Py_False);
 #endif
 
+  /** .. attribute:: no_change
+     :type: apsw.no_change
+
+    A sentinel used to indicate no change in a value when
+    used with :meth:`VTCursor.ColumnNoChange` and
+    :meth:`VTTable.UpdateChangeRow`
+  */
+
+  PyModule_AddObject(m, "no_change", Py_NewRef(&apsw_no_change_object));
+
   /**
 
 .. _sqliteconstants:

src/connection.c

@@ -113,6 +113,7 @@ typedef struct _vtableinfo
                              have to have a global table mapping sqlite3_db* to
                              Connection* */
   int bestindex_object;   /* 0: tuples are passed to xBestIndex, 1: object is */
+  int use_no_change;
   struct sqlite3_module *sqlite3_module_def;
 } vtableinfo;
 
@@ -2426,7 +2427,7 @@ getfunctionargs(sqlite3_context *context, PyObject *firstelement, int argc, sqli
 
   for (i = 0; i < argc; i++)
   {
-    PyObject *item = convert_value_to_pyobject(argv[i], 0);
+    PyObject *item = convert_value_to_pyobject(argv[i], 0, 0);
     if (!item)
     {
       sqlite3_result_error(context, "convert_value_to_pyobject failed", -1);
@@ -3571,14 +3572,15 @@ Connection_wal_checkpoint(Connection *self, PyObject *args, PyObject *kwds)
 static void apswvtabFree(void *context);
 static struct sqlite3_module *apswvtabSetupModuleDef(int iVersion, int eponymous, int eponymous_only, int read_only);
 
-/** .. method:: createmodule(name: str, datasource: Optional[VTModule], *, use_bestindex_object: bool = False, iVersion: int = 3, eponymous: bool=False, eponymous_only: bool = False, read_only: bool = False) -> None
+/** .. method:: createmodule(name: str, datasource: Optional[VTModule], *, use_bestindex_object: bool = False, use_no_change: bool = False, iVersion: int = 2, eponymous: bool=False, eponymous_only: bool = False, read_only: bool = False) -> None
 
     Registers a virtual table, or drops it if *datasource* is *None*.
     See :ref:`virtualtables` for details.
 
     :param name: Module name (what comes after USING in CREATE VIRTUAL TABLE tablename USING ...)
     :param datasource: Provides :class:`VTModule` methods
     :param use_bestindex_object: If True then BestIndexObject is used, else BestIndex
+    :param use_no_change: Turn on understanding :meth:`VTCursor.ColumnNoChange` and using :attr:`apsw.no_change` to reduce :meth:`VTTable.UpdateChangeRow` work
     :param iVersion: iVersion field in `sqlite3_module <https://www.sqlite.org/c3ref/module.html>`__
     :param eponymous: Configures module to be `eponymous <https://www.sqlite.org/vtab.html#eponymous_virtual_tables>`__
     :param eponymous_only: Configures module to be `eponymous only <https://www.sqlite.org/vtab.html#eponymous_only_virtual_tables>`__
@@ -3597,21 +3599,22 @@ Connection_createmodule(Connection *self, PyObject *args, PyObject *kwds)
   PyObject *datasource = NULL;
   vtableinfo *vti = NULL;
   int res;
-  int use_bestindex_object = 0;
+  int use_bestindex_object = 0, use_no_change = 0;
 
-  int iVersion = 3, eponymous = 0, eponymous_only = 0, read_only = 0;
+  int iVersion = 2, eponymous = 0, eponymous_only = 0, read_only = 0;
 
   CHECK_USE(NULL);
   CHECK_CLOSED(self, NULL);
 
   {
-    static char *kwlist[] = {"name", "datasource", "use_bestindex_object", "iVersion", "eponymous", "eponymous_only", "read_only", NULL};
+    static char *kwlist[] = {"name", "datasource", "use_bestindex_object", "use_no_change", "iVersion", "eponymous", "eponymous_only", "read_only", NULL};
     Connection_createmodule_CHECK;
     argcheck_bool_param use_bestindex_object_param = {&use_bestindex_object, Connection_createmodule_use_bestindex_object_MSG};
+    argcheck_bool_param use_no_change_param = {&use_no_change, Connection_createmodule_use_no_change_MSG};
     argcheck_bool_param eponymous_param = {&eponymous, Connection_createmodule_eponymous_MSG};
     argcheck_bool_param eponymous_only_param = {&eponymous_only, Connection_createmodule_eponymous_only_MSG};
     argcheck_bool_param read_only_param = {&read_only, Connection_createmodule_read_only_MSG};
-    if (!PyArg_ParseTupleAndKeywords(args, kwds, "sO|$O&iO&O&O&:" Connection_createmodule_USAGE, kwlist, &name, &datasource, argcheck_bool, &use_bestindex_object_param, &iVersion, argcheck_bool, &eponymous_param, argcheck_bool, &eponymous_only_param, argcheck_bool, &read_only_param))
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "sO|$O&O&iO&O&O&:" Connection_createmodule_USAGE, kwlist, &name, &datasource, argcheck_bool, &use_bestindex_object_param, argcheck_bool, &use_no_change_param, &iVersion, argcheck_bool, &eponymous_param, argcheck_bool, &eponymous_only_param, argcheck_bool, &read_only_param))
       return NULL;
   }
 
@@ -3627,6 +3630,7 @@ Connection_createmodule(Connection *self, PyObject *args, PyObject *kwds)
     vti->connection = self;
     vti->datasource = datasource;
     vti->bestindex_object = use_bestindex_object;
+    vti->use_no_change = use_no_change;
   }
 
   /* SQLite is really finnicky.  Note that it calls the destructor on

src/util.c

@@ -204,11 +204,14 @@ apsw_write_unraisable(PyObject *hookobject)
 
 /* Converts sqlite3_value to PyObject.  Returns a new reference. */
 static PyObject *
-convert_value_to_pyobject(sqlite3_value *value, int in_constraint_possible)
+convert_value_to_pyobject(sqlite3_value *value, int in_constraint_possible, int no_change_possible)
 {
   int coltype = sqlite3_value_type(value);
   sqlite3_value *in_value;
 
+  if (no_change_possible && sqlite3_value_nochange(value))
+    return Py_NewRef(&apsw_no_change_object);
+
   APSW_FAULT_INJECT(UnknownValueType, , coltype = 123456);
 
   switch (coltype)
@@ -235,7 +238,7 @@ convert_value_to_pyobject(sqlite3_value *value, int in_constraint_possible)
         return NULL;
       while (in_value)
       {
-        v = convert_value_to_pyobject(in_value, 0);
+        v = convert_value_to_pyobject(in_value, 0, 0);
         if (!v || 0 != PySet_Add(set, v))
           goto error;
         v = NULL;
@@ -270,7 +273,7 @@ convert_value_to_pyobject(sqlite3_value *value, int in_constraint_possible)
 static PyObject *
 convert_value_to_pyobject_not_in(sqlite3_value *value)
 {
-  return convert_value_to_pyobject(value, 0);
+  return convert_value_to_pyobject(value, 0, 0);
 }
 
 /* Converts column to PyObject.  Returns a new reference. Almost identical to above
@@ -423,4 +426,4 @@ assert fail or cause a valgrind error.
 
 /* some arbitrary magic numbers for call track */
 #define MAGIC_xConnect 0x008295ab
-#define MAGIC_xUpdate  0x119306bc
+#define MAGIC_xUpdate 0x119306bc