gh-91602: Add iterdump() support for filtering database objects (#114501

)

Add optional 'filter' parameter to iterdump() that allows a "LIKE"
pattern for filtering database objects to dump.

Co-authored-by: Erlend E. Aasland <erlend@python.org>

Doc/library/sqlite3.rst

@@ -1137,12 +1137,19 @@ Connection objects
 
    .. _Loading an Extension: https://www.sqlite.org/loadext.html#loading_an_extension_
 
-   .. method:: iterdump
+   .. method:: iterdump(*, filter=None)
 
       Return an :term:`iterator` to dump the database as SQL source code.
       Useful when saving an in-memory database for later restoration.
       Similar to the ``.dump`` command in the :program:`sqlite3` shell.
 
+      :param filter:
+
+        An optional ``LIKE`` pattern for database objects to dump, e.g. ``prefix_%``.
+        If ``None`` (the default), all database objects will be included.
+
+      :type filter: str | None
+
       Example:
 
       .. testcode::
@@ -1158,6 +1165,8 @@ Connection objects
 
          :ref:`sqlite3-howto-encoding`
 
+      .. versionchanged:: 3.13
+         Added the *filter* parameter.
 
    .. method:: backup(target, *, pages=-1, progress=None, name="main", sleep=0.250)
 

Doc/whatsnew/3.13.rst

@@ -438,6 +438,10 @@ sqlite3
   object is not :meth:`closed <sqlite3.Connection.close>` explicitly.
   (Contributed by Erlend E. Aasland in :gh:`105539`.)
 
+* Add *filter* keyword-only parameter to :meth:`sqlite3.Connection.iterdump`
+  for filtering database objects to dump.
+  (Contributed by Mariusz Felisiak in :gh:`91602`.)
+
 subprocess
 ----------
 

Include/internal/pycore_global_strings.h

@@ -429,6 +429,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(fileno)
         STRUCT_FOR_ID(filepath)
         STRUCT_FOR_ID(fillvalue)
+        STRUCT_FOR_ID(filter)
         STRUCT_FOR_ID(filters)
         STRUCT_FOR_ID(final)
         STRUCT_FOR_ID(find_class)

Lib/sqlite3/dump.py

@@ -15,7 +15,7 @@ def _quote_value(value):
     return "'{0}'".format(value.replace("'", "''"))
 
 
-def _iterdump(connection):
+def _iterdump(connection, *, filter=None):
     """
     Returns an iterator to the dump of the database in an SQL text format.
 
@@ -32,15 +32,23 @@ def _iterdump(connection):
         yield('PRAGMA foreign_keys=OFF;')
     yield('BEGIN TRANSACTION;')
 
+    if filter:
+        # Return database objects which match the filter pattern.
+        filter_name_clause = 'AND "name" LIKE ?'
+        params = [filter]
+    else:
+        filter_name_clause = ""
+        params = []
     # sqlite_master table contains the SQL CREATE statements for the database.
-    q = """
+    q = f"""
         SELECT "name", "type", "sql"
         FROM "sqlite_master"
             WHERE "sql" NOT NULL AND
             "type" == 'table'
+            {filter_name_clause}
             ORDER BY "name"
         """
-    schema_res = cu.execute(q)
+    schema_res = cu.execute(q, params)
     sqlite_sequence = []
     for table_name, type, sql in schema_res.fetchall():
         if table_name == 'sqlite_sequence':
@@ -82,13 +90,14 @@ def _iterdump(connection):
             yield("{0};".format(row[0]))
 
     # Now when the type is 'index', 'trigger', or 'view'
-    q = """
+    q = f"""
         SELECT "name", "type", "sql"
         FROM "sqlite_master"
             WHERE "sql" NOT NULL AND
             "type" IN ('index', 'trigger', 'view')
+            {filter_name_clause}
         """
-    schema_res = cu.execute(q)
+    schema_res = cu.execute(q, params)
     for name, type, sql in schema_res.fetchall():
         yield('{0};'.format(sql))
 

Lib/test/test_sqlite3/test_dump.py

@@ -54,6 +54,76 @@ def test_table_dump(self):
         [self.assertEqual(expected_sqls[i], actual_sqls[i])
             for i in range(len(expected_sqls))]
 
+    def test_table_dump_filter(self):
+        all_table_sqls = [
+            """CREATE TABLE "some_table_2" ("id_1" INTEGER);""",
+            """INSERT INTO "some_table_2" VALUES(3);""",
+            """INSERT INTO "some_table_2" VALUES(4);""",
+            """CREATE TABLE "test_table_1" ("id_2" INTEGER);""",
+            """INSERT INTO "test_table_1" VALUES(1);""",
+            """INSERT INTO "test_table_1" VALUES(2);""",
+        ]
+        all_views_sqls = [
+            """CREATE VIEW "view_1" AS SELECT * FROM "some_table_2";""",
+            """CREATE VIEW "view_2" AS SELECT * FROM "test_table_1";""",
+        ]
+        # Create database structure.
+        for sql in [*all_table_sqls, *all_views_sqls]:
+            self.cu.execute(sql)
+        # %_table_% matches all tables.
+        dump_sqls = list(self.cx.iterdump(filter="%_table_%"))
+        self.assertEqual(
+            dump_sqls,
+            ["BEGIN TRANSACTION;", *all_table_sqls, "COMMIT;"],
+        )
+        # view_% matches all views.
+        dump_sqls = list(self.cx.iterdump(filter="view_%"))
+        self.assertEqual(
+            dump_sqls,
+            ["BEGIN TRANSACTION;", *all_views_sqls, "COMMIT;"],
+        )
+        # %_1 matches tables and views with the _1 suffix.
+        dump_sqls = list(self.cx.iterdump(filter="%_1"))
+        self.assertEqual(
+            dump_sqls,
+            [
+                "BEGIN TRANSACTION;",
+                """CREATE TABLE "test_table_1" ("id_2" INTEGER);""",
+                """INSERT INTO "test_table_1" VALUES(1);""",
+                """INSERT INTO "test_table_1" VALUES(2);""",
+                """CREATE VIEW "view_1" AS SELECT * FROM "some_table_2";""",
+                "COMMIT;"
+            ],
+        )
+        # some_% matches some_table_2.
+        dump_sqls = list(self.cx.iterdump(filter="some_%"))
+        self.assertEqual(
+            dump_sqls,
+            [
+                "BEGIN TRANSACTION;",
+                """CREATE TABLE "some_table_2" ("id_1" INTEGER);""",
+                """INSERT INTO "some_table_2" VALUES(3);""",
+                """INSERT INTO "some_table_2" VALUES(4);""",
+                "COMMIT;"
+            ],
+        )
+        # Only single object.
+        dump_sqls = list(self.cx.iterdump(filter="view_2"))
+        self.assertEqual(
+            dump_sqls,
+            [
+                "BEGIN TRANSACTION;",
+                """CREATE VIEW "view_2" AS SELECT * FROM "test_table_1";""",
+                "COMMIT;"
+            ],
+        )
+        # % matches all objects.
+        dump_sqls = list(self.cx.iterdump(filter="%"))
+        self.assertEqual(
+            dump_sqls,
+            ["BEGIN TRANSACTION;", *all_table_sqls, *all_views_sqls, "COMMIT;"],
+        )
+
     def test_dump_autoincrement(self):
         expected = [
             'CREATE TABLE "t1" (id integer primary key autoincrement);',

Misc/NEWS.d/next/Library/2024-01-24-20-51-49.gh-issue-91602.8fOH8l.rst

@@ -0,0 +1,3 @@
+Add *filter* keyword-only parameter to
+:meth:`sqlite3.Connection.iterdump` for filtering database objects to dump.
+Patch by Mariusz Felisiak.

Modules/_sqlite/connection.c

@@ -1979,12 +1979,17 @@ pysqlite_connection_interrupt_impl(pysqlite_Connection *self)
 /*[clinic input]
 _sqlite3.Connection.iterdump as pysqlite_connection_iterdump
 
+    *
+    filter: object = None
+        An optional LIKE pattern for database objects to dump
+
 Returns iterator to the dump of the database in an SQL text format.
 [clinic start generated code]*/
 
 static PyObject *
-pysqlite_connection_iterdump_impl(pysqlite_Connection *self)
-/*[clinic end generated code: output=586997aaf9808768 input=1911ca756066da89]*/
+pysqlite_connection_iterdump_impl(pysqlite_Connection *self,
+                                  PyObject *filter)
+/*[clinic end generated code: output=fd81069c4bdeb6b0 input=4ae6d9a898f108df]*/
 {
     if (!pysqlite_check_connection(self)) {
         return NULL;
@@ -1998,9 +2003,16 @@ pysqlite_connection_iterdump_impl(pysqlite_Connection *self)
         }
         return NULL;
     }
-
-    PyObject *retval = PyObject_CallOneArg(iterdump, (PyObject *)self);
+    PyObject *args[3] = {NULL, (PyObject *)self, filter};
+    PyObject *kwnames = Py_BuildValue("(s)", "filter");
+    if (!kwnames) {
+        Py_DECREF(iterdump);
+        return NULL;
+    }
+    Py_ssize_t nargsf = 1 | PY_VECTORCALL_ARGUMENTS_OFFSET;
+    PyObject *retval = PyObject_Vectorcall(iterdump, args + 1, nargsf, kwnames);
     Py_DECREF(iterdump);
+    Py_DECREF(kwnames);
     return retval;
 }