bpo-45138: Expand traced SQL statements in sqlite3 trace callback

Doc/library/sqlite3.rst

@@ -470,6 +470,9 @@ Connection Objects
 
       Passing :const:`None` as *trace_callback* will disable the trace callback.
 
+      For SQLite 3.14.0 and newer, bound parameters are expanded in the passed
+      statement string.
+
       .. note::
          Exceptions raised in the trace callback are not propagated. As a
          development and debugging aid, use
@@ -478,6 +481,9 @@ Connection Objects
 
       .. versionadded:: 3.3
 
+      .. versionchanged:: 3.11
+         Added support for expanded SQL statements.
+
 
    .. method:: enable_load_extension(enabled)
 

Doc/whatsnew/3.11.rst

@@ -231,6 +231,11 @@ sqlite3
   (Contributed by Aviv Palivoda, Daniel Shahaf, and Erlend E. Aasland in
   :issue:`16379`.)
 
+* For SQLite 3.14.0 and newer, bound parameters are expanded in the statement
+  string passed to the trace callback. See :meth:`~sqlite3.Connection.set_trace_callback`.
+  (Contributed by Erlend E. Aasland in :issue:`45138`.)
+
+
 time
 ----
 

Lib/sqlite3/test/test_hooks.py

@@ -20,8 +20,8 @@
 #    misrepresented as being the original software.
 # 3. This notice may not be removed or altered from any source distribution.
 
-import unittest
 import sqlite3 as sqlite
+import unittest
 
 from test.support.os_helper import TESTFN, unlink
 from .test_userfunctions import with_tracebacks
@@ -289,6 +289,31 @@ def trace(statement):
             con2.close()
         self.assertEqual(traced_statements, queries)
 
+    @unittest.skipIf(sqlite.sqlite_version_info < (3, 14, 0),
+                     "Requires SQLite 3.14.0 or newer")
+    def test_trace_expanded_sql(self):
+        cx = sqlite.connect(":memory:")
+        traced = []
+        cx.set_trace_callback(lambda stmt: traced.append(stmt))
+        with cx:
+            cx.execute("create table t(t)")
+            cx.executemany("insert into t values(?)", ((v,) for v in range(3)))
+        expected = [
+            "create table t(t)",
+            "BEGIN ",
+            "insert into t values(0)",
+            "insert into t values(1)",
+            "insert into t values(2)",
+            "COMMIT",
+        ]
+        self.assertEqual(traced, expected)
+
+    @with_tracebacks(["ZeroDivisionError", "5/0"])
+    def test_trace_bad_handler(self):
+        cx = sqlite.connect(":memory:")
+        cx.set_trace_callback(lambda stmt: 5/0)
+        cx.execute("select 1")
+
 
 if __name__ == "__main__":
     unittest.main()

Misc/NEWS.d/next/Library/2021-09-08-16-21-03.bpo-45138.yghUrK.rst

@@ -0,0 +1,3 @@
+For SQLite 3.14.0 and newer, bound parameters are expanded in the statement
+string passed to the :mod:`sqlite3` trace callback. Patch by Erlend E.
+Aasland.

Modules/_sqlite/connection.c

@@ -1059,11 +1059,10 @@ progress_callback(void *ctx)
  * to ensure future compatibility.
  */
 static int
-trace_callback(unsigned int type, void *ctx, void *prepared_statement,
-               void *statement_string)
+trace_callback(unsigned int type, void *ctx, void *stmt, void *sql)
 #else
 static void
-trace_callback(void *ctx, const char *statement_string)
+trace_callback(void *ctx, const char *sql)
 #endif
 {
 #ifdef HAVE_TRACE_V2
@@ -1074,11 +1073,37 @@ trace_callback(void *ctx, const char *statement_string)
 
     PyGILState_STATE gilstate = PyGILState_Ensure();
 
+    assert(ctx != NULL);
     PyObject *py_statement = NULL;
+#ifdef HAVE_TRACE_V2
+    assert(stmt != NULL);
+    const char *expanded_sql = sqlite3_expanded_sql((sqlite3_stmt *)stmt);
+    if (expanded_sql == NULL) {
+        sqlite3 *db = sqlite3_db_handle((sqlite3_stmt *)stmt);
+        if (sqlite3_errcode(db) == SQLITE_NOMEM) {
+            (void)PyErr_NoMemory();
+            goto exit;
+        }
+
+        pysqlite_state *state = ((callback_context *)ctx)->state;
+        assert(state != NULL);
+        if (state->enable_callback_tracebacks) {
+            PyErr_SetString(state->DataError,
+                            "Expanded SQL string exceeds the maximum string "
+                            "length");
+            PyErr_Print();
+        }
+        // Fall back to unexpanded sql
+        py_statement = PyUnicode_FromString((const char *)sql);
+    }
+    else {
+        py_statement = PyUnicode_FromString(expanded_sql);
+        sqlite3_free((void *)expanded_sql);
+    }
+#else
+    py_statement = PyUnicode_FromString(sql);
+#endif
     PyObject *ret = NULL;
-    py_statement = PyUnicode_DecodeUTF8(statement_string,
-            strlen(statement_string), "replace");
-    assert(ctx != NULL);
     if (py_statement) {
         PyObject *callable = ((callback_context *)ctx)->callable;
         ret = PyObject_CallOneArg(callable, py_statement);
@@ -1089,9 +1114,10 @@ trace_callback(void *ctx, const char *statement_string)
         Py_DECREF(ret);
     }
     else {
-        print_or_clear_traceback(ctx);
+        print_or_clear_traceback((callback_context *)ctx);
     }
 
+exit:
     PyGILState_Release(gilstate);
 #ifdef HAVE_TRACE_V2
     return 0;