From 8ae3b16a9adfd49d1fb99dc8147b81964c7958bf Mon Sep 17 00:00:00 2001 From: Klemen Tusar Date: Mon, 6 Oct 2025 22:51:29 +0100 Subject: [PATCH 1/3] :safety_vest: add support for SQLite STRICT tables and enhance related logic --- src/mysql_to_sqlite3/cli.py | 8 ++ src/mysql_to_sqlite3/transporter.py | 14 ++- src/mysql_to_sqlite3/types.py | 2 + tests/unit/test_transporter.py | 152 ++++++++++++++++++++++++++++ 4 files changed, 175 insertions(+), 1 deletion(-) diff --git a/src/mysql_to_sqlite3/cli.py b/src/mysql_to_sqlite3/cli.py index 92f3d69..e664b71 100644 --- a/src/mysql_to_sqlite3/cli.py +++ b/src/mysql_to_sqlite3/cli.py @@ -143,6 +143,12 @@ help="Use the VACUUM command to rebuild the SQLite database file, " "repacking it into a minimal amount of disk space", ) +@click.option( + "-M", + "--strict", + is_flag=True, + help="Create SQLite STRICT tables when supported.", +) @click.option( "--use-buffered-cursors", is_flag=True, @@ -176,6 +182,7 @@ def cli( log_file: t.Union[str, "os.PathLike[t.Any]"], json_as_text: bool, vacuum: bool, + strict: bool, use_buffered_cursors: bool, quiet: bool, debug: bool, @@ -223,6 +230,7 @@ def cli( chunk=chunk, json_as_text=json_as_text, vacuum=vacuum, + sqlite_strict=strict, buffered=use_buffered_cursors, log_file=log_file, quiet=quiet, diff --git a/src/mysql_to_sqlite3/transporter.py b/src/mysql_to_sqlite3/transporter.py index 3e792cd..306fa10 100644 --- a/src/mysql_to_sqlite3/transporter.py +++ b/src/mysql_to_sqlite3/transporter.py @@ -120,8 +120,17 @@ def __init__(self, **kwargs: Unpack[MySQLtoSQLiteParams]) -> None: self._quiet = bool(kwargs.get("quiet", False)) + self._sqlite_strict = bool(kwargs.get("sqlite_strict", False)) + self._logger = self._setup_logger(log_file=kwargs.get("log_file") or None, quiet=self._quiet) + if self._sqlite_strict and sqlite3.sqlite_version < "3.37.0": + self._logger.warning( + "SQLite version %s does not support STRICT tables. Tables will be created without strict mode.", + sqlite3.sqlite_version, + ) + self._sqlite_strict = False + sqlite3.register_adapter(Decimal, adapt_decimal) sqlite3.register_converter("DECIMAL", convert_decimal) sqlite3.register_adapter(timedelta, adapt_timedelta) @@ -570,7 +579,10 @@ def _build_create_table_sql(self, table_name: str) -> str: "ON DELETE {on_delete}".format(**foreign_key) # type: ignore[str-bytes-safe] ) - sql += "\n);" + sql += "\n)" + if self._sqlite_strict: + sql += " STRICT" + sql += ";\n" sql += indices return sql diff --git a/src/mysql_to_sqlite3/types.py b/src/mysql_to_sqlite3/types.py index 0cdbcf8..9bc536f 100644 --- a/src/mysql_to_sqlite3/types.py +++ b/src/mysql_to_sqlite3/types.py @@ -39,6 +39,7 @@ class MySQLtoSQLiteParams(TypedDict): prefix_indices: t.Optional[bool] quiet: t.Optional[bool] sqlite_file: t.Union[str, "os.PathLike[t.Any]"] + sqlite_strict: t.Optional[bool] vacuum: t.Optional[bool] without_tables: t.Optional[bool] without_data: t.Optional[bool] @@ -74,6 +75,7 @@ class MySQLtoSQLiteAttributes: _sqlite: Connection _sqlite_cur: Cursor _sqlite_file: t.Union[str, "os.PathLike[t.Any]"] + _sqlite_strict: bool _without_tables: bool _sqlite_json1_extension_enabled: bool _vacuum: bool diff --git a/tests/unit/test_transporter.py b/tests/unit/test_transporter.py index 3ff5b91..9c91268 100644 --- a/tests/unit/test_transporter.py +++ b/tests/unit/test_transporter.py @@ -1,8 +1,11 @@ +import builtins import sqlite3 from unittest.mock import MagicMock, patch import pytest +from pytest_mock import MockerFixture +from mysql_to_sqlite3.sqlite_utils import CollatingSequences from mysql_to_sqlite3.transporter import MySQLtoSQLite @@ -152,6 +155,155 @@ def test_transfer_exception_handling(self, mock_sqlite_connect: MagicMock, mock_ # Verify that foreign keys are re-enabled in the finally block mock_sqlite_cursor.execute.assert_called_with("PRAGMA foreign_keys=ON") + @patch("mysql_to_sqlite3.transporter.sqlite3.connect") + @patch("mysql_to_sqlite3.transporter.mysql.connector.connect") + def test_sqlite_strict_supported_keeps_flag( + self, + mock_mysql_connect: MagicMock, + mock_sqlite_connect: MagicMock, + mocker: MockerFixture, + ) -> None: + """Ensure STRICT mode remains enabled when SQLite supports it.""" + + class FakeMySQLConnection: + def __init__(self) -> None: + self.database = None + + def is_connected(self) -> bool: + return True + + def cursor(self, *args, **kwargs) -> MagicMock: + return MagicMock() + + mock_logger = MagicMock() + mocker.patch.object(MySQLtoSQLite, "_setup_logger", return_value=mock_logger) + mocker.patch("mysql_to_sqlite3.transporter.sqlite3.sqlite_version", "3.38.0") + mock_mysql_connect.return_value = FakeMySQLConnection() + + mock_sqlite_cursor = MagicMock() + mock_sqlite_connection = MagicMock() + mock_sqlite_connection.cursor.return_value = mock_sqlite_cursor + mock_sqlite_connect.return_value = mock_sqlite_connection + + from mysql_to_sqlite3 import transporter as transporter_module + + original_isinstance = builtins.isinstance + + def fake_isinstance(obj: object, classinfo: object) -> bool: + if classinfo is transporter_module.MySQLConnectionAbstract: + return True + return original_isinstance(obj, classinfo) + + mocker.patch("mysql_to_sqlite3.transporter.isinstance", side_effect=fake_isinstance) + + instance = MySQLtoSQLite( + sqlite_file="file.db", + mysql_user="user", + mysql_password=None, + mysql_database="db", + mysql_host="localhost", + mysql_port=3306, + sqlite_strict=True, + ) + + assert instance._sqlite_strict is True + mock_logger.warning.assert_not_called() + + @patch("mysql_to_sqlite3.transporter.sqlite3.connect") + @patch("mysql_to_sqlite3.transporter.mysql.connector.connect") + def test_sqlite_strict_unsupported_disables_flag( + self, + mock_mysql_connect: MagicMock, + mock_sqlite_connect: MagicMock, + mocker: MockerFixture, + ) -> None: + """Ensure STRICT mode is disabled with a warning on old SQLite versions.""" + + class FakeMySQLConnection: + def __init__(self) -> None: + self.database = None + + def is_connected(self) -> bool: + return True + + def cursor(self, *args, **kwargs) -> MagicMock: + return MagicMock() + + mock_logger = MagicMock() + mocker.patch.object(MySQLtoSQLite, "_setup_logger", return_value=mock_logger) + mocker.patch("mysql_to_sqlite3.transporter.sqlite3.sqlite_version", "3.36.0") + mock_mysql_connect.return_value = FakeMySQLConnection() + + mock_sqlite_cursor = MagicMock() + mock_sqlite_connection = MagicMock() + mock_sqlite_connection.cursor.return_value = mock_sqlite_cursor + mock_sqlite_connect.return_value = mock_sqlite_connection + + from mysql_to_sqlite3 import transporter as transporter_module + + original_isinstance = builtins.isinstance + + def fake_isinstance(obj: object, classinfo: object) -> bool: + if classinfo is transporter_module.MySQLConnectionAbstract: + return True + return original_isinstance(obj, classinfo) + + mocker.patch("mysql_to_sqlite3.transporter.isinstance", side_effect=fake_isinstance) + + instance = MySQLtoSQLite( + sqlite_file="file.db", + mysql_user="user", + mysql_password=None, + mysql_database="db", + mysql_host="localhost", + mysql_port=3306, + sqlite_strict=True, + ) + + assert instance._sqlite_strict is False + mock_logger.warning.assert_called_once() + + def test_build_create_table_sql_appends_strict(self) -> None: + """Ensure STRICT is appended to CREATE TABLE statements when enabled.""" + with patch.object(MySQLtoSQLite, "__init__", return_value=None): + instance = MySQLtoSQLite() + + instance._sqlite_strict = True + instance._sqlite_json1_extension_enabled = False + instance._mysql_cur_dict = MagicMock() + instance._mysql_cur_dict.fetchall.side_effect = [ + [ + { + "Field": "id", + "Type": "INTEGER", + "Null": "NO", + "Default": None, + "Key": "PRI", + "Extra": "auto_increment", + }, + { + "Field": "name", + "Type": "TEXT", + "Null": "NO", + "Default": None, + "Key": "", + "Extra": "", + }, + ], + [], + ] + instance._mysql_cur_dict.fetchone.return_value = {"count": 0} + instance._mysql_database = "db" + instance._collation = CollatingSequences.BINARY + instance._prefix_indices = False + instance._without_tables = False + instance._without_foreign_keys = True + instance._logger = MagicMock() + + sql = instance._build_create_table_sql("products") + + assert "STRICT;" in sql + def test_constructor_missing_mysql_database(self) -> None: """Test constructor raises ValueError if mysql_database is missing.""" from mysql_to_sqlite3.transporter import MySQLtoSQLite From 8acb3458a316720916c0cfaa9c1b29c4903aceb3 Mon Sep 17 00:00:00 2001 From: Klemen Tusar Date: Mon, 6 Oct 2025 22:55:21 +0100 Subject: [PATCH 2/3] :safety_vest: add CLI option for creating SQLite STRICT tables --- src/mysql_to_sqlite3/cli.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/mysql_to_sqlite3/cli.py b/src/mysql_to_sqlite3/cli.py index e664b71..1c7ffe0 100644 --- a/src/mysql_to_sqlite3/cli.py +++ b/src/mysql_to_sqlite3/cli.py @@ -106,6 +106,12 @@ is_flag=True, help="Do not transfer table data, DDL only.", ) +@click.option( + "-M", + "--strict", + is_flag=True, + help="Create SQLite STRICT tables when supported.", +) @click.option("-h", "--mysql-host", default="localhost", help="MySQL host. Defaults to localhost.") @click.option("-P", "--mysql-port", type=int, default=3306, help="MySQL port. Defaults to 3306.") @click.option( @@ -143,12 +149,6 @@ help="Use the VACUUM command to rebuild the SQLite database file, " "repacking it into a minimal amount of disk space", ) -@click.option( - "-M", - "--strict", - is_flag=True, - help="Create SQLite STRICT tables when supported.", -) @click.option( "--use-buffered-cursors", is_flag=True, @@ -173,6 +173,7 @@ def cli( without_foreign_keys: bool, without_tables: bool, without_data: bool, + strict: bool, mysql_host: str, mysql_port: int, mysql_charset: str, @@ -182,7 +183,6 @@ def cli( log_file: t.Union[str, "os.PathLike[t.Any]"], json_as_text: bool, vacuum: bool, - strict: bool, use_buffered_cursors: bool, quiet: bool, debug: bool, @@ -222,6 +222,7 @@ def cli( without_foreign_keys=without_foreign_keys or bool(mysql_tables) or bool(exclude_mysql_tables), without_tables=without_tables, without_data=without_data, + sqlite_strict=strict, mysql_host=mysql_host, mysql_port=mysql_port, mysql_charset=mysql_charset, @@ -230,7 +231,6 @@ def cli( chunk=chunk, json_as_text=json_as_text, vacuum=vacuum, - sqlite_strict=strict, buffered=use_buffered_cursors, log_file=log_file, quiet=quiet, From 30c4dcd519ff2bd0e4baec8a26368880a95e889c Mon Sep 17 00:00:00 2001 From: Klemen Tusar Date: Mon, 6 Oct 2025 22:57:35 +0100 Subject: [PATCH 3/3] :bulb: add CLI option for creating SQLite STRICT tables in documentation --- README.md | 1 + docs/README.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/README.md b/README.md index 10cf1eb..aa79156 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,7 @@ Options: -X, --without-foreign-keys Do not transfer foreign keys. -Z, --without-tables Do not transfer tables, data only. -W, --without-data Do not transfer table data, DDL only. + -M, --strict Create SQLite STRICT tables when supported. -h, --mysql-host TEXT MySQL host. Defaults to localhost. -P, --mysql-port INTEGER MySQL port. Defaults to 3306. --mysql-charset TEXT MySQL database and table character set diff --git a/docs/README.rst b/docs/README.rst index 3eb421e..83574fd 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -38,6 +38,7 @@ Transfer Options - ``-X, --without-foreign-keys``: Do not transfer foreign keys. - ``-Z, --without-tables``: Do not transfer tables, data only. - ``-W, --without-data``: Do not transfer table data, DDL only. +- ``-M, --strict``: Create SQLite STRICT tables when supported. Connection Options """"""""""""""""""