From 25233066f233eb00f53f8cc9d75546e0dfd6eebf Mon Sep 17 00:00:00 2001 From: Finn Stutzenstein Date: Tue, 17 Aug 2021 09:47:46 +0200 Subject: [PATCH] Changed initial migration index (#149) Use -1 as the default migration index (previously it was 1 if not the import tools were used). Supersedes #145 --- cli/export_data_only.py | 3 +- .../sql_read_database_backend_service.py | 3 +- tests/migrations/system/conftest.py | 18 +++++ .../system/test_basic_migrate_cases.py | 36 ++++++--- tests/migrations/system/test_check_latest.py | 14 +--- .../system/test_errors_during_migration.py | 25 ++++-- .../migrations/system/test_event_overwrite.py | 21 ++++- .../system/test_finalize_noop_migration.py | 5 +- .../system/test_migration_index_too_high.py | 13 +++- .../system/test_migration_keyframes.py | 3 +- .../test_multiple_migrations_and_positions.py | 78 +++++++++++++++---- .../system/test_premade_migration_classes.py | 16 +++- .../migrations/system/test_reset_migration.py | 18 ++++- .../system/test_sample_migrations.py | 34 +++++++- .../system/test_wrong_new_events.py | 73 ++++++++++++----- .../test_sql_read_database_backend_service.py | 4 +- .../system/write/test_migration_index.py | 4 +- 17 files changed, 280 insertions(+), 88 deletions(-) diff --git a/cli/export_data_only.py b/cli/export_data_only.py index f4548b26..d9d7d452 100644 --- a/cli/export_data_only.py +++ b/cli/export_data_only.py @@ -9,6 +9,7 @@ DATASTORE_DEV_MODE_ENVIRONMENT_VAR, EnvironmentService, ) +from datastore.shared.util import is_reserved_field def main(): @@ -26,7 +27,7 @@ def main(): for collection, models in response.items(): for model in models: for field in list(model.keys()): - if field.startswith("meta_"): + if is_reserved_field(field): del model[field] response["_migration_index"] = cast(Any, migration_index) diff --git a/datastore/shared/postgresql_backend/sql_read_database_backend_service.py b/datastore/shared/postgresql_backend/sql_read_database_backend_service.py index 24fce700..d073c40d 100644 --- a/datastore/shared/postgresql_backend/sql_read_database_backend_service.py +++ b/datastore/shared/postgresql_backend/sql_read_database_backend_service.py @@ -122,6 +122,7 @@ def get_everything( result = self.connection.query(query, [], []) unsorted_data = defaultdict(list) + for row in result: collection, id = collection_and_id_from_fqid(row["__fqid__"]) model = row["data"] @@ -359,7 +360,7 @@ def get_current_migration_index(self) -> int: + f"different to {max_migration_index}" ) else: - max_migration_index = 1 + max_migration_index = -1 self.current_migration_index = max_migration_index return self.current_migration_index diff --git a/tests/migrations/system/conftest.py b/tests/migrations/system/conftest.py index c9602dad..81d5e1a8 100644 --- a/tests/migrations/system/conftest.py +++ b/tests/migrations/system/conftest.py @@ -1,3 +1,5 @@ +from unittest.mock import MagicMock + import pytest from datastore.migrations import MigrationHandler, setup as migration_setup @@ -145,3 +147,19 @@ def _assert_finalized(): assert_count("migration_positions", 0) return _assert_finalized + + +@pytest.fixture() +def set_migration_index_to_1(migration_handler): + def _set_migration_index_to_1(): + previous_logger = migration_handler.logger.info + migration_handler.logger.info = i = MagicMock() + migration_handler.migrate() # set target migration index to 1 + i.assert_called() + assert ( + "The datastore has a migration index of -1. Set the migration index to 1." + in i.call_args.args[0] + ) + migration_handler.logger.info = previous_logger + + yield _set_migration_index_to_1 diff --git a/tests/migrations/system/test_basic_migrate_cases.py b/tests/migrations/system/test_basic_migrate_cases.py index e4998fa7..463ce4a7 100644 --- a/tests/migrations/system/test_basic_migrate_cases.py +++ b/tests/migrations/system/test_basic_migrate_cases.py @@ -12,47 +12,62 @@ def test_no_migrations_to_apply( migration_handler, write, + set_migration_index_to_1, ): write({"type": "create", "fqid": "a/1", "fields": {}}) + set_migration_index_to_1() migration_handler.register_migrations(get_noop_migration(2)) + migration_handler.logger.info = i = MagicMock() migration_handler.finalize() - migration_handler.logger.info = i = MagicMock() + i.assert_called() + assert "Position 1 from MI 1 to MI 2" in i.call_args_list[1].args[0] + + i.reset_mock() migration_handler.migrate() i.assert_called() assert ( "No migrations to apply. The productive database is up to date." - in i.call_args[0][0] + in i.call_args.args[0] ) def test_finalizing_needed( migration_handler, write, + set_migration_index_to_1, ): write({"type": "create", "fqid": "a/1", "fields": {}}) + set_migration_index_to_1() migration_handler.register_migrations(get_noop_migration(2)) + migration_handler.logger.info = i = MagicMock() migration_handler.migrate() - migration_handler.logger.info = i = MagicMock() + i.assert_called() + assert "Position 1 from MI 1 to MI 2" in i.call_args_list[1].args[0] + assert "Done. Finalizing is still be needed." in i.call_args.args[0] + + i.reset_mock() migration_handler.migrate() i.assert_called() assert ( "No migrations to apply, but finalizing is still needed." - in i.call_args_list[1][0][0] + in i.call_args_list[1].args[0] ) - assert "Done. Finalizing is still be needed." in i.call_args_list[2][0][0] + assert "Done. Finalizing is still be needed." in i.call_args.args[0] def test_finalizing_not_needed( migration_handler, write, + set_migration_index_to_1, ): write({"type": "create", "fqid": "a/1", "fields": {}}) + set_migration_index_to_1() migration_handler.register_migrations(get_noop_migration(2)) migration_handler.finalize() @@ -72,17 +87,12 @@ def test_invalid_migration_index( connection_handler, ): write({"type": "create", "fqid": "a/1", "fields": {}}) - - with connection_handler.get_connection_context(): - connection_handler.execute( - "update positions set migration_index=%s", - [-1], - ) + # DS MI is -1 migrater = injector.get(Migrater) with pytest.raises(MismatchingMigrationIndicesException) as e: - migrater.migrate(2, get_noop_migration(2)()) + migrater.migrate(2, {2: get_noop_migration(2)()}) assert ( str(e.value) @@ -94,9 +104,11 @@ def test_raising_migration_index( migration_handler, write, connection_handler, + set_migration_index_to_1, ): write({"type": "create", "fqid": "a/1", "fields": {}}) write({"type": "create", "fqid": "a/2", "fields": {}}) + set_migration_index_to_1() with connection_handler.get_connection_context(): connection_handler.execute( diff --git a/tests/migrations/system/test_check_latest.py b/tests/migrations/system/test_check_latest.py index c4823aa4..7f997d14 100644 --- a/tests/migrations/system/test_check_latest.py +++ b/tests/migrations/system/test_check_latest.py @@ -8,13 +8,8 @@ def test_set_latest_migrate( ): write({"type": "create", "fqid": "a/1", "fields": {}}) write({"type": "create", "fqid": "a/2", "fields": {}}) - with connection_handler.get_connection_context(): - connection_handler.execute( - "update positions set migration_index=%s", - [-1], - ) - migration_handler.run_migrations = rm = MagicMock() + migration_handler.run_migrations = rm = MagicMock() migration_handler.register_migrations(get_noop_migration(2), get_noop_migration(3)) migration_handler.migrate() @@ -28,13 +23,8 @@ def test_migration_index_too_high_finalize( ): write({"type": "create", "fqid": "a/1", "fields": {}}) write({"type": "create", "fqid": "a/2", "fields": {}}) - with connection_handler.get_connection_context(): - connection_handler.execute( - "update positions set migration_index=%s", - [-1], - ) - migration_handler.run_migrations = rm = MagicMock() + migration_handler.run_migrations = rm = MagicMock() migration_handler.register_migrations(get_noop_migration(2), get_noop_migration(3)) migration_handler.finalize() diff --git a/tests/migrations/system/test_errors_during_migration.py b/tests/migrations/system/test_errors_during_migration.py index 20c86287..dc268f5e 100644 --- a/tests/migrations/system/test_errors_during_migration.py +++ b/tests/migrations/system/test_errors_during_migration.py @@ -26,13 +26,20 @@ def fail_handler(event): def test_failing_migration( - migration_handler, write, read_model, assert_model, assert_finalized + migration_handler, + write, + set_migration_index_to_1, + read_model, + assert_model, + assert_finalized, ): - fail_migration = get_lambda_migration(do_raise(AbortException())) - migration_handler.register_migrations(fail_migration) write({"type": "create", "fqid": "a/1", "fields": {"f": 1}}) + set_migration_index_to_1() previous_model = read_model("a/1") + fail_migration = get_lambda_migration(do_raise(AbortException())) + migration_handler.register_migrations(fail_migration) + with pytest.raises(AbortException): migration_handler.migrate() @@ -50,17 +57,20 @@ def test_failing_migration_multi_positions( connection_handler, migration_handler, write, + set_migration_index_to_1, read_model, assert_model, assert_finalized, assert_count, ): - migration_handler.register_migrations(get_lambda_migration(fail_handler)) write({"type": "create", "fqid": "a/1", "fields": {"f": 1}}) write({"type": "create", "fqid": "a/2", "fields": {"f": 1}}) write({"type": "create", "fqid": "a/3", "fields": {"f": 1}}) + set_migration_index_to_1() previous_models = (read_model("a/1"), read_model("a/2"), read_model("a/3")) + migration_handler.register_migrations(get_lambda_migration(fail_handler)) + with pytest.raises(AbortException): migration_handler.migrate() @@ -87,18 +97,21 @@ def test_failing_migration_multi_positions_new_migration_after_fail( connection_handler, migration_handler, write, + set_migration_index_to_1, read_model, assert_model, assert_finalized, assert_count, query_single_value, ): - migration_handler.register_migrations(get_lambda_migration(fail_handler)) write({"type": "create", "fqid": "a/1", "fields": {"f": 1}}) write({"type": "create", "fqid": "a/2", "fields": {"f": 1}}) write({"type": "create", "fqid": "a/3", "fields": {"f": 1}}) + set_migration_index_to_1() previous_models = (read_model("a/1"), read_model("a/2"), read_model("a/3")) + migration_handler.register_migrations(get_lambda_migration(fail_handler)) + with pytest.raises(AbortException): migration_handler.migrate() @@ -129,6 +142,7 @@ def test_use_basemigration( connection_handler, migration_handler, write, + set_migration_index_to_1, read_model, assert_model, assert_finalized, @@ -136,6 +150,7 @@ def test_use_basemigration( query_single_value, ): write({"type": "create", "fqid": "a/1", "fields": {"f": 1}}) + set_migration_index_to_1() class MyBaseMigration(BaseMigration): target_migration_index = 2 diff --git a/tests/migrations/system/test_event_overwrite.py b/tests/migrations/system/test_event_overwrite.py index d729752e..52a842d8 100644 --- a/tests/migrations/system/test_event_overwrite.py +++ b/tests/migrations/system/test_event_overwrite.py @@ -7,8 +7,11 @@ from ..util import get_lambda_migration, get_noop_migration -def test_returning_original_events(migration_handler, connection_handler, write): +def test_returning_original_events( + migration_handler, connection_handler, write, set_migration_index_to_1 +): write({"type": "create", "fqid": "a/1", "fields": {"f": 1}}) + set_migration_index_to_1() with connection_handler.get_connection_context(): original_events = connection_handler.query( @@ -26,12 +29,14 @@ def test_returning_original_events(migration_handler, connection_handler, write) def test_new_events( migration_handler, connection_handler, - assert_count, write, + set_migration_index_to_1, + assert_count, assert_model, exists_model, ): write({"type": "create", "fqid": "a/1", "fields": {}}) + set_migration_index_to_1() new_events = [CreateEvent(f"a/{i}", {}) for i in (2, 3, 4)] migration_handler.register_migrations(get_lambda_migration(lambda e: new_events)) @@ -48,13 +53,15 @@ def test_new_events( def test_new_events_multiple_positions( migration_handler, connection_handler, - assert_count, write, + set_migration_index_to_1, + assert_count, assert_model, exists_model, ): write({"type": "create", "fqid": "a/1", "fields": {}}) write({"type": "create", "fqid": "b/1", "fields": {}}) + set_migration_index_to_1() a_new_events = [CreateEvent(f"a/{i}", {}) for i in (2, 3)] b_new_events = [CreateEvent(f"b/{i}", {}) for i in (2, 3)] @@ -78,12 +85,14 @@ def test_new_events_rebuilding_order( migration_handler, connection_handler, write, + set_migration_index_to_1, assert_model, exists_model, assert_count, ): write({"type": "create", "fqid": "a/1", "fields": {"f": 1}}) write({"type": "update", "fqid": "a/1", "fields": {"f": 2}}) + set_migration_index_to_1() create_new_events = [ CreateEvent("a/1", {"f": 4}), @@ -114,11 +123,13 @@ def test_less_events( migration_handler, assert_count, write, + set_migration_index_to_1, assert_model, exists_model, ): write({"type": "create", "fqid": "a/1", "fields": {"f": 1}}) write({"type": "update", "fqid": "a/1", "fields": {"f": None, "f2": 2}}) + set_migration_index_to_1() assert_count("events", 3) migration_handler.register_migrations( @@ -140,10 +151,12 @@ def test_return_none_with_modifications( migration_handler, assert_count, write, + set_migration_index_to_1, assert_model, read_model, ): write({"type": "create", "fqid": "a/1", "fields": {"f": 1}}) + set_migration_index_to_1() previous_model = read_model("a/1") def handler(event): @@ -161,10 +174,12 @@ def test_additional_events( connection_handler, assert_count, write, + set_migration_index_to_1, assert_model, exists_model, ): write({"type": "create", "fqid": "a/1", "fields": {}}) + set_migration_index_to_1() class TestMigration(BaseMigration): target_migration_index = 2 diff --git a/tests/migrations/system/test_finalize_noop_migration.py b/tests/migrations/system/test_finalize_noop_migration.py index 5239d459..e6c1534c 100644 --- a/tests/migrations/system/test_finalize_noop_migration.py +++ b/tests/migrations/system/test_finalize_noop_migration.py @@ -12,15 +12,18 @@ def readback( self, migration_handler, write, + set_migration_index_to_1, read_model, assert_model, query_single_value, assert_finalized, ): def _readback(*data): - migration_handler.register_migrations(get_noop_migration(2)) self.write_data(write, *data) + set_migration_index_to_1() previous_model = read_model("a/1") + + migration_handler.register_migrations(get_noop_migration(2)) migration_handler.finalize() assert_model("a/1", previous_model) diff --git a/tests/migrations/system/test_migration_index_too_high.py b/tests/migrations/system/test_migration_index_too_high.py index 5cc33c2f..26c343a8 100644 --- a/tests/migrations/system/test_migration_index_too_high.py +++ b/tests/migrations/system/test_migration_index_too_high.py @@ -7,8 +7,11 @@ from ..util import get_noop_migration -def test_migration_index_too_high_migrate(migration_handler, write, query_single_value): +def test_migration_index_too_high_migrate( + migration_handler, write, set_migration_index_to_1, query_single_value +): write({"type": "create", "fqid": "a/1", "fields": {}}) + set_migration_index_to_1() migration_handler.register_migrations(get_noop_migration(2), get_noop_migration(3)) migration_handler.migrate() @@ -25,9 +28,10 @@ def test_migration_index_too_high_migrate(migration_handler, write, query_single def test_migration_index_too_high_finalize( - migration_handler, write, query_single_value + migration_handler, write, set_migration_index_to_1, query_single_value ): write({"type": "create", "fqid": "a/1", "fields": {}}) + set_migration_index_to_1() migration_handler.register_migrations(get_noop_migration(2), get_noop_migration(3)) migration_handler.finalize() @@ -43,8 +47,11 @@ def test_migration_index_too_high_finalize( rm.assert_not_called() -def test_migration_index_too_high_reset(migration_handler, write, query_single_value): +def test_migration_index_too_high_reset( + migration_handler, write, set_migration_index_to_1, query_single_value +): write({"type": "create", "fqid": "a/1", "fields": {}}) + set_migration_index_to_1() migration_handler.register_migrations(get_noop_migration(2), get_noop_migration(3)) migration_handler.finalize() diff --git a/tests/migrations/system/test_migration_keyframes.py b/tests/migrations/system/test_migration_keyframes.py index 1dec40b6..a19baa91 100644 --- a/tests/migrations/system/test_migration_keyframes.py +++ b/tests/migrations/system/test_migration_keyframes.py @@ -19,9 +19,10 @@ def _write(self, write, *data): raise NotImplementedError() @pytest.fixture() - def write_data(self, write): + def write_data(self, write, set_migration_index_to_1): def _write_data(*data): self._write(write, *data) + set_migration_index_to_1() yield _write_data diff --git a/tests/migrations/system/test_multiple_migrations_and_positions.py b/tests/migrations/system/test_multiple_migrations_and_positions.py index cd38d76d..489c7188 100644 --- a/tests/migrations/system/test_multiple_migrations_and_positions.py +++ b/tests/migrations/system/test_multiple_migrations_and_positions.py @@ -6,11 +6,13 @@ def test_multiple_migrations_together( - migration_handler, write, read_model, assert_model, assert_finalized + migration_handler, + write, + set_migration_index_to_1, + read_model, + assert_model, + assert_finalized, ): - migration_handler.register_migrations( - get_noop_migration(2), get_noop_migration(3), get_noop_migration(4) - ) write( {"type": "create", "fqid": "a/1", "fields": {"f": 1, "g": 1, "h": [], "i": [1]}} ) @@ -24,7 +26,12 @@ def test_multiple_migrations_together( ) write({"type": "delete", "fqid": "a/1"}) write({"type": "restore", "fqid": "a/1"}) + set_migration_index_to_1() previous_model = read_model("a/1") + + migration_handler.register_migrations( + get_noop_migration(2), get_noop_migration(3), get_noop_migration(4) + ) migration_handler.finalize() assert_model("a/1", previous_model) @@ -32,10 +39,16 @@ def test_multiple_migrations_together( def test_second_position_access_old_and_new_data( - migration_handler, write, read_model, assert_model, assert_finalized + migration_handler, + write, + set_migration_index_to_1, + read_model, + assert_model, + assert_finalized, ): write({"type": "create", "fqid": "a/1", "fields": {"f": 1}}) write({"type": "create", "fqid": "trigger/1", "fields": {}}) + set_migration_index_to_1() class TestMigration(BaseMigration): target_migration_index = 2 @@ -63,8 +76,16 @@ def migrate_event( def test_second_migration_gets_events_from_first( - migration_handler, write, read_model, assert_model, assert_finalized + migration_handler, + write, + set_migration_index_to_1, + read_model, + assert_model, + assert_finalized, ): + write({"type": "create", "fqid": "a/1", "fields": {}}) + set_migration_index_to_1() + first = get_lambda_migration(lambda _: [CreateEvent("a/2", {})]) captured_event = ( @@ -78,7 +99,6 @@ def capture_handler(event): second = get_lambda_migration(capture_handler, target_migration_index=3) migration_handler.register_migrations(first, second) - write({"type": "create", "fqid": "a/1", "fields": {}}) migration_handler.finalize() assert_finalized() @@ -86,22 +106,29 @@ def capture_handler(event): assert captured_event["event"].fqid == "a/2" -def test_amount_events(migration_handler, write, assert_count): - migration_handler.register_migrations(get_noop_migration(2), get_noop_migration(3)) +def test_amount_events( + migration_handler, write, set_migration_index_to_1, assert_count +): write( {"type": "create", "fqid": "a/1", "fields": {"f": 1, "g": 1, "h": [], "i": [1]}} ) + set_migration_index_to_1() + + migration_handler.register_migrations(get_noop_migration(2), get_noop_migration(3)) migration_handler.finalize() + assert_count("events", 1) assert_count("migration_events", 0) def test_migrate_finalize( - migration_handler, write, read_model, assert_model, assert_finalized + migration_handler, + write, + set_migration_index_to_1, + read_model, + assert_model, + assert_finalized, ): - migration_handler.register_migrations( - get_noop_migration(2), get_noop_migration(3), get_noop_migration(4) - ) write( {"type": "create", "fqid": "a/1", "fields": {"f": 1, "g": 1, "h": [], "i": [1]}} ) @@ -115,7 +142,12 @@ def test_migrate_finalize( ) write({"type": "delete", "fqid": "a/1"}) write({"type": "restore", "fqid": "a/1"}) + set_migration_index_to_1() previous_model = read_model("a/1") + + migration_handler.register_migrations( + get_noop_migration(2), get_noop_migration(3), get_noop_migration(4) + ) migration_handler.migrate() migration_handler.finalize() @@ -124,9 +156,13 @@ def test_migrate_finalize( def test_multiple_migrations_following( - migration_handler, write, read_model, assert_model, assert_finalized + migration_handler, + write, + set_migration_index_to_1, + read_model, + assert_model, + assert_finalized, ): - migration_handler.register_migrations(get_noop_migration(2), get_noop_migration(3)) write( {"type": "create", "fqid": "a/1", "fields": {"f": 1, "g": 1, "h": [], "i": [1]}} ) @@ -140,8 +176,10 @@ def test_multiple_migrations_following( ) write({"type": "delete", "fqid": "a/1"}) write({"type": "restore", "fqid": "a/1"}) + set_migration_index_to_1() previous_model = read_model("a/1") + migration_handler.register_migrations(get_noop_migration(2), get_noop_migration(3)) migration_handler.finalize() assert_model("a/1", previous_model) @@ -161,14 +199,20 @@ def test_multiple_migrations_following( def test_multiple_migrations_one_finalizing( - migration_handler, write, read_model, assert_model, assert_finalized + migration_handler, + write, + set_migration_index_to_1, + read_model, + assert_model, + assert_finalized, ): """This tests the deletion of the keyframe in move_to_next_position""" - migration_handler.register_migrations(get_noop_migration(2)) write({"type": "create", "fqid": "a/1", "fields": {"f": 1}}) write({"type": "update", "fqid": "a/1", "fields": {"f": 2}}) + set_migration_index_to_1() previous_model = read_model("a/1") + migration_handler.register_migrations(get_noop_migration(2)) migration_handler.migrate() migration_handler.migrations_by_target_migration_index = {} migration_handler.register_migrations(get_noop_migration(2), get_noop_migration(3)) diff --git a/tests/migrations/system/test_premade_migration_classes.py b/tests/migrations/system/test_premade_migration_classes.py index 061c51d8..e099d35b 100644 --- a/tests/migrations/system/test_premade_migration_classes.py +++ b/tests/migrations/system/test_premade_migration_classes.py @@ -2,7 +2,12 @@ def test_rename_field( - migration_handler, write, assert_model, query_single_value, assert_finalized + migration_handler, + write, + set_migration_index_to_1, + assert_model, + query_single_value, + assert_finalized, ): """f -> f_new""" write({"type": "create", "fqid": "a/1", "fields": {"f": [1]}}) @@ -11,6 +16,7 @@ def test_rename_field( write({"type": "update", "fqid": "a/1", "list_fields": {"add": {"f": [3]}}}) write({"type": "update", "fqid": "a/1", "fields": {"f": None}}) write({"type": "update", "fqid": "a/1", "fields": {"f": "Hello"}}) + set_migration_index_to_1() class RenameField(RenameFieldMigration): target_migration_index = 2 @@ -41,11 +47,17 @@ class RenameField(RenameFieldMigration): def test_add_field_with_default( - migration_handler, write, assert_model, query_single_value, assert_finalized + migration_handler, + write, + set_migration_index_to_1, + assert_model, + query_single_value, + assert_finalized, ): write({"type": "create", "fqid": "a/1", "fields": {"f": 3}}) write({"type": "create", "fqid": "b/1", "fields": {"x": 42}}) write({"type": "update", "fqid": "a/1", "fields": {"f": 5, "g": 127}}) + set_migration_index_to_1() class AddField(AddFieldMigration): target_migration_index = 2 diff --git a/tests/migrations/system/test_reset_migration.py b/tests/migrations/system/test_reset_migration.py index 6576c0f1..4a9c4902 100644 --- a/tests/migrations/system/test_reset_migration.py +++ b/tests/migrations/system/test_reset_migration.py @@ -3,9 +3,12 @@ from ..util import get_lambda_migration, get_noop_migration -def test_no_migrations(migration_handler, write, assert_count): +def test_no_migrations( + migration_handler, write, set_migration_index_to_1, assert_count +): write({"type": "create", "fqid": "a/1", "fields": {}}) write({"type": "create", "fqid": "a/2", "fields": {}}) + set_migration_index_to_1() assert_count("migration_keyframes", 0) assert_count("migration_positions", 0) @@ -19,9 +22,12 @@ def test_no_migrations(migration_handler, write, assert_count): assert_count("migration_events", 0) -def test_ongoing_migrations(migration_handler, write, assert_count): +def test_ongoing_migrations( + migration_handler, write, set_migration_index_to_1, assert_count +): write({"type": "create", "fqid": "a/1", "fields": {}}) write({"type": "create", "fqid": "a/2", "fields": {}}) + set_migration_index_to_1() assert_count("migration_keyframes", 0) assert_count("migration_positions", 0) @@ -42,9 +48,15 @@ def test_ongoing_migrations(migration_handler, write, assert_count): def test_actual_migration( - migration_handler, write, read_model, assert_model, query_single_value + migration_handler, + write, + set_migration_index_to_1, + read_model, + assert_model, + query_single_value, ): write({"type": "create", "fqid": "a/1", "fields": {}}) + set_migration_index_to_1() previous_model = read_model("a/1") event = CreateEvent("a/2", {}) diff --git a/tests/migrations/system/test_sample_migrations.py b/tests/migrations/system/test_sample_migrations.py index cbf195da..4c3c9ed9 100644 --- a/tests/migrations/system/test_sample_migrations.py +++ b/tests/migrations/system/test_sample_migrations.py @@ -19,6 +19,7 @@ def test_move_id( migration_handler, write, + set_migration_index_to_1, assert_model, exists_model, query_single_value, @@ -38,6 +39,7 @@ def test_move_id( write({"type": "update", "fqid": "a/1", "fields": {"f": None}}) write({"type": "delete", "fqid": "a/1"}) write({"type": "restore", "fqid": "a/1"}) + set_migration_index_to_1() class MoveId(BaseMigration): target_migration_index = 2 @@ -75,7 +77,12 @@ def migrate_event( def test_remove_field( - migration_handler, write, assert_model, query_single_value, assert_finalized + migration_handler, + write, + set_migration_index_to_1, + assert_model, + query_single_value, + assert_finalized, ): """remove f""" write({"type": "create", "fqid": "a/1", "fields": {"f": [1]}}) @@ -83,6 +90,7 @@ def test_remove_field( write({"type": "update", "fqid": "a/1", "list_fields": {"add": {"f": [3]}}}) write({"type": "update", "fqid": "a/1", "fields": {"f": None}}) write({"type": "update", "fqid": "a/1", "fields": {"f": "Hello"}}) + set_migration_index_to_1() class RemoveField(BaseMigration): target_migration_index = 2 @@ -116,10 +124,16 @@ def migrate_event( def test_add_required_field_based_on_migrated_data( - migration_handler, write, assert_model, query_single_value, assert_finalized + migration_handler, + write, + set_migration_index_to_1, + assert_model, + query_single_value, + assert_finalized, ): """First, rename f->f_new. Second migration adds `g`, which is f_new*2""" write({"type": "create", "fqid": "a/1", "fields": {"f": 3}}) + set_migration_index_to_1() class RenameField(RenameFieldMigration): target_migration_index = 2 @@ -146,11 +160,17 @@ def migrate_event( def test_create_additional_model( - migration_handler, write, assert_model, query_single_value, assert_finalized + migration_handler, + write, + set_migration_index_to_1, + assert_model, + query_single_value, + assert_finalized, ): """Also a setup-for-tests scenario here.""" write({"type": "create", "fqid": "config/1", "fields": {"create_b": True}}) write({"type": "create", "fqid": "a/1", "fields": {}}) + set_migration_index_to_1() class CreateModel(BaseMigration): target_migration_index = 2 @@ -173,7 +193,12 @@ def migrate_event( def test_access_field_after_rename( - migration_handler, write, assert_model, query_single_value, assert_finalized + migration_handler, + write, + set_migration_index_to_1, + assert_model, + query_single_value, + assert_finalized, ): """First rename f->f_new. In a second migration access both fields via both accessors""" write({"type": "create", "fqid": "a/1", "fields": {"f": [1]}}) @@ -182,6 +207,7 @@ def test_access_field_after_rename( write({"type": "update", "fqid": "a/1", "fields": {"f": "Hello"}}) write({"type": "delete", "fqid": "a/1"}) write({"type": "restore", "fqid": "a/1"}) + set_migration_index_to_1() class RenameField(RenameFieldMigration): target_migration_index = 2 diff --git a/tests/migrations/system/test_wrong_new_events.py b/tests/migrations/system/test_wrong_new_events.py index b22a2ff2..5144f673 100644 --- a/tests/migrations/system/test_wrong_new_events.py +++ b/tests/migrations/system/test_wrong_new_events.py @@ -22,10 +22,11 @@ def test_to_event_unknown_event(): class TestCreateNewEvent: @pytest.fixture() - def execute(self, migration_handler, write): - write({"type": "create", "fqid": "a/1", "fields": {"f": 1}}) - + def execute(self, migration_handler, write, set_migration_index_to_1): def _execute(event_fn, match): + write({"type": "create", "fqid": "a/1", "fields": {"f": 1}}) + set_migration_index_to_1() + migration_handler.register_migrations( get_lambda_migration(lambda _: [event_fn()]) ) @@ -64,9 +65,10 @@ def test_double_create(self, write, execute): class TestUpdate: @pytest.fixture() - def execute(self, migration_handler, write): + def execute(self, migration_handler, write, set_migration_index_to_1): write({"type": "create", "fqid": "a/1", "fields": {"f": 1}}) write({"type": "update", "fqid": "a/1", "fields": {"f": 2}}) + set_migration_index_to_1() def _execute(event_fn, match): migration_handler.register_migrations( @@ -97,20 +99,26 @@ def test_not_a_field(self, execute): def test_empty_field(self, execute): execute(lambda: UpdateEvent("a/1", {"empty": None}), match="empty") - def test_update_without_create(self, migration_handler, write): + def test_update_without_create( + self, migration_handler, write, set_migration_index_to_1 + ): write({"type": "create", "fqid": "a/1", "fields": {"f": 1}}) + set_migration_index_to_1() migration_handler.register_migrations( get_lambda_migration(lambda _: [UpdateEvent("a/1", {"f": 2})]) ) with pytest.raises(BadEventException, match="Model a/1 does not exist"): migration_handler.migrate() - def test_update_after_delete(self, migration_handler, write): + def test_update_after_delete( + self, migration_handler, write, set_migration_index_to_1 + ): write({"type": "create", "fqid": "a/1", "fields": {"f": 1}}) write({"type": "delete", "fqid": "a/1"}) write( {"type": "create", "fqid": "a/2", "fields": {"f": 1}} ) # this will be overwritten + set_migration_index_to_1() migration_handler.register_migrations( get_lambda_migration( lambda e: [UpdateEvent("a/1", {"f": 2})] if e.fqid == "a/2" else [e] @@ -122,11 +130,12 @@ def test_update_after_delete(self, migration_handler, write): class TestDeleteFields: @pytest.fixture() - def execute(self, migration_handler, write): + def execute(self, migration_handler, write, set_migration_index_to_1): write({"type": "create", "fqid": "a/1", "fields": {"f": 1}}) write( {"type": "update", "fqid": "a/1", "fields": {"f": None}} ) # will be converted to a deletefields event + set_migration_index_to_1() def _execute(event_fn, match): migration_handler.register_migrations( @@ -154,20 +163,26 @@ def test_not_a_field(self, execute): "not_%_a_field", ) - def test_deletefields_without_create(self, migration_handler, write): + def test_deletefields_without_create( + self, migration_handler, write, set_migration_index_to_1 + ): write({"type": "create", "fqid": "a/1", "fields": {"f": 1}}) + set_migration_index_to_1() migration_handler.register_migrations( get_lambda_migration(lambda _: [DeleteFieldsEvent("a/1", ["f"])]) ) with pytest.raises(BadEventException, match="Model a/1 does not exist"): migration_handler.migrate() - def test_deletefields_after_delete(self, migration_handler, write): + def test_deletefields_after_delete( + self, migration_handler, write, set_migration_index_to_1 + ): write({"type": "create", "fqid": "a/1", "fields": {"f": 1}}) write({"type": "delete", "fqid": "a/1"}) write( {"type": "create", "fqid": "a/2", "fields": {"f": 1}} ) # this will be overwritten + set_migration_index_to_1() migration_handler.register_migrations( get_lambda_migration( lambda e: [DeleteFieldsEvent("a/1", ["f"])] if e.fqid == "a/2" else [e] @@ -179,11 +194,12 @@ def test_deletefields_after_delete(self, migration_handler, write): class TestListUpdate: @pytest.fixture() - def execute(self, migration_handler, write): + def execute(self, migration_handler, write, set_migration_index_to_1): write({"type": "create", "fqid": "a/1", "fields": {"f": [1]}}) write( {"type": "update", "fqid": "a/1", "list_fields": {"add": {"f": [2]}}} ) # will be converted to a listfields event + set_migration_index_to_1() def _execute(event_fn, match): migration_handler.register_migrations( @@ -217,8 +233,11 @@ def test_additional_key(self, execute): "Only add and remove is allowed", ) - def test_listfields_without_create(self, migration_handler, write): + def test_listfields_without_create( + self, migration_handler, write, set_migration_index_to_1 + ): write({"type": "create", "fqid": "a/1", "fields": {"f": 1}}) + set_migration_index_to_1() migration_handler.register_migrations( get_lambda_migration( lambda _: [ListUpdateEvent("a/1", {"add": {"f": [2]}})] @@ -227,12 +246,15 @@ def test_listfields_without_create(self, migration_handler, write): with pytest.raises(BadEventException, match="Model a/1 does not exist"): migration_handler.migrate() - def test_listfields_after_delete(self, migration_handler, write): + def test_listfields_after_delete( + self, migration_handler, write, set_migration_index_to_1 + ): write({"type": "create", "fqid": "a/1", "fields": {"f": 1}}) write({"type": "delete", "fqid": "a/1"}) write( {"type": "create", "fqid": "a/2", "fields": {"f": 1}} ) # this will be overwritten + set_migration_index_to_1() migration_handler.register_migrations( get_lambda_migration( lambda e: [ListUpdateEvent("a/1", {"add": {"f": [2]}})] @@ -246,11 +268,12 @@ def test_listfields_after_delete(self, migration_handler, write): class TestListUpdateModify: @pytest.fixture() - def execute(self, migration_handler, write): + def execute(self, migration_handler, write, set_migration_index_to_1): write({"type": "create", "fqid": "a/1", "fields": {"f": [1]}}) write( {"type": "update", "fqid": "a/1", "list_fields": {"add": {"f": [2]}}} ) # will be converted to a listfields event + set_migration_index_to_1() def _execute(event_fn, match): migration_handler.register_migrations( @@ -286,9 +309,10 @@ def handle(event): class TestDelete: - def test_fqid(self, migration_handler, write): + def test_fqid(self, migration_handler, write, set_migration_index_to_1): write({"type": "create", "fqid": "a/1", "fields": {"f": [1]}}) write({"type": "delete", "fqid": "a/1"}) + set_migration_index_to_1() migration_handler.register_migrations( get_lambda_migration( lambda e: [DeleteEvent("xyz")] if e.type == "delete" else [e] @@ -297,18 +321,22 @@ def test_fqid(self, migration_handler, write): with pytest.raises(BadEventException, match="xyz"): migration_handler.migrate() - def test_delete_without_create(self, migration_handler, write): + def test_delete_without_create( + self, migration_handler, write, set_migration_index_to_1 + ): write({"type": "create", "fqid": "a/1", "fields": {"f": 1}}) + set_migration_index_to_1() migration_handler.register_migrations( get_lambda_migration(lambda _: [DeleteEvent("a/1")]) ) with pytest.raises(BadEventException, match="Model a/1 does not exist"): migration_handler.migrate() - def test_double_delete(self, migration_handler, write): + def test_double_delete(self, migration_handler, write, set_migration_index_to_1): write({"type": "create", "fqid": "a/1", "fields": {"f": 1}}) write({"type": "create", "fqid": "a/2", "fields": {"f": 1}}) write({"type": "update", "fqid": "a/2", "fields": {"f": 1}}) + set_migration_index_to_1() migration_handler.register_migrations( get_lambda_migration( lambda e: [DeleteEvent("a/1")] if e.fqid == "a/2" else [e] @@ -319,10 +347,11 @@ def test_double_delete(self, migration_handler, write): class TestRestore: - def test_fqid(self, migration_handler, write): + def test_fqid(self, migration_handler, write, set_migration_index_to_1): write({"type": "create", "fqid": "a/1", "fields": {"f": [1]}}) write({"type": "delete", "fqid": "a/1"}) write({"type": "restore", "fqid": "a/1"}) + set_migration_index_to_1() migration_handler.register_migrations( get_lambda_migration( lambda e: [RestoreEvent("xyz")] if e.type == "restore" else [e] @@ -331,17 +360,23 @@ def test_fqid(self, migration_handler, write): with pytest.raises(BadEventException, match="xyz"): migration_handler.migrate() - def test_restore_without_create(self, migration_handler, write): + def test_restore_without_create( + self, migration_handler, write, set_migration_index_to_1 + ): write({"type": "create", "fqid": "a/1", "fields": {"f": 1}}) + set_migration_index_to_1() migration_handler.register_migrations( get_lambda_migration(lambda _: [RestoreEvent("a/1")]) ) with pytest.raises(BadEventException, match="Model a/1 does not exist"): migration_handler.migrate() - def test_restore_without_delete(self, migration_handler, write): + def test_restore_without_delete( + self, migration_handler, write, set_migration_index_to_1 + ): write({"type": "create", "fqid": "a/1", "fields": {"f": 1}}) write({"type": "create", "fqid": "a/2", "fields": {"f": 1}}) + set_migration_index_to_1() migration_handler.register_migrations( get_lambda_migration( lambda e: [RestoreEvent("a/1")] if e.fqid == "a/2" else [e] diff --git a/tests/shared/unit/postgresql_backend/test_sql_read_database_backend_service.py b/tests/shared/unit/postgresql_backend/test_sql_read_database_backend_service.py index 77a11ded..9fb813d9 100644 --- a/tests/shared/unit/postgresql_backend/test_sql_read_database_backend_service.py +++ b/tests/shared/unit/postgresql_backend/test_sql_read_database_backend_service.py @@ -358,10 +358,10 @@ def test_get_current_migration_index_cached( read_database: ReadDatabase, connection: ConnectionHandler ): connection.query_single_value = qsv = MagicMock(return_value=None) - assert read_database.get_current_migration_index() == 1 + assert read_database.get_current_migration_index() == -1 qsv.assert_called_once() # second try; now it should be cached connection.query_single_value = qsv = MagicMock(return_value=None) - assert read_database.get_current_migration_index() == 1 + assert read_database.get_current_migration_index() == -1 qsv.assert_not_called() diff --git a/tests/writer/system/write/test_migration_index.py b/tests/writer/system/write/test_migration_index.py index 1d856173..6e7b8c3c 100644 --- a/tests/writer/system/write/test_migration_index.py +++ b/tests/writer/system/write/test_migration_index.py @@ -34,7 +34,7 @@ def test_initial_migration_index( db_cur.execute("select migration_index from positions where position=%s", [1]) migration_index = db_cur.fetchone()[0] - assert migration_index == 1 + assert migration_index == -1 def test_use_current_migration_index( @@ -106,4 +106,4 @@ def test_send_migration_index_not_empty( db_cur.execute("select migration_index from positions where position=%s", [1]) migration_index = db_cur.fetchone()[0] - assert migration_index == 1 + assert migration_index == -1