BackendIssue1031 Implement RemoveFieldMigration (#160)

Co-authored-by: Ralf Peschke <rpeschke@peschke-it.de>

datastore/migrations/__init__.py

@@ -24,4 +24,5 @@
 )
 from .core.setup import setup  # noqa
 from .migrations.add_field_migration import AddFieldMigration  # noqa
+from .migrations.remove_field_migration import RemoveFieldMigration  # noqa
 from .migrations.rename_field_migration import RenameFieldMigration  # noqa

datastore/migrations/migrations/remove_field_migration.py

@@ -0,0 +1,46 @@
+from typing import Any, Dict, List, Optional
+
+from datastore.shared.util import collection_from_fqid
+
+from .. import (
+    BaseEvent,
+    BaseMigration,
+    CreateEvent,
+    DeleteFieldsEvent,
+    ListUpdateEvent,
+    UpdateEvent,
+)
+
+
+class RemoveFieldMigration(BaseMigration):
+    """
+    This migration removes a field from all events for one collection.
+    """
+
+    collection: str
+    field: str
+
+    def remove_field(self, object: Dict[str, Any]) -> None:
+        if self.field in object:
+            del object[self.field]
+
+    def migrate_event(
+        self,
+        event: BaseEvent,
+    ) -> Optional[List[BaseEvent]]:
+        collection = collection_from_fqid(event.fqid)
+        if collection != self.collection:
+            return None
+
+        if isinstance(event, CreateEvent) or isinstance(event, UpdateEvent):
+            self.remove_field(event.data)
+
+        elif isinstance(event, DeleteFieldsEvent):
+            if self.field in event.data:
+                event.data.remove(self.field)
+
+        elif isinstance(event, ListUpdateEvent):
+            self.remove_field(event.add)
+            self.remove_field(event.remove)
+
+        return [event]

tests/migrations/system/test_premade_migration_classes.py

@@ -1,4 +1,8 @@
-from datastore.migrations import AddFieldMigration, RenameFieldMigration
+from datastore.migrations import (
+    AddFieldMigration,
+    RemoveFieldMigration,
+    RenameFieldMigration,
+)
 
 
 def test_rename_field(
@@ -80,3 +84,54 @@ class AddField(AddFieldMigration):
     assert_model(
         "a/1", {"f": 5, "g": 127, "meta_deleted": False, "meta_position": 3}, position=3
     )
+
+
+def test_remove_field(
+    migration_handler,
+    write,
+    set_migration_index_to_1,
+    assert_model,
+    query_single_value,
+    assert_finalized,
+):
+    write({"type": "create", "fqid": "a/1", "fields": {"a": 5, "r": [3]}})
+    write({"type": "update", "fqid": "a/1", "fields": {"a": 6, "r": [20]}})
+    write({"type": "update", "fqid": "a/1", "fields": {"a": 6, "r": [20]}})
+    write(
+        {
+            "type": "update",
+            "fqid": "a/1",
+            "list_fields": {"add": {"r": [3]}, "remove": {"r": [20]}},
+        }
+    )
+    write({"type": "delete", "fqid": "a/1"})
+    write({"type": "restore", "fqid": "a/1"})
+    # Test update-event ("a":7) and deletefields/deleteFieldsEvent-event by setting "r" to None
+    write({"type": "update", "fqid": "a/1", "fields": {"a": 7, "r": None}})
+    write({"type": "create", "fqid": "b/1", "fields": {"a": 5, "r": [3]}})
+
+    set_migration_index_to_1()
+
+    class RemoveField(RemoveFieldMigration):
+        target_migration_index = 2
+        collection = "a"
+        field = "r"
+
+    migration_handler.register_migrations(RemoveField)
+    migration_handler.finalize()
+
+    assert_finalized()
+    assert_model(
+        "a/1",
+        {"a": 5, "meta_deleted": False, "meta_position": 1},
+        position=1,
+    )
+    assert_model("a/1", {"a": 6, "meta_deleted": False, "meta_position": 2}, position=2)
+    assert_model("a/1", {"a": 6, "meta_deleted": False, "meta_position": 3}, position=3)
+    assert_model("a/1", {"a": 6, "meta_deleted": False, "meta_position": 4}, position=4)
+    assert_model("a/1", {"a": 6, "meta_deleted": True, "meta_position": 5}, position=5)
+    assert_model("a/1", {"a": 6, "meta_deleted": False, "meta_position": 6}, position=6)
+    assert_model("a/1", {"a": 7, "meta_deleted": False, "meta_position": 7}, position=7)
+    assert_model(
+        "b/1", {"a": 5, "r": [3], "meta_deleted": False, "meta_position": 8}, position=8
+    )

tests/writer/unit/core/test_writer_service.py

@@ -124,7 +124,7 @@ def wait_for_lock(*args, **kwargs):
     assert not writer.locks[1].locked()
 
     writer.locks[0].release()
-    thread1.join(0.05)
-    thread2.join(0.05)
+    thread1.join(0.10)
+    thread2.join(0.10)
     assert not thread1.is_alive()
     assert not thread2.is_alive()