Extended datastore adapter (#1013)

* Split additional model usage into own class

* Implement Min/Max/Filter. Replace get with the old fetch_model.

* Add meeting id to add_to_preview filter

* Full rework of the extended datastore adapter

* Fix bugs & tests

* Fix last tests & locked fields error

* Automatically update changed_models. Add WeightMixin

* Fix tests. Calculate relation fields automatically

cli/generate_permissions.py

@@ -26,9 +26,9 @@
     """\
     # Code generated. DO NOT EDIT.
 
-    from enum import Enum
     from typing import Dict, List
 
+    from openslides_backend.shared.enum import Enum
     from .base_classes import Permission
 
     PERMISSION_YML_CHECKSUM = "{}"

openslides_backend/action/action.py

@@ -38,6 +38,7 @@
 from ..shared.interfaces.services import Services
 from ..shared.interfaces.write_request import WriteRequest
 from ..shared.patterns import Collection, FullQualifiedId, transform_to_fqids
+from ..shared.typing import DeletedModel
 from .relations.relation_manager import RelationManager, RelationUpdates
 from .relations.typing import FieldUpdateElement, ListUpdateElement
 from .util.action_type import ActionType
@@ -90,6 +91,7 @@ class Action(BaseAction, metaclass=SchemaProvider):
     permission: Optional[Union[Permission, OrganizationManagementLevel]] = None
     permission_model: Optional[Model] = None
     permission_id: Optional[str] = None
+    skip_archived_meeting_check: bool = False
     relation_manager: RelationManager
 
     write_requests: List[WriteRequest]
@@ -101,7 +103,7 @@ def __init__(
         datastore: DatastoreService,
         relation_manager: RelationManager,
         logging: LoggingModule,
-        skip_archived_meeting_check: bool = False,
+        skip_archived_meeting_check: Optional[bool] = None,
     ) -> None:
         self.services = services
         self.auth = services.authentication()
@@ -111,11 +113,7 @@ def __init__(
         self.relation_manager = relation_manager
         self.logging = logging
         self.logger = logging.getLogger(__name__)
-        if hasattr(self.__class__, "skip_archived_meeting_check"):
-            self.skip_archived_meeting_check: bool = (
-                self.__class__.skip_archived_meeting_check
-            )
-        else:
+        if skip_archived_meeting_check is not None:
             self.skip_archived_meeting_check = skip_archived_meeting_check
         self.write_requests = []
         self.results = []
@@ -219,7 +217,7 @@ def check_for_archived_meeting(self, instance: Dict[str, Any]) -> None:
             )
 
         fqid = FullQualifiedId(Collection("meeting"), meeting_id)
-        meeting = self.datastore.fetch_model(
+        meeting = self.datastore.get(
             fqid,
             ["is_active_in_organization_id", "name"],
         )
@@ -249,10 +247,9 @@ def get_meeting_id(self, instance: Dict[str, Any]) -> int:
             identifier = "id"
             if self.permission_id:
                 identifier = self.permission_id
-            db_instance = self.datastore.fetch_model(
+            db_instance = self.datastore.get(
                 FullQualifiedId(model.collection, instance[identifier]),
                 ["meeting_id"],
-                exception=True,
             )
             return db_instance["meeting_id"]
 
@@ -359,7 +356,7 @@ def build_write_request(
         )
         if fields:
             event["fields"] = fields
-            self.datastore.update_additional_models(fqid, fields)
+            self.datastore.apply_changed_model(fqid, fields)
         if list_fields:
             event["list_fields"] = list_fields
         return WriteRequest(
@@ -397,12 +394,23 @@ def process_write_requests(
             # sort events: create - update - delete
             events_by_type: Dict[EventType, List[Event]] = defaultdict(list)
             for event in write_request.events:
+                self.apply_event(event)
                 events_by_type[event["type"]].append(event)
             write_request.events = []
             for event_type in (EventType.Create, EventType.Update, EventType.Delete):
                 write_request.events.extend(events_by_type[event_type])
         return write_request
 
+    def apply_event(self, event: Event) -> None:
+        """
+        Applies the given event to the changed_models in the datastore.
+        """
+        if event["type"] in (EventType.Create, EventType.Update):
+            if fields := event.get("fields"):
+                self.datastore.apply_changed_model(event["fqid"], fields)
+        elif event["type"] == EventType.Delete:
+            self.datastore.apply_changed_model(event["fqid"], DeletedModel())
+
     def validate_required_fields(self, write_request: WriteRequest) -> None:
         """
         Validate required fields with the events of one WriteRequest.
@@ -479,7 +487,7 @@ def validate_relation_fields(self, instance: Dict[str, Any]) -> None:
             for equal_field in field.equal_fields:
                 if not (own_equal_field_value := instance.get(equal_field)):
                     fqid = FullQualifiedId(self.model.collection, instance["id"])
-                    db_instance = self.datastore.fetch_model(
+                    db_instance = self.datastore.get(
                         fqid,
                         [equal_field],
                     )
@@ -497,7 +505,7 @@ def validate_relation_fields(self, instance: Dict[str, Any]) -> None:
                         )
                     else:
                         for fqid in fqids:
-                            related_instance = self.datastore.fetch_model(
+                            related_instance = self.datastore.get(
                                 fqid,
                                 [equal_field],
                             )
@@ -533,7 +541,7 @@ def apply_instance(
     ) -> None:
         if not fqid:
             fqid = FullQualifiedId(self.model.collection, instance["id"])
-        self.datastore.update_additional_models(fqid, instance)
+        self.datastore.apply_changed_model(fqid, instance)
 
     def execute_other_action(
         self,
@@ -551,8 +559,8 @@ def execute_other_action(
         to the called class if set. Usually this is needed for cascading deletes from
         outside of meeting.
         """
-        if hasattr(self.__class__, "skip_archived_meeting_check"):
-            skip_archived_meeting_check = self.__class__.skip_archived_meeting_check
+        if self.skip_archived_meeting_check:
+            skip_archived_meeting_check = self.skip_archived_meeting_check
 
         action = ActionClass(
             self.services,

openslides_backend/action/action_handler.py

@@ -228,7 +228,7 @@ def perform_action(
 
                 # add locked_fields to request
                 write_request.locked_fields = self.datastore.locked_fields
-                # reset locked fields, but not addtional relation models - these might be needed
+                # reset locked fields, but not changed models - these might be needed
                 # by another action
                 self.datastore.locked_fields = {}
 

openslides_backend/action/actions/agenda_item/create.py

@@ -40,13 +40,13 @@ def update_instance(self, instance: Dict[str, Any]) -> Dict[str, Any]:
         instance = super().update_instance(instance)
         if instance.get("parent_id") is None:
             return instance
-        parent = self.datastore.fetch_model(
+        parent = self.datastore.get(
             FullQualifiedId(Collection("agenda_item"), instance["parent_id"]),
             ["child_ids"],
         )
         max_weight = 0
         for child_id in parent.get("child_ids", []):
-            child = self.datastore.fetch_model(
+            child = self.datastore.get(
                 FullQualifiedId(Collection("agenda_item"), child_id),
                 ["weight"],
             )

openslides_backend/action/actions/agenda_item/delete.py

@@ -28,9 +28,9 @@ def update_instance(self, instance: Dict[str, Any]) -> Dict[str, Any]:
         )
         if agenda_item.get("content_object_id"):
             content_object_fqid = string_to_fqid(agenda_item["content_object_id"])
-            if content_object_fqid.collection.collection == "topic" and not isinstance(
-                self.datastore.additional_relation_models.get(content_object_fqid),
-                DeletedModel,
+            if (
+                content_object_fqid.collection.collection == "topic"
+                and not self.datastore.is_deleted(content_object_fqid)
             ):
                 self.apply_instance(DeletedModel(), fqid)
                 self.execute_other_action(

openslides_backend/action/actions/assignment_candidate/sort.py

@@ -1,5 +1,6 @@
 from ....models.models import Assignment, AssignmentCandidate
 from ....permissions.permissions import Permissions
+from ....shared.filters import FilterOperator
 from ...generics.update import UpdateAction
 from ...mixins.linear_sort_mixin import LinearSortMixin
 from ...mixins.singular_action_mixin import SingularActionMixin
@@ -28,7 +29,6 @@ def get_updated_instances(self, action_data: ActionData) -> ActionData:
         # Action data is an iterable with exactly one item
         instance = next(iter(action_data))
         yield from self.sort_linear(
-            nodes=instance["candidate_ids"],
-            filter_id=instance["assignment_id"],
-            filter_str="assignment_id",
+            instance["candidate_ids"],
+            FilterOperator("assignment_id", "=", instance["assignment_id"]),
         )

openslides_backend/action/actions/chat_group/create.py

@@ -2,16 +2,17 @@
 
 from ....models.models import ChatGroup
 from ....permissions.permissions import Permissions
-from ....shared.filters import FilterOperator
-from ....shared.patterns import Collection
 from ...generics.create import CreateAction
+from ...mixins.weight_mixin import WeightMixin
 from ...util.default_schema import DefaultSchema
 from ...util.register import register_action
 from .mixins import ChatEnabledMixin, CheckUniqueNameMixin
 
 
 @register_action("chat_group.create")
-class ChatGroupCreate(ChatEnabledMixin, CheckUniqueNameMixin, CreateAction):
+class ChatGroupCreate(
+    WeightMixin, ChatEnabledMixin, CheckUniqueNameMixin, CreateAction
+):
     """
     Action to create a chat group.
     """
@@ -28,10 +29,3 @@ def update_instance(self, instance: Dict[str, Any]) -> Dict[str, Any]:
         self.check_name_unique(instance)
         instance["weight"] = self.get_weight(instance["meeting_id"])
         return instance
-
-    def get_weight(self, meeting_id: int) -> int:
-        filter_ = FilterOperator("meeting_id", "=", meeting_id)
-        maximum = self.datastore.max(Collection("chat_group"), filter_, "weight", "int")
-        if maximum is None:
-            return 1
-        return maximum + 1

openslides_backend/action/actions/chat_group/sort.py

@@ -1,5 +1,6 @@
 from ....models.models import ChatGroup
 from ....permissions.permissions import Permissions
+from ....shared.filters import FilterOperator
 from ...generics.update import UpdateAction
 from ...mixins.linear_sort_mixin import LinearSortMixin
 from ...mixins.singular_action_mixin import SingularActionMixin
@@ -29,7 +30,6 @@ def get_updated_instances(self, action_data: ActionData) -> ActionData:
         # Action data is an iterable with exactly one item
         instance = next(iter(action_data))
         yield from self.sort_linear(
-            nodes=instance["chat_group_ids"],
-            filter_id=instance["meeting_id"],
-            filter_str="meeting_id",
+            instance["chat_group_ids"],
+            FilterOperator("meeting_id", "=", instance["meeting_id"]),
         )

openslides_backend/action/actions/group/create.py

@@ -3,16 +3,14 @@
 from ....models.models import Group
 from ....permissions.permission_helper import filter_surplus_permissions
 from ....permissions.permissions import Permissions
-from ....shared.filters import FilterOperator
-from ...action import original_instances
 from ...generics.create import CreateAction
+from ...mixins.weight_mixin import WeightMixin
 from ...util.default_schema import DefaultSchema
 from ...util.register import register_action
-from ...util.typing import ActionData
 
 
 @register_action("group.create")
-class GroupCreate(CreateAction):
+class GroupCreate(WeightMixin, CreateAction):
     """
     Action to create a group.
     """
@@ -26,27 +24,10 @@ class GroupCreate(CreateAction):
     )
     permission = Permissions.User.CAN_MANAGE
 
-    @original_instances
-    def get_updated_instances(self, action_data: ActionData) -> ActionData:
-        self.weight_map: Dict[int, int] = {}
-        return super().get_updated_instances(action_data)
-
     def update_instance(self, instance: Dict[str, Any]) -> Dict[str, Any]:
         if instance.get("permissions"):
             instance["permissions"] = filter_surplus_permissions(
                 instance["permissions"]
             )
-        self.set_weight(instance)
+        instance["weight"] = self.get_weight(instance["meeting_id"])
         return instance
-
-    def set_weight(self, instance: Dict[str, Any]) -> None:
-        meeting_id = instance["meeting_id"]
-        if meeting_id in self.weight_map:
-            max_weight = self.weight_map[meeting_id]
-        else:
-            filter_ = FilterOperator("meeting_id", "=", meeting_id)
-            max_weight = (
-                self.datastore.max(self.model.collection, filter_, "weight", "int") or 0
-            )
-        self.weight_map[meeting_id] = max_weight + 1
-        instance["weight"] = self.weight_map[meeting_id]

openslides_backend/action/actions/list_of_speakers/delete_all_speakers.py

@@ -24,7 +24,7 @@ class ListOfSpeakersDeleteAllSpeakersAction(DeleteAction):
 
     def get_updated_instances(self, action_data: ActionData) -> ActionData:
         for instance in action_data:
-            list_of_speakers = self.datastore.fetch_model(
+            list_of_speakers = self.datastore.get(
                 FullQualifiedId(Collection("list_of_speakers"), instance["id"]),
                 mapped_fields=["speaker_ids"],
             )

openslides_backend/action/actions/meeting/clone.py

@@ -283,7 +283,7 @@ def check_permissions(self, instance: Dict[str, Any]) -> None:
         if instance.get("committee_id"):
             committee_id = instance["committee_id"]
         else:
-            meeting = self.datastore.fetch_model(
+            meeting = self.datastore.get(
                 FullQualifiedId(Collection("meeting"), instance["meeting_id"]),
                 ["committee_id"],
             )

openslides_backend/action/actions/meeting/create.py

@@ -148,17 +148,10 @@ def update_instance(self, instance: Dict[str, Any]) -> Dict[str, Any]:
         )
         fqid_admin_group = FullQualifiedId(Collection("group"), action_results[1]["id"])  # type: ignore
         fqid_delegates_group = FullQualifiedId(Collection("group"), action_results[2]["id"])  # type: ignore
+        assert self.datastore.changed_models[fqid_default_group]["name"] == "Default"
+        assert self.datastore.changed_models[fqid_admin_group]["name"] == "Admin"
         assert (
-            self.datastore.additional_relation_models[fqid_default_group]["name"]
-            == "Default"
-        )
-        assert (
-            self.datastore.additional_relation_models[fqid_admin_group]["name"]
-            == "Admin"
-        )
-        assert (
-            self.datastore.additional_relation_models[fqid_delegates_group]["name"]
-            == "Delegates"
+            self.datastore.changed_models[fqid_delegates_group]["name"] == "Delegates"
         )
 
         instance["default_group_id"] = fqid_default_group.id