Merge pull request #71 from jsangmeister/cleanup

Cleanup

docs/concepts.md

@@ -36,4 +36,14 @@ The keys of the `locked_fields` object can be fqfields, fqids or whole collectio
 
 ## Null values
 
-To simplify communication, the datastore assumes `null === undefined`. This means that writing `null` to any field or model equals the deletion of it. As a consequence, the datastore will never return `null` values on any `read` request.
+To simplify communication, the datastore assumes `null === undefined`. This means that writing `null` to any field or model equals the deletion of it. As a consequence, the datastore will never return `null` values on any `read` request.
+
+## PostgreSQL backend limitations
+
+The current implementation of the datastore uses a PostgreSQL backend as its database. For better indexing purposes, the keys (collections, ids and fields) are stored with a fixed length instead of a variable one. The following maximum length restrictions apply when using the PostgreSQL backend:
+
+    collection: 32
+    id: 16
+    field: 207
+
+Longer keys will be rejected with an `InvalidFormat` error.

writer/tests/integration/reserve_ids/test_reserve_ids.py

@@ -52,12 +52,10 @@ def connection_handler():
 
 
 def test_simple(reserve_ids_handler, connection_handler):
-    ids = reserve_ids_handler.reserve_ids(
-        {"amount": 1, "collection": "test_collection"}
-    )
+    ids = reserve_ids_handler.reserve_ids({"amount": 1, "collection": "a"})
 
     assert ids == [1]
-    assert connection_handler.storage.get("test_collection") == 2
+    assert connection_handler.storage.get("a") == 2
 
 
 def test_wrong_format(reserve_ids_handler):
@@ -67,7 +65,7 @@ def test_wrong_format(reserve_ids_handler):
 
 def test_negative_amount(reserve_ids_handler, connection_handler):
     with pytest.raises(InvalidFormat):
-        reserve_ids_handler.reserve_ids({"amount": -1, "collection": "test_collection"})
+        reserve_ids_handler.reserve_ids({"amount": -1, "collection": "a"})
 
 
 def test_too_long_collection(reserve_ids_handler, connection_handler):
@@ -78,29 +76,23 @@ def test_too_long_collection(reserve_ids_handler, connection_handler):
 
 
 def test_multiple_ids(reserve_ids_handler, connection_handler):
-    ids = reserve_ids_handler.reserve_ids(
-        {"amount": 4, "collection": "test_collection"}
-    )
+    ids = reserve_ids_handler.reserve_ids({"amount": 4, "collection": "a"})
 
     assert ids == [1, 2, 3, 4]
-    assert connection_handler.storage.get("test_collection") == 5
+    assert connection_handler.storage.get("a") == 5
 
 
 def test_successive_collections(reserve_ids_handler, connection_handler):
-    reserve_ids_handler.reserve_ids({"amount": 2, "collection": "test_collection1"})
-    ids = reserve_ids_handler.reserve_ids(
-        {"amount": 3, "collection": "test_collection2"}
-    )
+    reserve_ids_handler.reserve_ids({"amount": 2, "collection": "a"})
+    ids = reserve_ids_handler.reserve_ids({"amount": 3, "collection": "b"})
 
     assert ids == [1, 2, 3]
-    assert connection_handler.storage.get("test_collection2") == 4
+    assert connection_handler.storage.get("b") == 4
 
 
 def test_successive_ids(reserve_ids_handler, connection_handler):
-    reserve_ids_handler.reserve_ids({"amount": 2, "collection": "test_collection"})
-    ids = reserve_ids_handler.reserve_ids(
-        {"amount": 3, "collection": "test_collection"}
-    )
+    reserve_ids_handler.reserve_ids({"amount": 2, "collection": "a"})
+    ids = reserve_ids_handler.reserve_ids({"amount": 3, "collection": "a"})
 
     assert ids == [3, 4, 5]
-    assert connection_handler.storage.get("test_collection") == 6
+    assert connection_handler.storage.get("a") == 6

writer/writer/flask_frontend/json_handlers.py

@@ -1,11 +1,12 @@
+from dataclasses import dataclass
 from typing import Any, Dict, List, TypedDict, cast
 
 import fastjsonschema
 
 from shared.di import injector
 from shared.flask_frontend import InvalidRequest
-from shared.typing import JSON
-from shared.util import BadCodingError
+from shared.typing import JSON, Collection
+from shared.util import BadCodingError, SelfValidatingDataclass
 from writer.core import (
     BaseRequestEvent,
     RequestCreateEvent,
@@ -119,17 +120,18 @@ def create_event(self, event: Dict[str, Any]) -> BaseRequestEvent:
 )
 
 
-class ReserveIdsRequestJSON(TypedDict):
-    collection: str
+@dataclass
+class ReserveIdsRequestJSON(SelfValidatingDataclass):
+    collection: Collection
     amount: int
 
 
 class ReserveIdsHandler:
     def reserve_ids(self, data: JSON) -> List[int]:
         try:
-            parsed_data = cast(ReserveIdsRequestJSON, reserve_ids_schema(data))
+            parsed_data = ReserveIdsRequestJSON(**reserve_ids_schema(data))
         except fastjsonschema.JsonSchemaException as e:
             raise InvalidRequest(e.message)
 
         writer = injector.get(Writer)
-        return writer.reserve_ids(parsed_data["collection"], parsed_data["amount"])
+        return writer.reserve_ids(parsed_data.collection, parsed_data.amount)

writer/writer/postgresql_backend/sql_database_backend_service.py

@@ -27,9 +27,14 @@
 )
 
 
-FQID_MAX_LEN = 48
+# Max lengths (??) of the important key parts:
+# collection: 32
+# id: 16
+# field: 207
+# -> collection + id + field = 255
 COLLECTION_MAX_LEN = 32
-COLLECTIONFIELD_MAX_LEN = 255
+FQID_MAX_LEN = 48  # collection + id
+COLLECTIONFIELD_MAX_LEN = 239  # collection + field
 
 
 @service_as_singleton