Merge pull request #126 from open-zaak/feature/external-documents-api

Add support for external documents API

requirements/base.txt

@@ -16,7 +16,7 @@ django-cors-middleware==1.3.1
 django-extra-fields==1.2.4
 django-filter==2.0
 django-ipware==2.1.0      # via django-axes
-django-loose-fk==0.5.3
+django-loose-fk==0.5.4
 django-markup==1.3
 django-privates==1.0.1.post0
 django-redis==4.10.0

requirements/ci.txt

@@ -17,7 +17,7 @@ django-cors-middleware==1.3.1
 django-extra-fields==1.2.4
 django-filter==2.0.0
 django-ipware==2.1.0
-django-loose-fk==0.5.3
+django-loose-fk==0.5.4
 django-markup==1.3
 django-privates==1.0.1.post0
 django-redis==4.10.0

requirements/dev.txt

@@ -26,7 +26,7 @@ django-extensions==2.2.1
 django-extra-fields==1.2.4
 django-filter==2.0.0
 django-ipware==2.1.0
-django-loose-fk==0.5.3
+django-loose-fk==0.5.4
 django-markup==1.3
 django-privates==1.0.1.post0
 django-redis==4.10.0

src/openzaak/components/documenten/api/fields.py

@@ -0,0 +1,33 @@
+from django_loose_fk.drf import FKOrURLField
+from rest_framework.fields import empty
+from rest_framework.reverse import reverse
+
+from ..models import EnkelvoudigInformatieObjectCanonical
+
+
+class EnkelvoudigInformatieObjectField(FKOrURLField):
+    """
+    Custom field to construct the url for models that have a ForeignKey to
+    `EnkelvoudigInformatieObject`
+
+    Needed because the canonical `EnkelvoudigInformatieObjectCanonical` no longer stores
+    the uuid, but the `EnkelvoudigInformatieObject`s related to it do
+    store the uuid
+    """
+
+    def to_representation(self, value):
+        if not isinstance(value, EnkelvoudigInformatieObjectCanonical):
+            return super().to_representation(value)
+
+        value = value.latest_version
+        return reverse(
+            "enkelvoudiginformatieobject-detail",
+            kwargs={"uuid": value.uuid},
+            request=self.context.get("request"),
+        )
+
+    def run_validation(self, data=empty):
+        value = super().run_validation(data=data)
+        if value.pk:
+            return value.canonical
+        return value

src/openzaak/components/documenten/query.py

@@ -5,6 +5,7 @@
 from django.db import models
 from django.db.models import Case, IntegerField, Value, When
 
+from django_loose_fk.virtual_models import ProxyMixin
 from vng_api_common.constants import ObjectTypes, VertrouwelijkheidsAanduiding
 from vng_api_common.scopes import Scope
 from vng_api_common.utils import get_resource_for_path
@@ -125,7 +126,10 @@ class ObjectInformatieObjectQuerySet(BlockChangeMixin, InformatieobjectRelatedQu
         ZaakInformatieObject: ObjectTypes.zaak,
     }
 
-    def create_from(self, relation: IORelation) -> models.Model:
+    def create_from(self, relation: IORelation) -> [models.Model, None]:
+        if isinstance(relation.informatieobject, ProxyMixin):
+            return None
+
         object_type = self.RELATIONS[type(relation)]
         relation_field = {object_type: getattr(relation, object_type)}
         return self.create(
@@ -135,6 +139,9 @@ def create_from(self, relation: IORelation) -> models.Model:
         )
 
     def delete_for(self, relation: IORelation) -> Tuple[int, Dict[str, int]]:
+        if isinstance(relation.informatieobject, ProxyMixin):
+            return (0, {})
+
         object_type = self.RELATIONS[type(relation)]
         relation_field = {object_type: getattr(relation, object_type)}
 

src/openzaak/components/documenten/tests/utils.py

@@ -1,9 +1,63 @@
+import uuid
+from datetime import date
+from typing import Dict, Union
+
 from django.conf import settings
+from django.utils import timezone
 
 from vng_api_common.tests import get_operation_url as _get_operation_url
 
+from openzaak.components.zaken.api.serializers.utils import _get_oio_endpoint
+
+JsonValue = Union[str, None, int, float]
+
 
 def get_operation_url(operation, **kwargs):
     return _get_operation_url(
         operation, spec_path=settings.SPEC_URL["documenten"], **kwargs
     )
+
+
+def get_eio_response(url: str, **overrides) -> Dict[str, JsonValue]:
+    eio_type = (
+        f"https://external.catalogus.nl/api/v1/informatieobjecttypen/{uuid.uuid4()}"
+    )
+    eio = {
+        "url": url,
+        "identificatie": "DOCUMENT-00001",
+        "bronorganisatie": "272618196",
+        "creatiedatum": date.today().isoformat(),
+        "titel": "some titel",
+        "auteur": "some auteur",
+        "status": "",
+        "formaat": "some formaat",
+        "taal": "nld",
+        "beginRegistratie": timezone.now().isoformat().replace("+00:00", "Z"),
+        "versie": 1,
+        "bestandsnaam": "",
+        "inhoud": f"{url}/download?versie=1",
+        "bestandsomvang": 100,
+        "link": "",
+        "beschrijving": "",
+        "ontvangstdatum": None,
+        "verzenddatum": None,
+        "ondertekening": {"soort": "", "datum": None},
+        "indicatieGebruiksrecht": None,
+        "vertrouwelijkheidaanduiding": "openbaar",
+        "integriteit": {"algoritme": "", "waarde": "", "datum": None},
+        "informatieobjecttype": eio_type,
+        "locked": False,
+    }
+    eio.update(**overrides)
+    return eio
+
+
+def get_oio_response(io_url: str, object_url: str) -> Dict[str, JsonValue]:
+    url = f"{_get_oio_endpoint(io_url)}/{uuid.uuid4()}"
+    oio = {
+        "url": url,
+        "informatieobject": io_url,
+        "object": object_url,
+        "objectType": "zaak",
+    }
+    return oio

src/openzaak/components/zaken/admin/zaken.py

@@ -92,9 +92,9 @@ class ZaakEigenschapAdmin(admin.ModelAdmin):
 
 @admin.register(ZaakInformatieObject)
 class ZaakInformatieObjectAdmin(admin.ModelAdmin):
-    list_display = ["zaak", "informatieobject"]
-    list_select_related = ["zaak"]
-    raw_id_fields = ["zaak"]
+    list_display = ["zaak", "_informatieobject", "_informatieobject_url"]
+    list_select_related = ["zaak", "_informatieobject"]
+    raw_id_fields = ["zaak", "_informatieobject"]
 
 
 @admin.register(Resultaat)

src/openzaak/components/zaken/api/serializers/utils.py

@@ -0,0 +1,35 @@
+import logging
+
+import requests
+from vng_api_common.models import APICredential
+
+logger = logging.getLogger(__name__)
+
+
+def _get_oio_endpoint(io_url: str) -> str:
+    """
+    Build the OIO endpoint from the EIO URL.
+
+    .. todo: TODO: clean this mess up - ideally this would use
+    gemma_zds_client.Client.from_url() & fetch the URL from the associated
+    API spec, but that requires mocking out the api spec fetch + setting up
+    the schema in the mock. A refactor in gemma-zds-client for this is
+    suitable.
+    """
+    start = io_url.split("enkelvoudiginformatieobjecten")[0]
+    url = f"{start}objectinformatieobjecten"
+    return url
+
+
+def create_remote_oio(io_url: str, zaak_url: str) -> dict:
+    client_auth = APICredential.get_auth(io_url)
+    if client_auth is None:
+        logger.warning("Missing credentials for %s", io_url)
+
+    url = _get_oio_endpoint(io_url)
+    headers = client_auth.credentials() if client_auth else {}
+    body = {"informatieobject": io_url, "object": zaak_url, "objectType": "zaak"}
+
+    response = requests.post(url, json=body, headers=headers)
+    response.raise_for_status()
+    return response.json()

src/openzaak/components/zaken/api/serializers/zaken.py

@@ -40,9 +40,7 @@
     StatusType,
     ZaakType,
 )
-from openzaak.components.documenten.api.serializers import (
-    EnkelvoudigInformatieObjectHyperlinkedRelatedField,
-)
+from openzaak.components.documenten.api.fields import EnkelvoudigInformatieObjectField
 from openzaak.components.documenten.models import (
     EnkelvoudigInformatieObject,
     EnkelvoudigInformatieObjectCanonical,
@@ -80,6 +78,7 @@
     RolOrganisatorischeEenheidSerializer,
     RolVestigingSerializer,
 )
+from .utils import create_remote_oio
 
 logger = logging.getLogger(__name__)
 
@@ -355,9 +354,10 @@ def validate(self, attrs):
             != Archiefstatus.nog_te_archiveren
         )
         if archiefstatus:
+            # TODO: check remote ZIO.informatieobject
             # search for related informatieobjects with status != 'gearchiveerd'
             canonical_ids = self.instance.zaakinformatieobject_set.values(
-                "informatieobject_id"
+                "_informatieobject_id"
             )
             io_ids = (
                 EnkelvoudigInformatieObjectCanonical.objects.filter(
@@ -378,8 +378,8 @@ def validate(self, attrs):
                         "archiefstatus",
                         _(
                             "Er zijn gerelateerde informatieobjecten waarvan de `status` nog niet gelijk is aan "
-                            "`gearchiveerd`. Dit is een voorwaarde voor het zetten van de `archiefstatus` op een andere "
-                            "waarde dan `nog_te_archiveren`."
+                            "`gearchiveerd`. Dit is een voorwaarde voor het zetten van de `archiefstatus` "
+                            "op een andere waarde dan `nog_te_archiveren`."
                         ),
                     },
                     code="documents-not-archived",
@@ -460,15 +460,17 @@ def validate(self, attrs):
         if validated_attrs["__is_eindstatus"]:
             zaak = validated_attrs["zaak"]
 
+            # TODO: check remote documents!
             if zaak.zaakinformatieobject_set.exclude(
-                informatieobject__lock=""
+                _informatieobject__lock=""
             ).exists():
                 raise serializers.ValidationError(
                     "Er zijn gerelateerde informatieobjecten die nog gelocked zijn."
                     "Deze informatieobjecten moet eerst unlocked worden voordat de zaak afgesloten kan worden.",
                     code="informatieobject-locked",
                 )
-            canonical_ids = zaak.zaakinformatieobject_set.values("informatieobject_id")
+            # TODO: support external IO
+            canonical_ids = zaak.zaakinformatieobject_set.values("_informatieobject_id")
             io_ids = (
                 EnkelvoudigInformatieObjectCanonical.objects.filter(
                     id__in=Subquery(canonical_ids)
@@ -576,14 +578,11 @@ class ZaakInformatieObjectSerializer(serializers.HyperlinkedModelSerializer):
         read_only=True,
         choices=[(force_text(value), key) for key, value in RelatieAarden.choices],
     )
-    informatieobject = EnkelvoudigInformatieObjectHyperlinkedRelatedField(
-        view_name="enkelvoudiginformatieobject-detail",
-        lookup_field="uuid",
-        queryset=EnkelvoudigInformatieObject.objects.all(),
-        min_length=1,
-        max_length=1000,
-        help_text=get_help_text("documenten.Gebruiksrechten", "informatieobject"),
+    informatieobject = EnkelvoudigInformatieObjectField(
         validators=[IsImmutableValidator()],
+        max_length=1000,
+        min_length=1,
+        help_text=get_help_text("zaken.ZaakInformatieObject", "informatieobject"),
     )
 
     class Meta:
@@ -611,6 +610,32 @@ class Meta:
             "zaak": {"lookup_field": "uuid", "validators": [IsImmutableValidator()]},
         }
 
+    def create(self, validated_data):
+        with transaction.atomic():
+            zio = super().create(validated_data)
+
+        # local FK - nothing to do -> our signals create the OIO
+        if zio.informatieobject.pk:
+            return zio
+
+        # we know that we got valid URLs in the initial data
+        io_url = self.initial_data["informatieobject"]
+        zaak_url = self.initial_data["zaak"]
+
+        # manual transaction management - documents API checks that the ZIO
+        # exists, so that transaction must be committed.
+        # If it fails in any other way, we need to handle that by rolling back
+        # the ZIO creation.
+        try:
+            create_remote_oio(io_url, zaak_url)
+        except Exception as exception:
+            zio.delete()
+            raise serializers.ValidationError(
+                _("Could not create remote relation: {exception}"),
+                params={"exception": exception},
+            )
+        return zio
+
 
 class ZaakEigenschapSerializer(NestedHyperlinkedModelSerializer):
     parent_lookup_kwargs = {"zaak_uuid": "zaak__uuid"}

src/openzaak/components/zaken/api/validators.py

@@ -9,6 +9,8 @@
     UniekeIdentificatieValidator as _UniekeIdentificatieValidator,
 )
 
+from openzaak.components.documenten.models import EnkelvoudigInformatieObject
+
 
 class RolOccurenceValidator:
     """
@@ -130,9 +132,15 @@ def __call__(self, attrs):
         if not informatieobject or not zaak:
             return
 
-        io = informatieobject.enkelvoudiginformatieobject_set.first()
+        if not isinstance(informatieobject, EnkelvoudigInformatieObject):
+            io_type_id = informatieobject.enkelvoudiginformatieobject_set.values_list(
+                "informatieobjecttype_id", flat=True
+            ).first()
+        else:
+            io_type_id = informatieobject.informatieobjecttype.id
+
         if not zaak.zaaktype.heeft_relevant_informatieobjecttype.filter(
-            id=io.informatieobjecttype_id, concept=False
+            id=io_type_id, concept=False
         ).exists():
             raise serializers.ValidationError(self.message, code=self.code)
 

src/openzaak/components/zaken/api/viewsets.py

@@ -459,7 +459,8 @@ class ZaakInformatieObjectViewSet(
     destroy:
     Verwijder een ZAAK-INFORMATIEOBJECT relatie.
 
-    De gespiegelde relatie in de Documenten API wordt door de Zaken API verwijderd. Consumers kunnen dit niet handmatig doen..
+    De gespiegelde relatie in de Documenten API wordt door de Zaken API
+    verwijderd. Consumers kunnen dit niet handmatig doen.
     """
 
     queryset = ZaakInformatieObject.objects.all()

src/openzaak/components/zaken/loaders.py

@@ -0,0 +1,34 @@
+from urllib.parse import urlparse
+
+from django.db.models.base import ModelBase
+
+from vng_api_common.utils import get_resource_for_path
+
+from openzaak.components.catalogi.models import InformatieObjectType
+from openzaak.loaders import AuthorizedRequestsLoader
+
+
+class EIOLoader(AuthorizedRequestsLoader):
+    """
+    Load the EIO directly instead of going through EIOCanonical.
+    """
+
+    def load(self, url: str, model: ModelBase):
+        from openzaak.components.documenten.models import (
+            EnkelvoudigInformatieObject,
+            EnkelvoudigInformatieObjectCanonical,
+        )
+
+        if model is EnkelvoudigInformatieObjectCanonical:
+            model = EnkelvoudigInformatieObject
+
+        if model is InformatieObjectType:
+            return self.resolve_io_type(url)
+
+        return super().load(url, model=model)
+
+    def resolve_io_type(self, url: str):
+        try:
+            return get_resource_for_path(urlparse(url).path)
+        except InformatieObjectType.DoesNotExist:
+            return super().load(url, model=InformatieObjectType)