Merge pull request #212 from open-zaak/feature/loose-fk-resource-vali…

…dator

Feature/loose fk resource validator

requirements/base.txt

@@ -17,7 +17,7 @@ django-cors-middleware==1.3.1
 django-extra-fields==1.2.4
 django-filter==2.0
 django-ipware==2.1.0      # via django-axes
-django-loose-fk==0.5.6
+django-loose-fk==0.5.7
 django-markup==1.3
 django-ordered-model==2.1.0  # via django-admin-index
 django-privates==1.0.1.post0

requirements/ci.txt

@@ -18,7 +18,7 @@ django-cors-middleware==1.3.1
 django-extra-fields==1.2.4
 django-filter==2.0.0
 django-ipware==2.1.0
-django-loose-fk==0.5.6
+django-loose-fk==0.5.7
 django-markup==1.3
 django-ordered-model==2.1.0
 django-privates==1.0.1.post0

requirements/dev.txt

@@ -27,7 +27,7 @@ django-extensions==2.2.1
 django-extra-fields==1.2.4
 django-filter==2.0.0
 django-ipware==2.1.0
-django-loose-fk==0.5.6
+django-loose-fk==0.5.7
 django-markup==1.3
 django-ordered-model==2.1.0
 django-privates==1.0.1.post0

src/openzaak/components/besluiten/api/serializers.py

@@ -1,18 +1,22 @@
 """
 Serializers of the Besluit Registratie Component REST API
 """
-from django_loose_fk.drf import FKOrURLField
+from django.conf import settings
+
 from rest_framework import serializers
 from rest_framework.validators import UniqueTogetherValidator
 from vng_api_common.serializers import add_choice_values_help_text
-from vng_api_common.utils import get_help_text
 from vng_api_common.validators import IsImmutableValidator, validate_rsin
 
 from openzaak.components.documenten.api.serializers import (
     EnkelvoudigInformatieObjectHyperlinkedRelatedField,
 )
 from openzaak.components.documenten.models import EnkelvoudigInformatieObject
-from openzaak.utils.validators import LooseFkIsImmutableValidator, PublishValidator
+from openzaak.utils.validators import (
+    LooseFkIsImmutableValidator,
+    LooseFkResourceValidator,
+    PublishValidator,
+)
 
 from ..constants import VervalRedenen
 from ..models import Besluit, BesluitInformatieObject
@@ -53,7 +57,11 @@ class Meta:
             "besluittype": {
                 "lookup_field": "uuid",
                 "max_length": 200,
-                "validators": [LooseFkIsImmutableValidator(), PublishValidator()],
+                "validators": [
+                    LooseFkResourceValidator("BesluitType", settings.ZTC_API_SPEC),
+                    LooseFkIsImmutableValidator(),
+                    PublishValidator(),
+                ],
             },
             # per BRC API spec!
             "zaak": {"lookup_field": "uuid", "max_length": 200},

src/openzaak/components/besluiten/tests/test_besluit_create.py

@@ -181,7 +181,7 @@ def test_create_external_besluittype(self):
         besluittype = "https://externe.catalogus.nl/api/v1/besluittypen/b71f72ef-198d-44d8-af64-ae1932df830a"
         url = get_operation_url("besluit_create")
 
-        with requests_mock.Mocker() as m:
+        with requests_mock.Mocker(real_http=True) as m:
             m.register_uri(
                 "GET",
                 besluittype,
@@ -228,7 +228,7 @@ def test_create_external_besluittype_fail_bad_url(self):
             {
                 "verantwoordelijke_organisatie": "517439943",  # RSIN
                 "identificatie": "123123",
-                "besluittype": "http://example.com",
+                "besluittype": "abcd",
                 "datum": "2018-09-06",
                 "toelichting": "Vergunning verleend.",
                 "ingangsdatum": "2018-10-01",
@@ -243,3 +243,75 @@ def test_create_external_besluittype_fail_bad_url(self):
 
         error = get_validation_errors(response, "besluittype")
         self.assertEqual(error["code"], "bad-url")
+
+    def test_create_external_besluittype_fail_not_json_url(self):
+        url = reverse(Besluit)
+
+        response = self.client.post(
+            url,
+            {
+                "verantwoordelijke_organisatie": "517439943",  # RSIN
+                "identificatie": "123123",
+                "besluittype": "http://example.com",
+                "datum": "2018-09-06",
+                "toelichting": "Vergunning verleend.",
+                "ingangsdatum": "2018-10-01",
+                "vervaldatum": "2018-11-01",
+                "vervalreden": VervalRedenen.tijdelijk,
+            },
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+
+        error = get_validation_errors(response, "besluittype")
+        self.assertEqual(error["code"], "invalid-resource")
+
+    def test_create_external_besluittype_fail_invalid_schema(self):
+        catalogus = "https://externe.catalogus.nl/api/v1/catalogussen/1c8e36be-338c-4c07-ac5e-1adf55bec04a"
+        besluittype = "https://externe.catalogus.nl/api/v1/besluittypen/b71f72ef-198d-44d8-af64-ae1932df830a"
+        url = get_operation_url("besluit_create")
+
+        with requests_mock.Mocker(real_http=True) as m:
+            m.register_uri(
+                "GET",
+                besluittype,
+                json={
+                    "url": besluittype,
+                    "catalogus": catalogus,
+                    "zaaktypes": [],
+                    "informatieobjecttypen": [],
+                    "beginGeldigheid": "2018-01-01",
+                    "eindeGeldigheid": None,
+                    "concept": False,
+                },
+            )
+            m.register_uri(
+                "GET",
+                catalogus,
+                json={
+                    "url": catalogus,
+                    "domein": "PUB",
+                    "informatieobjecttypen": [],
+                    "zaaktypen": [],
+                    "besluittypen": [besluittype],
+                },
+            )
+
+            response = self.client.post(
+                url,
+                {
+                    "verantwoordelijke_organisatie": "517439943",  # RSIN
+                    "identificatie": "123123",
+                    "besluittype": besluittype,
+                    "datum": "2018-09-06",
+                    "toelichting": "Vergunning verleend.",
+                    "ingangsdatum": "2018-10-01",
+                    "vervaldatum": "2018-11-01",
+                    "vervalreden": VervalRedenen.tijdelijk,
+                },
+            )
+
+        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+
+        error = get_validation_errors(response, "besluittype")
+        self.assertEqual(error["code"], "invalid-resource")

src/openzaak/components/zaken/api/serializers/zaken.py

@@ -48,7 +48,11 @@
 from openzaak.utils.auth import get_auth
 from openzaak.utils.exceptions import DetermineProcessEndDateException
 from openzaak.utils.serializer_fields import LengthHyperlinkedRelatedField
-from openzaak.utils.validators import LooseFkIsImmutableValidator, PublishValidator
+from openzaak.utils.validators import (
+    LooseFkIsImmutableValidator,
+    LooseFkResourceValidator,
+    PublishValidator,
+)
 
 from ...brondatum import BrondatumCalculator
 from ...constants import AardZaakRelatie, BetalingsIndicatie, IndicatieMachtiging
@@ -580,7 +584,12 @@ class ZaakInformatieObjectSerializer(serializers.HyperlinkedModelSerializer):
         choices=[(force_text(value), key) for key, value in RelatieAarden.choices],
     )
     informatieobject = EnkelvoudigInformatieObjectField(
-        validators=[LooseFkIsImmutableValidator(instance_path="canonical")],
+        validators=[
+            LooseFkIsImmutableValidator(instance_path="canonical"),
+            LooseFkResourceValidator(
+                "EnkelvoudigInformatieObject", settings.DRC_API_SPEC
+            ),
+        ],
         max_length=1000,
         min_length=1,
         help_text=get_help_text("zaken.ZaakInformatieObject", "informatieobject"),

src/openzaak/components/zaken/api/viewsets.py

@@ -393,7 +393,7 @@ class ZaakObjectViewSet(
     Een specifiek ZAAKOBJECT opvragen.
     """
 
-    queryset = ZaakObject.objects.all()
+    queryset = ZaakObject.objects.select_related("zaak").all()
     serializer_class = ZaakObjectSerializer
     filterset_class = ZaakObjectFilter
     lookup_field = "uuid"
@@ -705,7 +705,7 @@ class ResultaatViewSet(
 
     """
 
-    queryset = Resultaat.objects.all()
+    queryset = Resultaat.objects.select_related("resultaattype", "zaak").all()
     serializer_class = ResultaatSerializer
     filterset_class = ResultaatFilter
     lookup_field = "uuid"

src/openzaak/components/zaken/tests/test_zaakinformatieobjecten.py

@@ -295,8 +295,6 @@ def test_delete(self):
 class ExternalDocumentsAPITests(JWTAuthMixin, APITestCase):
     heeft_alle_autorisaties = True
 
-    # TODO: validation tests with bad inputs
-
     def test_relate_external_document(self):
         base = "https://external.documenten.nl/api/v1/"
         document = f"{base}enkelvoudiginformatieobjecten/{uuid.uuid4()}"
@@ -311,7 +309,7 @@ def test_relate_external_document(self):
         )
 
         with self.subTest(section="zio-create"):
-            with requests_mock.Mocker() as m:
+            with requests_mock.Mocker(real_http=True) as m:
                 m.get(document, json=eio_response)
                 m.post(
                     "https://external.documenten.nl/api/v1/objectinformatieobjecten",
@@ -358,3 +356,60 @@ def test_relate_external_document(self):
 
             self.assertEqual(len(data), 1)
             self.assertEqual(data[0]["informatieobject"], document)
+
+    def test_create_zio_fail_bad_url(self):
+        zaak = ZaakFactory.create(zaaktype__concept=False)
+        zaak_url = f"http://openzaak.nl{reverse(zaak)}"
+        list_url = reverse(ZaakInformatieObject)
+        data = {"zaak": zaak_url, "informatieobject": "abcd"}
+
+        response = self.client.post(list_url, data, HTTP_HOST="openzaak.nl")
+
+        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+
+        error = get_validation_errors(response, "informatieobject")
+        self.assertEqual(error["code"], "bad-url")
+
+    def test_create_zio_fail_not_json(self):
+        zaak = ZaakFactory.create(zaaktype__concept=False)
+        zaak_url = f"http://openzaak.nl{reverse(zaak)}"
+        list_url = reverse(ZaakInformatieObject)
+        data = {"zaak": zaak_url, "informatieobject": "http://example.com"}
+
+        response = self.client.post(list_url, data, HTTP_HOST="openzaak.nl")
+
+        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+
+        error = get_validation_errors(response, "informatieobject")
+        self.assertEqual(error["code"], "invalid-resource")
+
+    def test_create_zio_fail_invalid_schema(self):
+        base = "https://external.documenten.nl/api/v1/"
+        document = f"{base}enkelvoudiginformatieobjecten/{uuid.uuid4()}"
+        zio_type = ZaakInformatieobjectTypeFactory.create(
+            informatieobjecttype__concept=False, zaaktype__concept=False
+        )
+        zaak = ZaakFactory.create(zaaktype=zio_type.zaaktype)
+        zaak_url = f"http://openzaak.nl{reverse(zaak)}"
+
+        with requests_mock.Mocker(real_http=True) as m:
+            m.get(
+                document,
+                json={
+                    "url": document,
+                    "beschrijving": "",
+                    "ontvangstdatum": None,
+                    "informatieobjecttype": f"http://testserver{reverse(zio_type.informatieobjecttype)}",
+                    "locked": False,
+                },
+            )
+
+            response = self.client.post(
+                reverse(ZaakInformatieObject),
+                {"zaak": zaak_url, "informatieobject": document},
+            )
+
+        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+
+        error = get_validation_errors(response, "informatieobject")
+        self.assertEqual(error["code"], "invalid-resource")

src/openzaak/conf/includes/api.py

@@ -50,7 +50,7 @@
 VRL_API_SPEC = "https://referentielijsten-api.vng.cloud/api/v1/schema/openapi.yaml?v=3"
 
 ztc_repo = "vng-Realisatie/gemma-zaaktypecatalogus"
-ztc_commit = "9c51082d6399060bff6bee2e23d0f22472bfa47f"
+ztc_commit = "6d599e0a49f8348bf70abb47c683fb542a0b6e6d"
 ZTC_API_SPEC = (
     f"https://raw.githubusercontent.com/{ztc_repo}/{ztc_commit}/src/openapi.yaml"
 )

src/openzaak/loaders.py

@@ -1,7 +1,7 @@
 import json
 
 import requests
-from django_loose_fk.loaders import BaseLoader, FetchError
+from django_loose_fk.loaders import BaseLoader, FetchError, FetchJsonError
 
 
 class AuthorizedRequestsLoader(BaseLoader):
@@ -29,6 +29,6 @@ def fetch_object(url: str) -> dict:
         try:
             data = response.json()
         except json.JSONDecodeError as exc:
-            raise FetchError(exc.args[0]) from exc
+            raise FetchJsonError(exc.args[0]) from exc
 
         return data

src/openzaak/utils/validators.py

@@ -1,9 +1,14 @@
+from urllib.parse import urlparse
+
 from django.utils.translation import ugettext_lazy as _
 
 from django_loose_fk.drf import FKOrURLField, FKOrURLValidator
 from rest_framework import serializers
+from vng_api_common.oas import fetcher, obj_has_shape
 from vng_api_common.validators import IsImmutableValidator
 
+from ..loaders import AuthorizedRequestsLoader
+
 
 class PublishValidator(FKOrURLValidator):
     publish_code = "not-published"
@@ -17,7 +22,11 @@ def set_context(self, serializer_field):
     def __call__(self, value):
         # loose-fk field
         if value and isinstance(value, str):
-            super().__call__(value)
+            # not to double FKOrURLValidator
+            try:
+                super().__call__(value)
+            except serializers.ValidationError:
+                return
             value = self.resolver.resolve(self.host, value)
 
         if value.concept:
@@ -53,7 +62,12 @@ def __call__(self, new_value):
 
         # loose-fk field
         if new_value and isinstance(new_value, str):
-            super().__call__(new_value)
+            # not to double FKOrURLValidator
+            try:
+                super().__call__(new_value)
+            except serializers.ValidationError:
+                return
+
             new_value = self.resolver.resolve(self.host, new_value)
 
         if self.instance_path:
@@ -64,3 +78,40 @@ def __call__(self, new_value):
             raise serializers.ValidationError(
                 IsImmutableValidator.message, code=IsImmutableValidator.code
             )
+
+
+class LooseFkResourceValidator(FKOrURLValidator):
+    resource_message = _(
+        "The URL {url} resource did not look like a(n) `{resource}`. Please provide a valid URL."
+    )
+    resource_code = "invalid-resource"
+
+    def __init__(self, resource: str, oas_schema: str, *args, **kwargs):
+        self.resource = resource
+        self.oas_schema = oas_schema
+        super().__init__(*args, **kwargs)
+
+    def __call__(self, value: str):
+        # not to double FKOrURLValidator
+        try:
+            super().__call__(value)
+        except serializers.ValidationError:
+            return
+
+        # if local - do nothing
+        parsed = urlparse(value)
+        is_local = parsed.netloc == self.host
+        if is_local:
+            return
+
+        obj = AuthorizedRequestsLoader.fetch_object(value)
+
+        # check if the shape matches
+        schema = fetcher.fetch(self.oas_schema)
+        if not obj_has_shape(obj, schema, self.resource):
+            raise serializers.ValidationError(
+                self.resource_message.format(url=value, resource=self.resource),
+                code=self.resource_code,
+            )
+
+        return obj