added Inclusions to documenten api (#209)

**Restructured API Enhancements and Bug Fixes**

- **Added:** Implementation of 'trefwoorden' to the 'EnkelvoudigInformatieobject' model for improved metadata organization.
- **Added:** Integration of 'trefwoorden' filter to facilitate precise content filtering.
- **Changed:** Adjusted test suite to accommodate the new 'trefwoorden' attribute.
- **Added:** Introduction of 'expand' feature to the 'documenten' API, enabling more comprehensive data retrieval.
- **Changed:** Replaced hard-coded search logic for models and serializers.
- **Changed:** Renamed '_inclusions' to '_expand' for better clarity and consistency.
- **Changed:** Updated the 'expand' model to resolve previous errors and improve performance.
- **Fixed:** Corrected the URL for signals to ensure proper functionality.
- **Updated:** Expanded OpenAPI Specification (OAS) to include the 'expand' feature.
- **Added:** Custom renderer to enable camel case formatting for improved API response consistency.
- **Fixed:** Adjusted renderers to handle nested dictionaries and replace 'Expand' with '_expand'.
- **Updated:** 'zrc expansions.py' for alignment with the latest enhancements.
- **Fixed:** Handled scenarios involving empty values more gracefully.
- **Fixed:** Corrected an issue where a URL within a nested dictionary caused a server error.
- **Updated:** 'exclusions.py' to handle non-type objects more effectively during iteration.
- **Updated:** Modified 'expfield' to 'sub_field' for clarity and precision in naming.

These changes collectively enhance the API with new features, bug fixes, and code improvements. The 'trefwoorden' attribute, 'Inclusions' feature, and the 'expand' model enrich data handling, while code fixes and adjustments contribute to a more reliable and user-friendly system.

src/drc/api/filters.py

@@ -1,4 +1,8 @@
+from django.utils.translation import ugettext_lazy as _
+
 from django_filters import rest_framework as filters
+from drf_spectacular.types import OpenApiTypes
+from drf_spectacular.utils import extend_schema_field
 from vng_api_common.filters import URLModelChoiceFilter
 from vng_api_common.filtersets import FilterSet
 from vng_api_common.utils import get_help_text
@@ -12,10 +16,28 @@
 )
 
 
+def expand_filter(queryset, name, value):
+    """expansion filter logic is placed at view level"""
+    return queryset
+
+
 class EnkelvoudigInformatieObjectListFilter(FilterSet):
+    trefwoorden = filters.CharFilter(lookup_expr="icontains")
+
+    expand = extend_schema_field(OpenApiTypes.STR)(
+        filters.CharFilter(
+            method=expand_filter,
+            help_text=_(
+                "Examples: \n"
+                "`expand=zaaktype, status, status.statustype, hoofdzaak.status.statustype, hoofdzaak.deelzaken.status.statustype`\n"
+                "Haal details van gelinkte resources direct op. Als je meerdere resources tegelijk wilt ophalen kun je deze scheiden met een komma. Voor het ophalen van resources die een laag dieper genest zijn wordt de punt-notatie gebruikt.",
+            ),
+        )
+    )
+
     class Meta:
         model = EnkelvoudigInformatieObject
-        fields = ("identificatie", "bronorganisatie")
+        fields = ("identificatie", "bronorganisatie", "trefwoorden")
 
 
 class EnkelvoudigInformatieObjectDetailFilter(FilterSet):
@@ -31,6 +53,16 @@ class ObjectInformatieObjectFilter(FilterSet):
         instance_path="canonical",
         help_text=get_help_text("datamodel.ObjectInformatieObject", "informatieobject"),
     )
+    expand = extend_schema_field(OpenApiTypes.STR)(
+        filters.CharFilter(
+            method=expand_filter,
+            help_text=_(
+                "Examples: \n"
+                "`expand=zaaktype, status, status.statustype, hoofdzaak.status.statustype, hoofdzaak.deelzaken.status.statustype`\n"
+                "Haal details van gelinkte resources direct op. Als je meerdere resources tegelijk wilt ophalen kun je deze scheiden met een komma. Voor het ophalen van resources die een laag dieper genest zijn wordt de punt-notatie gebruikt.",
+            ),
+        )
+    )
 
     class Meta:
         model = ObjectInformatieObject
@@ -43,6 +75,16 @@ class GebruiksrechtenFilter(FilterSet):
         instance_path="canonical",
         help_text=get_help_text("datamodel.Gebruiksrechten", "informatieobject"),
     )
+    expand = extend_schema_field(OpenApiTypes.STR)(
+        filters.CharFilter(
+            method=expand_filter,
+            help_text=_(
+                "Examples: \n"
+                "`expand=zaaktype, status, status.statustype, hoofdzaak.status.statustype, hoofdzaak.deelzaken.status.statustype`\n"
+                "Haal details van gelinkte resources direct op. Als je meerdere resources tegelijk wilt ophalen kun je deze scheiden met een komma. Voor het ophalen van resources die een laag dieper genest zijn wordt de punt-notatie gebruikt.",
+            ),
+        )
+    )
 
     class Meta:
         model = Gebruiksrechten
@@ -59,6 +101,16 @@ class VerzendingFilter(FilterSet):
         instance_path="canonical",
         help_text=get_help_text("datamodel.Verzending", "informatieobject"),
     )
+    expand = extend_schema_field(OpenApiTypes.STR)(
+        filters.CharFilter(
+            method=expand_filter,
+            help_text=_(
+                "Examples: \n"
+                "`expand=zaaktype, status, status.statustype, hoofdzaak.status.statustype, hoofdzaak.deelzaken.status.statustype`\n"
+                "Haal details van gelinkte resources direct op. Als je meerdere resources tegelijk wilt ophalen kun je deze scheiden met een komma. Voor het ophalen van resources die een laag dieper genest zijn wordt de punt-notatie gebruikt.",
+            ),
+        )
+    )
 
     class Meta:
         model = Verzending

src/drc/api/renderers.py

@@ -1,3 +1,7 @@
+import json
+from json.decoder import JSONDecodeError
+
+from djangorestframework_camel_case.render import CamelCaseJSONRenderer
 from rest_framework.renderers import BaseRenderer
 
 
@@ -15,3 +19,45 @@ def render(
         if isinstance(data, str):
             return data.encode("utf-8")
         return data
+
+
+class CustomCamelCaseJSONRenderer(CamelCaseJSONRenderer):
+    """search and replace Expand with _expand in the response"""
+
+    def render(self, data, *args, **kwargs):
+        rendered = super().render(data, *args, **kwargs)
+        try:
+            list_of_responses = json.loads(rendered)
+        except JSONDecodeError as e:
+            return rendered
+
+        list_of_responses = self.replace_key_nested_dicts(
+            list_of_responses, "Expand", "_expand"
+        )
+        data_bytes = json.dumps(list_of_responses).encode("utf-8")
+        return data_bytes
+
+    def replace_key_nested_dicts(self, nested_dicts, old_key, new_key):
+        if isinstance(nested_dicts, list):
+            new_list = []
+            for dictionary in nested_dicts:
+                new_list.append(self.replace_key(dictionary, old_key, new_key))
+            return new_list
+        else:
+            return self.replace_key(nested_dicts, old_key, new_key)
+
+    def replace_key(self, dictionary, old_key, new_key):
+        if isinstance(dictionary, dict):
+            new_dict = {}
+            for key, value in dictionary.items():
+                if key == old_key:
+                    key = new_key
+                new_dict[key] = self.replace_key(value, old_key, new_key)
+            return new_dict
+        elif isinstance(dictionary, list):
+            new_list = []
+            for item in dictionary:
+                new_list.append(self.replace_key(item, old_key, new_key))
+            return new_list
+        else:
+            return dictionary

src/drc/api/serializers/enkelvoudig_informatieobject.py

@@ -128,6 +128,7 @@ class Meta:
             "informatieobjecttype",  # van-relatie,
             "locked",
             "bestandsdelen",
+            "trefwoorden",
         )
         extra_kwargs = {
             "informatieobjecttype": {

src/drc/api/tests/test_enkelvoudiginformatieobject.py

@@ -116,6 +116,7 @@ def test_create(self, *mocks):
                 "locked": False,
                 "bestandsdelen": [],
                 "lock": "",
+                "trefwoorden": None,
             }
         )
         response_data = response.json()
@@ -173,6 +174,7 @@ def test_read(self):
             "informatieobjecttype": INFORMATIEOBJECTTYPE,
             "locked": False,
             "bestandsdelen": [],
+            "trefwoorden": None,
         }
         response_data = response.json()
         self.assertEqual(sorted(response_data.keys()), sorted(expected.keys()))

src/drc/api/tests/test_gebruiksrechten.py

@@ -131,3 +131,17 @@ def test_filter_invalid_resource_url(self):
 
                 self.assertEqual(response.status_code, status.HTTP_200_OK)
                 self.assertEqual(len(response.data), 0)
+
+    def test_expand_mechanism(self):
+        gebruiksrechten = GebruiksrechtenFactory.create(
+            informatieobject__latest_version__informatieobjecttype=INFORMATIEOBJECTTYPE
+        )
+        url = reverse("gebruiksrechten-list")
+
+        response = self.client.get(url, {"expand": "informatieobject"})
+
+        from pprint import pprint
+
+        pprint(response.json())
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)

src/drc/api/tests/test_objectinformatieobject.py

@@ -49,6 +49,14 @@ def test_create_with_objecttype_zaak(self, *mocks):
         zio = eio.canonical.objectinformatieobject_set.get()
         self.assertEqual(zio.object, ZAAK)
 
+    def test_expand_mechanism(self):
+        oio = ObjectInformatieObjectFactory.create(is_zaak=True)
+        oio2 = ObjectInformatieObjectFactory.create(is_zaak=True)
+
+        response = self.client.get(self.list_url, {"expand": "informatieobject"})
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+
     @patch("vng_api_common.validators.obj_has_shape", return_value=True)
     def test_create_with_objecttype_besluit(self, *mocks):
         eio = EnkelvoudigInformatieObjectFactory.create()

src/drc/api/views/enkelvoudig_informatieobject.py

@@ -26,6 +26,7 @@
 
 from drc.api.audits import AUDIT_DRC
 from drc.api.data_filtering import ListFilterByAuthorizationsMixin
+from drc.api.exclusions import EXPAND_QUERY_PARAM, ExpandFieldValidator, ExpansionMixin
 from drc.api.filters import (
     EnkelvoudigInformatieObjectDetailFilter,
     EnkelvoudigInformatieObjectListFilter,
@@ -146,6 +147,8 @@ class EnkelvoudigInformatieObjectViewSet(
     SearchMixin,
     ListFilterByAuthorizationsMixin,
     AuditTrailViewsetMixin,
+    ExpandFieldValidator,
+    ExpansionMixin,
     viewsets.ModelViewSet,
 ):
     global_description = _(
@@ -157,7 +160,7 @@ class EnkelvoudigInformatieObjectViewSet(
     lookup_field = "uuid"
     pagination_class = PageNumberPagination
     search_input_serializer_class = EIOZoekSerializer
-
+    serializer_class = EnkelvoudigInformatieObjectSerializer
     permission_classes = (InformationObjectAuthScopesRequired,)
     required_scopes = {
         "list": SCOPE_DOCUMENTEN_ALLES_LEZEN,
@@ -242,7 +245,7 @@ def filterset_class(self):
             return EnkelvoudigInformatieObjectDetailFilter
         return EnkelvoudigInformatieObjectListFilter
 
-    def get_serializer_class(self):
+    def get_serializer_class(self, *args, **kwargs):
         """
         To validate that a lock id is sent only with PUT and PATCH operations
         """
@@ -252,7 +255,9 @@ def get_serializer_class(self):
             return EnkelvoudigInformatieObjectCreateLockSerializer
         return EnkelvoudigInformatieObjectSerializer
 
-    @extend_schema(parameters=[VERSIE_QUERY_PARAM, REGISTRATIE_QUERY_PARAM])
+    @extend_schema(
+        parameters=[VERSIE_QUERY_PARAM, REGISTRATIE_QUERY_PARAM, EXPAND_QUERY_PARAM]
+    )
     def retrieve(self, request, *args, **kwargs):
         return super().retrieve(request, *args, **kwargs)
 

src/drc/api/views/gebruiksrechten.py

@@ -9,6 +9,7 @@
 
 from drc.api.audits import AUDIT_DRC
 from drc.api.data_filtering import ListFilterByAuthorizationsMixin
+from drc.api.exclusions import ExpandFieldValidator, ExpansionMixin
 from drc.api.filters import GebruiksrechtenFilter
 from drc.api.kanalen import KANAAL_DOCUMENTEN
 from drc.api.permissions import InformationObjectRelatedAuthScopesRequired
@@ -64,6 +65,8 @@ class GebruiksrechtenViewSet(
     CheckQueryParamsMixin,
     ListFilterByAuthorizationsMixin,
     AuditTrailViewsetMixin,
+    ExpandFieldValidator,
+    ExpansionMixin,
     viewsets.ModelViewSet,
 ):
     global_description = _(

src/drc/api/views/object_informatieobject.py

@@ -7,6 +7,7 @@
 from vng_api_common.viewsets import CheckQueryParamsMixin
 
 from drc.api.data_filtering import ListFilterByAuthorizationsMixin
+from drc.api.exclusions import ExpandFieldValidator, ExpansionMixin
 from drc.api.filters import ObjectInformatieObjectFilter
 from drc.api.permissions import InformationObjectRelatedAuthScopesRequired
 from drc.api.scopes import (
@@ -54,6 +55,8 @@
 class ObjectInformatieObjectViewSet(
     CheckQueryParamsMixin,
     ListFilterByAuthorizationsMixin,
+    ExpandFieldValidator,
+    ExpansionMixin,
     mixins.CreateModelMixin,
     mixins.DestroyModelMixin,
     viewsets.ReadOnlyModelViewSet,

src/drc/api/views/verzending.py

@@ -6,6 +6,7 @@
 from vng_api_common.caching.decorators import conditional_retrieve
 from vng_api_common.viewsets import CheckQueryParamsMixin
 
+from drc.api.exclusions import ExpandFieldValidator, ExpansionMixin
 from drc.api.filters import VerzendingFilter
 from drc.api.scopes import (
     SCOPE_DOCUMENTEN_AANMAKEN,
@@ -48,6 +49,8 @@
 )
 class VerzendingViewSet(
     CheckQueryParamsMixin,
+    ExpandFieldValidator,
+    ExpansionMixin,
     viewsets.ModelViewSet,
 ):
 

src/drc/conf/api.py

@@ -6,6 +6,9 @@
 
 REST_FRAMEWORK = BASE_REST_FRAMEWORK.copy()
 REST_FRAMEWORK["PAGE_SIZE"] = 100
+REST_FRAMEWORK["DEFAULT_RENDERER_CLASSES"] = (
+    "drc.api.renderers.CustomCamelCaseJSONRenderer",
+)
 
 SECURITY_DEFINITION_NAME = "JWT-Claims"
 

src/drc/datamodel/migrations/0063_enkelvoudiginformatieobject_trefwoorden.py

@@ -0,0 +1,25 @@
+# Generated by Django 3.2.13 on 2023-04-19 12:43
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("datamodel", "0062_auto_20230222_1424"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="enkelvoudiginformatieobject",
+            name="trefwoorden",
+            field=django.contrib.postgres.fields.ArrayField(
+                base_field=models.CharField(max_length=100),
+                blank=True,
+                help_text="Een lijst van trefwoorden gescheiden door comma's.",
+                null=True,
+                size=None,
+            ),
+        ),
+    ]

src/drc/datamodel/models/enkelvoudig_informatieobject.py

@@ -1,5 +1,6 @@
 import uuid as _uuid
 
+from django.contrib.postgres.fields import ArrayField
 from django.core.validators import MinValueValidator
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
@@ -148,5 +149,12 @@ class EnkelvoudigInformatieObject(ETagMixin, APIMixin, InformatieObject):
         ),
     )
 
+    trefwoorden = ArrayField(
+        models.CharField(max_length=100),
+        help_text="Een lijst van trefwoorden gescheiden door comma's.",
+        blank=True,
+        null=True,
+    )
+
     class Meta:
         unique_together = ("uuid", "versie")

src/drc/tests/test_upload.py

@@ -790,7 +790,6 @@ def test_update_metadata_without_upload(self):
         * bestandsdelen objects are created
         """
         self._create_metadata()
-
         # update file metadata
         eio_url = get_operation_url(
             "enkelvoudiginformatieobject_retrieve", uuid=self.eio.uuid
@@ -809,6 +808,13 @@ def test_update_metadata_without_upload(self):
         self.assertEqual(len(data["bestandsdelen"]), 2)
         self.assertEqual(self.eio.beschrijving, "beschrijving2")
 
+    def test_expand_mechanism(self):
+        self._create_metadata()
+        list_call = get_operation_url("enkelvoudiginformatieobject_list")
+        response = self.client.get(list_call, {"expand": "bestandsdelen"})
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+
     def test_update_metadata_after_unfinished_upload(self):
         """
         Test the update process of the document metadata with some of part files uploaded

src/resources.md

@@ -111,6 +111,7 @@ Uitleg bij mogelijke waarden:
 | informatieobjecttype | URL-referentie naar het INFORMATIEOBJECTTYPE (in de Catalogi API). | string | ja | C​R​U​D |
 | locked | Geeft aan of het document gelocked is. Alleen als een document gelocked is, mogen er aanpassingen gemaakt worden. | boolean | ja | ~~C~~​R​~~U~~​~~D~~ |
 | bestandsdelen |  | array | ja | ~~C~~​R​~~U~~​~~D~~ |
+| trefwoorden | Een lijst van trefwoorden gescheiden door comma's. | array | nee | C​R​U​D |
 
 ## Gebruiksrechten