KER-280 | GDPR API data export

.pre-commit-config.yaml

@@ -15,7 +15,7 @@ repos:
     hooks:
       - id: black
   - repo: https://github.com/pycqa/flake8
-    rev: 4.0.1
+    rev: 6.1.0
     hooks:
       - id: flake8
         exclude: "migrations,tests"

Dockerfile

@@ -26,7 +26,7 @@ RUN apt-get update && apt-get install -y postgresql-client less netcat-openbsd g
 RUN sed -i 's/^# *\(fi_FI.UTF-8\)/\1/' /etc/locale.gen
 RUN locale-gen
 
-RUN pip install --upgrade pip && \
+RUN pip install --upgrade pip setuptools wheel && \
     pip install --no-cache-dir uwsgi
 
 # Sentry CLI for sending events from non-Python processes to Sentry

README.md

@@ -99,8 +99,8 @@ After you've got tunnistamo running locally, ssh to the tunnistamo docker contai
 and execute the following four commands inside your docker container:
 
 ```bash
-./manage.py add_oidc_client -n kerrokantasi-ui -t "id_token token" -u "http://localhost:8086/callback" "http://localhost:8086/silent-callback" -i https://api.hel.fi/auth/kerrokantasi-ui -m github -s dev
-./manage.py add_oidc_client -n kerrokantasi-api -t "code" -u http://localhost:8080/pysocial/complete/tunnistamo/ -i https://api.hel.fi/auth/kerrokantasi -m github -s dev -c
+./manage.py add_oidc_client -n kerrokantasi-ui -t "id_token token" -u "http://localhost:8086/callback" "http://localhost:8086/silent-renew" -i https://api.hel.fi/auth/kerrokantasi-ui -m github -s dev
+./manage.py add_oidc_client -n kerrokantasi-api -t "code" -u http://localhost:8080/complete/tunnistamo/ -i https://api.hel.fi/auth/kerrokantasi -m github -s dev -c
 ./manage.py add_oidc_api -n kerrokantasi -d https://api.hel.fi/auth -s email,profile -c https://api.hel.fi/auth/kerrokantasi
 ./manage.py add_oidc_api_scope -an kerrokantasi -c https://api.hel.fi/auth/kerrokantasi -n "Kerrokantasi" -d "Lorem ipsum"
 ./manage.py add_oidc_client_to_api_scope -asi https://api.hel.fi/auth/kerrokantasi -c https://api.hel.fi/auth/kerrokantasi-ui

democracy/factories/hearing.py

@@ -1,15 +1,18 @@
 import factory
 import factory.fuzzy
 import logging
+import os
 import random
 from datetime import timedelta
 from django.utils.timezone import now
 
 from democracy.enums import Commenting
 from democracy.factories.comment import BaseCommentFactory
 from democracy.models import Hearing, Label, Section, SectionComment, SectionType
+from democracy.models.section import CommentImage, SectionFile, SectionImage
+from democracy.tests.utils import FILE_SOURCE_PATH, FILES
 
-LOG = logging.getLogger(__name__)
+logger = logging.getLogger(__name__)
 
 
 class LabelFactory(factory.django.DjangoModelFactory):
@@ -24,6 +27,15 @@ class Meta:
         model = SectionComment
 
 
+class CommentImageFactory(factory.django.DjangoModelFactory):
+    title = factory.Faker("word")
+    caption = factory.Faker("word")
+    image = factory.django.ImageField()
+
+    class Meta:
+        model = CommentImage
+
+
 class HearingFactory(factory.django.DjangoModelFactory):
     class Meta:
         model = Hearing
@@ -48,7 +60,7 @@ def post(obj, create, extracted, **kwargs):
 
         for x in range(random.randint(1, 4)):
             section = SectionFactory(hearing=obj)
-            LOG.info("Hearing %s: Created section %s", obj, section)
+            logger.info("Hearing %s: Created section %s", obj, section)
 
         obj.recache_n_comments()
 
@@ -67,5 +79,24 @@ class Meta:
     def post(obj, create, extracted, **kwargs):
         for x in range(random.randint(1, 5)):
             comment = SectionCommentFactory(section=obj)
-            LOG.info("Hearing %s: Section %s: Created section comment %s", obj.hearing, obj, comment.pk)
+            logger.info("Hearing %s: Section %s: Created section comment %s", obj.hearing, obj, comment.pk)
         obj.recache_n_comments()
+
+
+class SectionFileFactory(factory.django.DjangoModelFactory):
+    title = factory.Faker("word")
+    caption = factory.Faker("word")
+    file = factory.django.FileField(from_path=os.path.join(FILE_SOURCE_PATH, FILES["PDF"]))
+
+    class Meta:
+        model = SectionFile
+
+
+class SectionImageFactory(factory.django.DjangoModelFactory):
+    title = factory.Faker("word")
+    caption = factory.Faker("word")
+    alt_text = factory.Faker("word")
+    image = factory.django.ImageField()
+
+    class Meta:
+        model = SectionImage

democracy/models/base.py

@@ -7,6 +7,7 @@
 from django.utils.translation import gettext_lazy as _
 from enumfields.fields import EnumIntegerField
 from functools import lru_cache
+from helsinki_gdpr.models import SerializableMixin
 
 from democracy.enums import Commenting, CommentingMapTools
 
@@ -34,6 +35,10 @@ def everything(self, *args, **kwargs):
         return super().get_queryset().filter(*args, **kwargs)
 
 
+class SerializableBaseModelManager(SerializableMixin.SerializableManager, BaseModelManager):
+    """Add serialization support needed for GDPR API to the base model manager."""
+
+
 class BaseModel(models.Model):
     created_at = models.DateTimeField(
         verbose_name=_("time of creation"), default=timezone.now, editable=False, db_index=True

democracy/models/hearing.py

@@ -8,12 +8,13 @@
 from django.utils.timezone import now
 from django.utils.translation import gettext_lazy as _
 from djgeojson.fields import GeoJSONField
+from helsinki_gdpr.models import SerializableMixin
 from parler.managers import TranslatableQuerySet
 from parler.models import TranslatableModel, TranslatedFields
 from urllib.parse import urljoin
 
 from democracy.enums import InitialSectionType
-from democracy.models.base import BaseModelManager, StringIdBaseModel
+from democracy.models.base import SerializableBaseModelManager, StringIdBaseModel
 from democracy.models.organization import ContactPerson, ContactPersonOrder, Organization
 from democracy.models.project import ProjectPhase
 from democracy.utils.geo import get_geometry_from_geojson
@@ -28,7 +29,14 @@ def filter_by_id_or_slug(self, id_or_slug):
         return self.filter(models.Q(pk=id_or_slug) | models.Q(slug=id_or_slug))
 
 
-class Hearing(StringIdBaseModel, TranslatableModel):
+class Hearing(StringIdBaseModel, TranslatableModel, SerializableMixin):
+    serialize_fields = (
+        {"name": "id"},
+        {"name": "title"},
+        {"name": "geojson"},
+        {"name": "sections"},
+    )
+
     open_at = models.DateTimeField(verbose_name=_("opening time"), default=timezone.now)
     close_at = models.DateTimeField(verbose_name=_("closing time"), default=timezone.now)
     force_closed = models.BooleanField(verbose_name=_("force hearing closed"), default=False)
@@ -77,7 +85,7 @@ class Hearing(StringIdBaseModel, TranslatableModel):
         blank=True,
     )
 
-    objects = BaseModelManager.from_queryset(HearingQueryset)()
+    objects = SerializableBaseModelManager.from_queryset(HearingQueryset)()
     original_manager = models.Manager()
 
     class Meta:

democracy/models/organization.py

@@ -1,12 +1,18 @@
 from django.conf import settings
 from django.db import models
 from django.utils.translation import gettext_lazy as _
+from helsinki_gdpr.models import SerializableMixin
 from parler.models import TranslatableModel, TranslatedFields
 
-from democracy.models.base import StringIdBaseModel
+from democracy.models.base import SerializableBaseModelManager, StringIdBaseModel
 
 
-class Organization(StringIdBaseModel):
+class Organization(StringIdBaseModel, SerializableMixin):
+    serialize_fields = (
+        {"name": "id"},
+        {"name": "name"},
+    )
+
     name = models.CharField(verbose_name=_("name"), max_length=255, unique=True)
     admin_users = models.ManyToManyField(settings.AUTH_USER_MODEL, blank=True, related_name="admin_organizations")
     parent = models.ForeignKey(
@@ -21,6 +27,8 @@ class Organization(StringIdBaseModel):
         ),
     )
 
+    objects = SerializableBaseModelManager()
+
     class Meta:
         verbose_name = _("organization")
         verbose_name_plural = _("organizations")