feat(gdpr): add gdpr query

Serialize user related objects in the GDPR query operation

- written comments (SectionComment)
- voted comments (SectionComment)
- images related to comments (CommentImage)
- followed hearings (Hearing)
- admin organizations (Organization)

Add test_media folder which gets removed when tests have been run.

Refs: KER-280

democracy/factories/hearing.py

@@ -8,6 +8,7 @@
 from democracy.enums import Commenting
 from democracy.factories.comment import BaseCommentFactory
 from democracy.models import Hearing, Label, Section, SectionComment, SectionType
+from democracy.models.section import CommentImage
 
 LOG = logging.getLogger(__name__)
 
@@ -24,6 +25,15 @@ class Meta:
         model = SectionComment
 
 
+class CommentImageFactory(factory.django.DjangoModelFactory):
+    caption = factory.Faker("word")
+    title = factory.Faker("word")
+    image = factory.django.ImageField()
+
+    class Meta:
+        model = CommentImage
+
+
 class HearingFactory(factory.django.DjangoModelFactory):
     class Meta:
         model = Hearing

democracy/models/base.py

@@ -7,6 +7,7 @@
 from django.utils.translation import gettext_lazy as _
 from enumfields.fields import EnumIntegerField
 from functools import lru_cache
+from helsinki_gdpr.models import SerializableMixin
 
 from democracy.enums import Commenting, CommentingMapTools
 
@@ -34,6 +35,10 @@ def everything(self, *args, **kwargs):
         return super().get_queryset().filter(*args, **kwargs)
 
 
+class SerializableBaseModelManager(SerializableMixin.SerializableManager, BaseModelManager):
+    """Add serialization support needed for GDPR API to the base model manager."""
+
+
 class BaseModel(models.Model):
     created_at = models.DateTimeField(
         verbose_name=_("time of creation"), default=timezone.now, editable=False, db_index=True

democracy/models/hearing.py

@@ -8,12 +8,13 @@
 from django.utils.timezone import now
 from django.utils.translation import gettext_lazy as _
 from djgeojson.fields import GeoJSONField
+from helsinki_gdpr.models import SerializableMixin
 from parler.managers import TranslatableQuerySet
 from parler.models import TranslatableModel, TranslatedFields
 from urllib.parse import urljoin
 
 from democracy.enums import InitialSectionType
-from democracy.models.base import BaseModelManager, StringIdBaseModel
+from democracy.models.base import SerializableBaseModelManager, StringIdBaseModel
 from democracy.models.organization import ContactPerson, ContactPersonOrder, Organization
 from democracy.models.project import ProjectPhase
 from democracy.utils.geo import get_geometry_from_geojson
@@ -28,7 +29,12 @@ def filter_by_id_or_slug(self, id_or_slug):
         return self.filter(models.Q(pk=id_or_slug) | models.Q(slug=id_or_slug))
 
 
-class Hearing(StringIdBaseModel, TranslatableModel):
+class Hearing(StringIdBaseModel, TranslatableModel, SerializableMixin):
+    serialize_fields = (
+        {"name": "id"},
+        {"name": "title"},
+    )
+
     open_at = models.DateTimeField(verbose_name=_("opening time"), default=timezone.now)
     close_at = models.DateTimeField(verbose_name=_("closing time"), default=timezone.now)
     force_closed = models.BooleanField(verbose_name=_("force hearing closed"), default=False)
@@ -77,7 +83,7 @@ class Hearing(StringIdBaseModel, TranslatableModel):
         blank=True,
     )
 
-    objects = BaseModelManager.from_queryset(HearingQueryset)()
+    objects = SerializableBaseModelManager.from_queryset(HearingQueryset)()
     original_manager = models.Manager()
 
     class Meta:

democracy/models/organization.py

@@ -1,12 +1,18 @@
 from django.conf import settings
 from django.db import models
 from django.utils.translation import gettext_lazy as _
+from helsinki_gdpr.models import SerializableMixin
 from parler.models import TranslatableModel, TranslatedFields
 
-from democracy.models.base import StringIdBaseModel
+from democracy.models.base import SerializableBaseModelManager, StringIdBaseModel
 
 
-class Organization(StringIdBaseModel):
+class Organization(StringIdBaseModel, SerializableMixin):
+    serialize_fields = (
+        {"name": "id"},
+        {"name": "name"},
+    )
+
     name = models.CharField(verbose_name=_("name"), max_length=255, unique=True)
     admin_users = models.ManyToManyField(settings.AUTH_USER_MODEL, blank=True, related_name="admin_organizations")
     parent = models.ForeignKey(
@@ -21,6 +27,8 @@ class Organization(StringIdBaseModel):
         ),
     )
 
+    objects = SerializableBaseModelManager()
+
     class Meta:
         verbose_name = _("organization")
         verbose_name_plural = _("organizations")

democracy/models/section.py

@@ -6,12 +6,20 @@
 from django.db import models
 from django.urls import get_resolver
 from django.utils.translation import gettext_lazy as _
+from helsinki_gdpr.models import SerializableMixin
 from parler.managers import TranslatableQuerySet
 from parler.models import TranslatableModel, TranslatedFields
 from reversion import revisions
 
 from democracy.enums import InitialSectionType
-from democracy.models.base import ORDERING_HELP, BaseModel, BaseModelManager, Commentable, StringIdBaseModel
+from democracy.models.base import (
+    ORDERING_HELP,
+    BaseModel,
+    BaseModelManager,
+    Commentable,
+    SerializableBaseModelManager,
+    StringIdBaseModel,
+)
 from democracy.models.comment import BaseComment, recache_on_save
 from democracy.models.files import BaseFile
 from democracy.models.hearing import Hearing
@@ -157,7 +165,22 @@ def __str__(self):
 
 @revisions.register
 @recache_on_save
-class SectionComment(Commentable, BaseComment):
+class SectionComment(Commentable, BaseComment, SerializableMixin):
+    serialize_fields = (
+        {"name": "id"},
+        {"name": "section_id"},
+        {"name": "author_name"},
+        {"name": "comment_id"},
+        {"name": "title"},
+        {"name": "content"},
+        {"name": "published"},
+        {"name": "created_at"},
+        {"name": "modified_at"},
+        {"name": "deleted"},
+        {"name": "deleted_at"},
+        {"name": "images"},
+    )
+
     parent_field = "section"
     parent_model = Section
     section = models.ForeignKey(Section, related_name="comments", on_delete=models.PROTECT)
@@ -180,6 +203,8 @@ class SectionComment(Commentable, BaseComment):
         on_delete=models.SET_NULL,
     )
 
+    objects = SerializableBaseModelManager()
+
     class Meta:
         verbose_name = _("section comment")
         verbose_name_plural = _("section comments")
@@ -296,12 +321,27 @@ class Meta:
         verbose_name_plural = _("section poll answers")
 
 
-class CommentImage(BaseImage):
+class CommentImage(BaseImage, SerializableMixin):
+
+    serialize_fields = (
+        {"name": "id"},
+        {"name": "title"},
+        {"name": "caption"},
+        {"name": "image", "accessor": lambda x: x.url},
+        {"name": "published"},
+        {"name": "created_at"},
+        {"name": "modified_at"},
+        {"name": "deleted"},
+        {"name": "deleted_at"},
+    )
+
     title = models.CharField(verbose_name=_("title"), max_length=255, blank=True, default="")
     caption = models.TextField(verbose_name=_("caption"), blank=True, default="")
     parent_field = "sectioncomment"
     comment = models.ForeignKey(SectionComment, related_name="images", on_delete=models.CASCADE)
 
+    objects = SerializableBaseModelManager()
+
     class Meta:
         verbose_name = _("comment image")
         verbose_name_plural = _("comment images")

kerrokantasi/models.py

@@ -8,9 +8,15 @@ class User(AbstractUser, SerializableMixin):
         {"name": "id"},
         {"name": "uuid"},
         {"name": "username"},
+        {"name": "nickname"},
         {"name": "first_name"},
         {"name": "last_name"},
         {"name": "email"},
+        {"name": "has_strong_auth"},
+        {"name": "sectioncomment_created"},
+        {"name": "voted_democracy_sectioncomment"},
+        {"name": "followed_hearings"},
+        {"name": "admin_organizations"},
     )
 
     nickname = models.CharField(max_length=50, blank=True)

kerrokantasi/tests/gdpr/conftest.py

@@ -1,13 +1,7 @@
 import datetime
 import pytest
-import sys
-import uuid
-from django.conf import settings
-from django.core.signals import setting_changed
-from django.dispatch import receiver
-from django.urls import clear_url_caches
+import shutil
 from helusers.settings import api_token_auth_settings
-from importlib import reload
 from jose import jwt
 from rest_framework.test import APIClient
 
@@ -22,6 +16,15 @@ def api_client():
     return APIClient()
 
 
+@pytest.fixture(autouse=True)
+def setup_test_media(settings):
+    """Create folder for test media/file uploads."""
+    settings.MEDIA_ROOT = "test_media"
+    settings.MEDIA_URL = "/media/"
+    yield
+    shutil.rmtree("test_media", ignore_errors=True)
+
+
 @pytest.fixture
 def single_comment_user():
     user = UserFactory()
@@ -32,11 +35,6 @@ def single_comment_user():
     return user
 
 
-@pytest.fixture
-def uuid_value():
-    return uuid.uuid4()
-
-
 @pytest.fixture
 def user():
     return UserFactory()
@@ -86,27 +84,3 @@ def get_api_token_for_user_with_scopes(user, scopes: list, requests_mock):
     auth_header = f"{api_token_auth_settings.AUTH_SCHEME} {encoded_jwt}"
 
     return auth_header
-
-
-def model_lookup_that_returns_none(model, instance_id):
-    return model.objects.filter(user__uuid=instance_id).first()
-
-
-def model_lookup_that_throws_exception(model, instance_id):
-    return model.objects.get(user__uuid=instance_id)
-
-
-def get_user_from_extra_data(extra_data):
-    return extra_data.profile.user
-
-
-@receiver(setting_changed)
-def _reload_url_conf(setting, **kwargs):
-    if setting == "GDPR_API_URL_PATTERN":
-        clear_url_caches()
-
-        if "helsinki_gdpr.urls" in sys.modules:
-            reload(sys.modules["helsinki_gdpr.urls"])
-
-        if settings.ROOT_URLCONF in sys.modules:
-            reload(sys.modules[settings.ROOT_URLCONF])