KER-280 | GDPR additions and changes

democracy/models/gdpr_data_serialization_mixin.py

@@ -0,0 +1,14 @@
+from kerrokantasi.utils import get_current_request
+
+
+class FileFieldUrlSerializerMixin:
+    field_to_use_as_url_field = None  # Override in subclass.
+
+    @property
+    def url(self):
+        field = getattr(self, self.field_to_use_as_url_field, None)
+
+        if request := get_current_request():
+            return request.build_absolute_uri(field.url)
+
+        return field.url

democracy/models/section.py

@@ -22,11 +22,11 @@
 )
 from democracy.models.comment import BaseComment, recache_on_save
 from democracy.models.files import BaseFile
+from democracy.models.gdpr_data_serialization_mixin import FileFieldUrlSerializerMixin
 from democracy.models.hearing import Hearing
 from democracy.models.images import BaseImage
 from democracy.models.poll import BasePoll, BasePollAnswer, BasePollOption, poll_option_recache_on_save
 from democracy.plugins import get_implementation
-from democracy.utils.file_to_base64 import file_to_base64
 
 CLOSURE_INFO_ORDERING = -10000
 
@@ -141,13 +141,15 @@ def plugin_implementation(self):
         return get_implementation(self.plugin_identifier)
 
 
-class SectionImage(BaseImage, TranslatableModel, SerializableMixin):
+class SectionImage(BaseImage, TranslatableModel, SerializableMixin, FileFieldUrlSerializerMixin):
+    field_to_use_as_url_field = "image"
+
     serialize_fields = (
         {"name": "id"},
         {"name": "title"},
         {"name": "caption"},
         {"name": "alt_text"},
-        {"name": "image", "accessor": lambda x: file_to_base64(x)},
+        {"name": "url"},
         {"name": "published"},
         {"name": "created_at"},
         {"name": "modified_at"},
@@ -170,12 +172,14 @@ class Meta:
         ordering = ("ordering",)
 
 
-class SectionFile(BaseFile, TranslatableModel, SerializableMixin):
+class SectionFile(BaseFile, TranslatableModel, SerializableMixin, FileFieldUrlSerializerMixin):
+    field_to_use_as_url_field = "file"
+
     serialize_fields = (
         {"name": "id"},
         {"name": "title"},
         {"name": "caption"},
-        {"name": "file", "accessor": lambda x: file_to_base64(x)},
+        {"name": "url"},
         {"name": "published"},
         {"name": "created_at"},
         {"name": "modified_at"},
@@ -383,12 +387,14 @@ class Meta:
         verbose_name_plural = _("section poll answers")
 
 
-class CommentImage(BaseImage, SerializableMixin):
+class CommentImage(BaseImage, SerializableMixin, FileFieldUrlSerializerMixin):
+    field_to_use_as_url_field = "image"
+
     serialize_fields = (
         {"name": "id"},
         {"name": "title"},
         {"name": "caption"},
-        {"name": "image", "accessor": lambda x: file_to_base64(x)},
+        {"name": "url"},
         {"name": "published"},
         {"name": "created_at"},
         {"name": "modified_at"},

kerrokantasi/gdpr.py

@@ -1,9 +1,12 @@
+import threading
 from django.contrib.auth import get_user_model
 from helsinki_gdpr.types import ErrorResponse
 from typing import Optional
 
 from democracy.models import SectionComment
 
+_thread_locals = threading.local()
+
 
 def get_user(user: get_user_model()) -> get_user_model():
     """
@@ -37,3 +40,27 @@ def delete_data(user: get_user_model(), dry_run: bool) -> Optional[ErrorResponse
     SectionComment.objects.filter(created_by=user).update(author_name=None)
 
     return None
+
+
+class CurrentRequestMiddleware:
+    """
+    Middleware to store the current request in a thread_locals.
+
+    In the GDPR API user data fetch the files (images & files) need to contain the absolute URL to the file.
+    """
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        _thread_locals.request = request
+
+        try:
+            response = self.get_response(request)
+        except Exception:
+            _thread_locals.request = None
+            raise
+
+        _thread_locals.request = None
+
+        return response

kerrokantasi/models.py

@@ -13,8 +13,8 @@ class User(AbstractUser, SerializableMixin):
         {"name": "last_name"},
         {"name": "email"},
         {"name": "has_strong_auth"},
-        {"name": "sectioncomment_created"},
-        {"name": "voted_democracy_sectioncomment"},
+        {"name": "sectioncomments"},
+        {"name": "voted_sectioncomments"},
         {"name": "followed_hearings"},
         {"name": "admin_organizations"},
         {"name": "hearing_created"},
@@ -23,6 +23,15 @@ class User(AbstractUser, SerializableMixin):
     nickname = models.CharField(max_length=50, blank=True)
     has_strong_auth = models.BooleanField(default=False)
 
+    # Properties for GDPR api serialization
+    @property
+    def sectioncomments(self):
+        return [s.serialize() for s in self.sectioncomment_created.everything().iterator()]
+
+    @property
+    def voted_sectioncomments(self):
+        return [s.serialize() for s in self.voted_democracy_sectioncomment.everything().iterator()]
+
     def __str__(self):
         return " - ".join([super().__str__(), self.get_display_name(), self.email])
 

kerrokantasi/settings/base.py

@@ -217,6 +217,7 @@ def get_git_revision_hash():
     "django.contrib.messages.middleware.MessageMiddleware",
     "django.middleware.clickjacking.XFrameOptionsMiddleware",
     "social_django.middleware.SocialAuthExceptionMiddleware",
+    "kerrokantasi.gdpr.CurrentRequestMiddleware",
 ]
 
 # django-extensions is a set of developer friendly tools

kerrokantasi/tests/gdpr/test_gdpr_query.py

@@ -19,7 +19,6 @@
 from democracy.factories.user import UserFactory
 from democracy.models import Hearing, Organization, SectionComment, SectionPollAnswer
 from democracy.models.section import CommentImage, Section, SectionFile, SectionImage, SectionPoll, SectionPollOption
-from democracy.utils.file_to_base64 import file_to_base64
 from kerrokantasi.tests.gdpr.conftest import get_api_token_for_user_with_scopes
 
 User = get_user_model()
@@ -32,13 +31,19 @@ def do_query(user, id_value, scopes=(settings.GDPR_API_QUERY_SCOPE,)):
         auth_header = get_api_token_for_user_with_scopes(user, scopes, req_mock)
         api_client.credentials(HTTP_AUTHORIZATION=auth_header)
 
-        return api_client.get(reverse("helsinki_gdpr:gdpr_v1", kwargs={settings.GDPR_API_MODEL_LOOKUP: id_value}))
+        response = api_client.get(reverse("helsinki_gdpr:gdpr_v1", kwargs={settings.GDPR_API_MODEL_LOOKUP: id_value}))
+
+        return response
 
 
 def _format_datetime(dt):
     return dt.isoformat().replace("+00:00", "Z") if dt else dt
 
 
+def _get_full_url(url):
+    return f"http://testserver{url}"
+
+
 def _get_section_poll_option_data(section_poll_option: SectionPollOption) -> dict:
     return {
         "key": "SECTIONPOLLOPTION",
@@ -75,7 +80,7 @@ def _get_section_image_data(section_image: SectionImage) -> dict:
             {"key": "TITLE", "value": section_image.title},
             {"key": "CAPTION", "value": section_image.caption},
             {"key": "ALT_TEXT", "value": section_image.alt_text},
-            {"key": "IMAGE", "value": file_to_base64(section_image.image.file)},
+            {"key": "URL", "value": _get_full_url(section_image.image.url)},
             {"key": "PUBLISHED", "value": section_image.published},
             {"key": "CREATED_AT", "value": _format_datetime(section_image.created_at)},
             {"key": "MODIFIED_AT", "value": _format_datetime(section_image.modified_at)},
@@ -92,7 +97,7 @@ def _get_section_file_data(section_file: SectionFile) -> dict:
             {"key": "ID", "value": section_file.id},
             {"key": "TITLE", "value": section_file.title},
             {"key": "CAPTION", "value": section_file.caption},
-            {"key": "FILE", "value": file_to_base64(section_file.file.file)},
+            {"key": "URL", "value": _get_full_url(section_file.url)},
             {"key": "PUBLISHED", "value": section_file.published},
             {"key": "CREATED_AT", "value": _format_datetime(section_file.created_at)},
             {"key": "MODIFIED_AT", "value": _format_datetime(section_file.modified_at)},
@@ -147,7 +152,7 @@ def _get_comment_image_data(comment_image: CommentImage) -> dict:
             {"key": "ID", "value": comment_image.id},
             {"key": "TITLE", "value": comment_image.title},
             {"key": "CAPTION", "value": comment_image.caption},
-            {"key": "IMAGE", "value": file_to_base64(comment_image.image.file)},
+            {"key": "URL", "value": _get_full_url(comment_image.image.url)},
             {"key": "PUBLISHED", "value": comment_image.published},
             {"key": "CREATED_AT", "value": _format_datetime(comment_image.created_at)},
             {"key": "MODIFIED_AT", "value": _format_datetime(comment_image.modified_at)},
@@ -203,12 +208,14 @@ def _get_user_data(user: User) -> List[dict]:
         {"key": "EMAIL", "value": user.email},
         {"key": "HAS_STRONG_AUTH", "value": user.has_strong_auth},
         {
-            "key": "SECTIONCOMMENT_CREATED",
-            "children": [_get_section_comment_data(comment) for comment in user.sectioncomment_created.all()],
+            "key": "SECTIONCOMMENTS",
+            "value": [_get_section_comment_data(comment) for comment in user.sectioncomment_created.everything()],
         },
         {
-            "key": "VOTED_DEMOCRACY_SECTIONCOMMENT",
-            "children": [_get_section_comment_data(comment) for comment in user.voted_democracy_sectioncomment.all()],
+            "key": "VOTED_SECTIONCOMMENTS",
+            "value": [
+                _get_section_comment_data(comment) for comment in user.voted_democracy_sectioncomment.everything()
+            ],
         },
         {
             "key": "FOLLOWED_HEARINGS",
@@ -244,6 +251,12 @@ def test_get_user_information_from_gdpr_api(user, geojson_feature):
     comment_voted = SectionCommentFactory(section=section)
     comment_voted.voters.add(user)
     SectionPollAnswer.objects.create(comment=comment, created_by=user, option=poll_option)
+
+    deleted_comment = SectionCommentFactory(
+        section=section, created_by=user, author_name="Name of the Author", geojson=geojson_feature
+    )
+    deleted_comment.soft_delete()
+
     CommentImageFactory(comment=comment, created_by=user)
     hearing_followed = HearingFactory()
     hearing_followed.followers.add(user)
@@ -273,3 +286,14 @@ def test_gdpr_api_requires_authentication(api_client, user):
     response = api_client.get(reverse("helsinki_gdpr:gdpr_v1", kwargs={settings.GDPR_API_MODEL_LOOKUP: user.uuid}))
 
     assert response.status_code == status.HTTP_401_UNAUTHORIZED
+
+
+@pytest.mark.django_db
+def test_request_is_not_found_thread_locals(user, api_client):
+    from kerrokantasi.gdpr import _thread_locals
+
+    response = do_query(user, user.uuid)
+
+    assert response.status_code == status.HTTP_200_OK
+
+    assert getattr(_thread_locals, "request", None) is None

kerrokantasi/utils.py

@@ -0,0 +1,4 @@
+def get_current_request():
+    from kerrokantasi.gdpr import _thread_locals
+
+    return getattr(_thread_locals, "request", None)