Merge d1fd4dc into bf3daf2

ephios/core/forms/users.py

@@ -341,6 +341,9 @@ def save(self, commit=True):
                 Q(id__in=self.cleaned_data["groups"]) | Q(id__in=(g.id for g in self.locked_groups))
             )
         )
+        # if the user is re-activated after the email has been deemed invalid, reset the flag
+        if userprofile.is_active and userprofile.email_invalid:
+            userprofile.email_invalid = False
         userprofile.save()
         return userprofile
 
@@ -449,21 +452,33 @@ def __init__(self, *args, **kwargs):
         self.user = kwargs.pop("user")
         super().__init__(*args, **kwargs)
 
-        preferences = self.user.preferences["notifications__notifications"]
+        self.all_backends = {backend.slug for backend in enabled_notification_backends()}
         for notification_type in enabled_notification_types():
             if notification_type.unsubscribe_allowed:
                 self.fields[notification_type.slug] = MultipleChoiceField(
                     label=notification_type.title,
                     choices=[
                         (backend.slug, backend.title) for backend in enabled_notification_backends()
                     ],
-                    initial=preferences.get(notification_type.slug, {}),
+                    initial=list(
+                        self.all_backends
+                        - {
+                            backend
+                            for backend, notificaton_type in self.user.disabled_notifications
+                            if notificaton_type == notification_type.slug
+                        }
+                    ),
                     widget=CheckboxSelectMultiple,
                     required=False,
                 )
 
-    def update_preferences(self):
-        self.user.preferences["notifications__notifications"] = self.cleaned_data
+    def save_preferences(self):
+        disabled_notifications = []
+        for notification_type, preferred_backends in self.cleaned_data.items():
+            for backend in self.all_backends - set(preferred_backends):
+                disabled_notifications.append([backend, notification_type])
+        self.user.disabled_notifications = disabled_notifications
+        self.user.save()
 
 
 class UserOwnDataForm(ModelForm):

ephios/core/migrations/0025_remove_notification_failed_notification_processed_by_and_more.py

@@ -0,0 +1,43 @@
+# Generated by Django 4.2.6 on 2023-11-28 22:10
+
+from django.db import migrations, models
+
+import ephios.extra.json
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0024_identityprovider_create_missing_groups_and_more"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="notification",
+            name="failed",
+        ),
+        migrations.AddField(
+            model_name="notification",
+            name="processed_by",
+            field=models.JSONField(
+                blank=True,
+                decoder=ephios.extra.json.CustomJSONDecoder,
+                default=list,
+                encoder=ephios.extra.json.CustomJSONEncoder,
+                help_text="List of slugs of notification backends that have processed this notification",
+            ),
+        ),
+        migrations.AddField(
+            model_name="notification",
+            name="processing_completed",
+            field=models.BooleanField(
+                default=False,
+                verbose_name="processing completed",
+                help_text="All enabled notification backends have processed this notification when flag is set",
+            ),
+        ),
+        migrations.AddField(
+            model_name="notification",
+            name="read",
+            field=models.BooleanField(default=False, verbose_name="read"),
+        ),
+    ]

ephios/core/migrations/0026_userprofile_disabled_notifications_and_more.py

@@ -0,0 +1,81 @@
+# Generated by Django 4.2.7 on 2024-01-06 13:21
+from json import JSONDecodeError
+
+from django.db import migrations, models
+
+import ephios.extra.json
+from ephios.core.services.notifications.backends import enabled_notification_backends
+from ephios.core.services.notifications.types import notification_type_from_slug
+from ephios.extra.preferences import JSONSerializer
+
+
+def migrate_disabled_notifications(apps, schema_editor):
+    UserProfile = apps.get_model("core", "UserProfile")
+    UserPreferenceModel = apps.get_model("dynamic_preferences_users", "UserPreferenceModel")
+    db_alias = schema_editor.connection.alias
+    backends = [backend.slug for backend in enabled_notification_backends()]
+    for profile in UserProfile.all_objects.using(db_alias).all():
+        try:
+            preferences = JSONSerializer.deserialize(
+                UserPreferenceModel.objects.using(db_alias)
+                .get(section="notifications", name="notifications", instance__pk=profile.pk)
+                .raw_value
+            )
+        except (UserPreferenceModel.DoesNotExist, ValueError, JSONDecodeError):
+            continue
+        for notification_type, active_backends in preferences.items():
+            for backend in set(backends) - set(active_backends):
+                profile.disabled_notifications.append([backend, notification_type])
+        profile.save()
+
+
+def revert_disabled_notifications(apps, schema_editor):
+    UserProfile = apps.get_model("core", "UserProfile")
+    UserPreferenceModel = apps.get_model("dynamic_preferences_users", "UserPreferenceModel")
+    db_alias = schema_editor.connection.alias
+    backends = [backend.slug for backend in enabled_notification_backends()]
+    for profile in UserProfile.all_objects.using(db_alias).all():
+        try:
+            preferences_instance = UserPreferenceModel.objects.using(db_alias).get(
+                section="notifications", name="notifications", instance__pk=profile.pk
+            )
+            preferences = JSONSerializer.deserialize(preferences_instance.raw_value)
+            for disabled_tuple in profile.disabled_notifications:
+                backend_slug, type_slug = disabled_tuple
+                try:
+                    notification_type_from_slug(type_slug)
+                except ValueError:
+                    continue
+                enabled = preferences.get(type_slug, backends)
+                if backend_slug in enabled:
+                    enabled.remove(backend_slug)
+                preferences[type_slug] = enabled
+            preferences_instance.raw_value = JSONSerializer.serialize(preferences)
+            preferences_instance.save()
+        except (UserPreferenceModel.DoesNotExist, JSONDecodeError):
+            continue
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0025_remove_notification_failed_notification_processed_by_and_more"),
+        ("dynamic_preferences_users", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="userprofile",
+            name="disabled_notifications",
+            field=models.JSONField(
+                decoder=ephios.extra.json.CustomJSONDecoder,
+                default=list,
+                encoder=ephios.extra.json.CustomJSONEncoder,
+            ),
+        ),
+        migrations.RunPython(migrate_disabled_notifications, revert_disabled_notifications),
+        migrations.AddField(
+            model_name="userprofile",
+            name="email_invalid",
+            field=models.BooleanField(default=False, verbose_name="Email address invalid"),
+        ),
+    ]

ephios/core/models/users.py

@@ -20,6 +20,7 @@
     ExpressionWrapper,
     F,
     ForeignKey,
+    JSONField,
     Max,
     Model,
     Q,
@@ -96,6 +97,7 @@ def get_queryset(self):
 
 class UserProfile(guardian.mixins.GuardianUserMixin, PermissionsMixin, AbstractBaseUser):
     email = EmailField(_("email address"), unique=True)
+    email_invalid = BooleanField(default=False, verbose_name=_("Email address invalid"))
     is_active = BooleanField(default=True, verbose_name=_("Active"))
     is_visible = BooleanField(default=True, verbose_name=_("Visible"))
     is_staff = BooleanField(
@@ -112,6 +114,9 @@ class UserProfile(guardian.mixins.GuardianUserMixin, PermissionsMixin, AbstractB
         default=settings.LANGUAGE_CODE,
         choices=settings.LANGUAGES,
     )
+    disabled_notifications = JSONField(
+        default=list, encoder=CustomJSONEncoder, decoder=CustomJSONDecoder
+    )
 
     USERNAME_FIELD = "email"
     REQUIRED_FIELDS = [
@@ -490,7 +495,21 @@ class Notification(Model):
         verbose_name=_("affected user"),
         null=True,
     )
-    failed = models.BooleanField(default=False, verbose_name=_("failed"))
+    read = models.BooleanField(default=False, verbose_name=_("read"))
+    processing_completed = models.BooleanField(
+        default=False,
+        verbose_name=_("processing completed"),
+        help_text=_(
+            "All enabled notification backends have processed this notification when flag is set"
+        ),
+    )
+    processed_by = models.JSONField(
+        blank=True,
+        default=list,
+        encoder=CustomJSONEncoder,
+        decoder=CustomJSONDecoder,
+        help_text=_("List of slugs of notification backends that have processed this notification"),
+    )
     data = models.JSONField(
         blank=True, default=dict, encoder=CustomJSONEncoder, decoder=CustomJSONDecoder
     )
@@ -512,6 +531,13 @@ def body(self):
         """The body text of the notification."""
         return self.notification_type.get_body(self)
 
+    @property
+    def is_obsolete(self):
+        return self.notification_type.is_obsolete(self)
+
+    def __str__(self):
+        return _("{subject} for {user}").format(subject=self.subject, user=self.user or _("Guest"))
+
     def as_html(self):
         """The notification rendered as HTML."""
         return self.notification_type.as_html(self)
@@ -521,7 +547,7 @@ def as_plaintext(self):
         return self.notification_type.as_plaintext(self)
 
     def get_actions(self):
-        return self.notification_type.get_actions(self)
+        return self.notification_type.get_actions_with_referrer(self)
 
 
 class IdentityProvider(Model):

ephios/core/services/notifications/backends.py

@@ -1,9 +1,13 @@
 import logging
 import smtplib
 import traceback
+import uuid
 from email.utils import formataddr
+from typing import Iterable
 
 from django.conf import settings
+from django.core.cache import cache
+from django.core.exceptions import ObjectDoesNotExist
 from django.core.mail import mail_admins
 from django.utils.translation import gettext_lazy as _
 from webpush import send_user_notification
@@ -30,28 +34,44 @@ def enabled_notification_backends():
 
 
 def send_all_notifications():
-    for backend in installed_notification_backends():
-        for notification in Notification.objects.filter(failed=False):
-            if backend.can_send(notification) and backend.user_prefers_sending(notification):
-                with language((notification.user and notification.user.preferred_language) or None):
-                    try:
-                        backend.send(notification)
-                    except Exception as e:  # pylint: disable=broad-except
-                        if settings.DEBUG:
-                            raise e
-                        notification.failed = True
-                        notification.save()
-                        try:
-                            mail_admins(
-                                "Notification sending failed",
-                                f"Notification: {notification}\nException: {e}\n{traceback.format_exc()}",
-                            )
-                        except smtplib.SMTPConnectError:
-                            pass  # if the mail backend threw this, mail admin will probably throw this as well
-                        logger.warning(
-                            f"Notification sending failed for notification object #{notification.pk} ({notification}) for backend {backend} with {e}"
-                        )
-    Notification.objects.filter(failed=False).delete()
+    CACHE_LOCK_KEY = "notification_sending_running"
+    if cache.get(CACHE_LOCK_KEY):
+        return
+    cache.set(CACHE_LOCK_KEY, str(uuid.uuid4()), timeout=1800)
+    backends = set(installed_notification_backends())
+
+    for backend in backends:
+        unprocessed_notifications = [
+            notification
+            for notification in Notification.objects.filter(processing_completed=False).order_by(
+                "created_at"
+            )
+            if backend.slug not in notification.processed_by
+        ]
+        try:
+            backend.send_multiple(unprocessed_notifications)
+        except Exception as e:  # pylint: disable=broad-except
+            if settings.DEBUG:
+                raise e
+            try:
+                mail_admins(
+                    "Notification sending failed",
+                    f"Exception: {e}\n{traceback.format_exc()}",
+                )
+            except smtplib.SMTPConnectError:
+                pass  # if the mail backend threw this, mail admin will probably throw this as well
+            logger.warning(f"Notification sending failed with {e}")
+    mark_complete_processing(backends)
+    cache.delete(CACHE_LOCK_KEY)
+
+
+def mark_complete_processing(backends):
+    backend_slugs = {b.slug for b in backends}
+    for notification in Notification.objects.filter(processing_completed=False):
+        # can't check with __contains as that is not supported by Sqlite
+        if set(notification.processed_by) == backend_slugs:
+            notification.processing_completed = True
+            notification.save()
 
 
 class AbstractNotificationBackend:
@@ -64,20 +84,47 @@ def title(self):
         return NotImplementedError
 
     @classmethod
-    def can_send(cls, notification):
+    def sending_possible(cls, notification):
         return notification.user is not None
 
+    @classmethod
+    def should_send(cls, notification):
+        return (
+            cls.sending_possible(notification)
+            and cls.user_prefers_sending(notification)
+            and not (notification.read or notification.is_obsolete)
+        )
+
     @classmethod
     def user_prefers_sending(cls, notification):
-        if notification.notification_type.unsubscribe_allowed and notification.user is not None:
-            if not notification.user.is_active:
-                return False
-            backends = notification.user.preferences["notifications__notifications"].get(
-                notification.slug
-            )
-            if backends is not None:
-                return cls.slug in backends
-        return True
+        if not notification.user:
+            return True
+        if not notification.user.is_active:
+            return False
+        if (
+            acting_user := notification.data.get("acting_user", None)
+        ) and acting_user == notification.user:
+            return False
+        if not notification.notification_type.unsubscribe_allowed:
+            return True
+        return [cls.slug, notification.slug] not in notification.user.disabled_notifications
+
+    @classmethod
+    def send_multiple(cls, notifications: Iterable[Notification]):
+        to_delete = []
+        for notification in notifications:
+            try:
+                if cls.should_send(notification):
+                    with language(
+                        (notification.user and notification.user.preferred_language) or None
+                    ):
+                        cls.send(notification)
+            except ObjectDoesNotExist:
+                to_delete.append(notification.pk)
+                continue
+            notification.processed_by.append(cls.slug)
+            notification.save()
+        Notification.objects.filter(pk__in=to_delete).delete()
 
     @classmethod
     def send(cls, notification: Notification):
@@ -89,7 +136,7 @@ class EmailNotificationBackend(AbstractNotificationBackend):
     title = _("via email")
 
     @classmethod
-    def can_send(cls, notification):
+    def sending_possible(cls, notification):
         return notification.user is not None or "email" in notification.data
 
     @classmethod