Initial mattermost notification system

udata/core/discussions/models.py

@@ -20,6 +20,12 @@ class Message(SpamMixin, db.EmbeddedDocument):
     def attributes_to_check_for_spam(self):
         return [self.content]
 
+    def spam_report_title(self):
+        return self._instance.title
+
+    def spam_report_link(self):
+        return self._instance.external_url
+
 
 class Discussion(SpamMixin, db.Document):
     user = db.ReferenceField('User')
@@ -59,6 +65,12 @@ def external_url(self):
             _anchor='discussion-{id}'.format(id=self.id),
             _external=True)
 
+    def spam_report_title(self):
+        return self.title
+
+    def spam_report_link(self):
+        return self.external_url
+
     @spam_protected()
     def signal_new(self):
         on_new_discussion.send(self)

udata/core/spam/models.py

@@ -3,6 +3,7 @@
 from langdetect import detect
 
 from udata.models import db
+from .signals import on_new_potential_spam
 
 NOT_CHECKED = 'not_checked'
 POTENTIEL_SPAM = 'potentiel_spam'
@@ -58,13 +59,15 @@ def detect_spam(self):
                 for word in SpamMixin.spam_words():
                     if word in text.lower():
                         self.spam.status = POTENTIEL_SPAM
+                        on_new_potential_spam.send(title=self.spam_report_title(), link=self.spam_report_link(), text=text, reason=f"contains spam words {word}")
                         return
 
                 # Language detection is not working well with texts of a few words.
                 if SpamMixin.allowed_langs() and len(text) > 30:
                     lang = detect(text)
                     if lang not in SpamMixin.allowed_langs():
                         self.spam.status = POTENTIEL_SPAM
+                        on_new_potential_spam.send(title=self.spam_report_title(), link=self.spam_report_link(), text=text, reason="not allowed lang " + lang)
                         return
 
         for embed in self.embeds_to_check_for_spam():
@@ -92,6 +95,12 @@ def attributes_to_check_for_spam(self):
 
     def embeds_to_check_for_spam(self):
         return []
+
+    def spam_report_title(self):
+        return f"{type(self).__name__}#{self.id}"
+
+    def spam_report_link(self):
+        return None
 
 def spam_protected(get_model_to_check=None):
     """

udata/core/spam/signals.py

@@ -0,0 +1,6 @@
+from blinker import Namespace
+
+namespace = Namespace()
+
+#: Trigerred when a spam is detected
+on_new_potential_spam = namespace.signal('on-new-potential-spam')

udata/notifications/mattermost.py

@@ -0,0 +1,29 @@
+import requests
+from udata.tasks import connect
+from udata.core.spam.signals import on_new_potential_spam
+from flask import current_app
+
+@connect(on_new_potential_spam)
+def notify_potential_spam(title, link, text, reason):
+    message = ':warning: @all Spam potentiel sur \n'
+    if link:
+        message += f'[{title}]({link})'
+    else:
+        message += title
+
+    message += f' ({reason})\n\n'
+    message += f'> {text}'
+
+    send_message(message)
+
+
+def send_message(text: str):
+    """Send a message to a mattermost channel
+
+    Args:
+        text (str): Text to send to a channel
+    """
+    data = {'text': text}
+
+    r = requests.post(current_app.config.get('MATTERMOST_WEBHOOK', []), json=data)
+    r.raise_for_status()

udata/tests/test_discussions.py

@@ -1,7 +1,6 @@
 from datetime import datetime
 
 from flask import url_for
-from udata.core.spam.models import SpamMixin
 import pytest
 
 from udata.models import Dataset, Member
@@ -12,6 +11,7 @@
     on_new_discussion, on_new_discussion_comment,
     on_discussion_closed, on_discussion_deleted,
 )
+from udata.core.spam.signals import on_new_potential_spam
 from udata.core.discussions.tasks import (
     notify_new_discussion, notify_new_discussion_comment,
     notify_discussion_closed
@@ -73,15 +73,16 @@ def test_spam_in_new_discussion_title(self):
         dataset = Dataset.objects.create(title='Test dataset')
 
         with assert_not_emit(on_new_discussion):
-            response = self.post(url_for('api.discussions'), {
-                'title': 'spam and blah',
-                'comment': 'bla bla',
-                'subject': {
-                    'class': 'Dataset',
-                    'id': dataset.id,
-                }
-            })
-            self.assertStatus(response, 201)
+            with assert_emit(on_new_potential_spam):
+                response = self.post(url_for('api.discussions'), {
+                    'title': 'spam and blah',
+                    'comment': 'bla bla',
+                    'subject': {
+                        'class': 'Dataset',
+                        'id': dataset.id,
+                    }
+                })
+                self.assertStatus(response, 201)
 
         discussions = Discussion.objects(subject=dataset)
         self.assertEqual(len(discussions), 1)
@@ -101,15 +102,16 @@ def test_spam_in_new_discussion_comment(self):
         dataset = Dataset.objects.create(title='Test dataset')
 
         with assert_not_emit(on_new_discussion):
-            response = self.post(url_for('api.discussions'), {
-                'title': 'title and blah',
-                'comment': 'bla bla spam',
-                'subject': {
-                    'class': 'Dataset',
-                    'id': dataset.id,
-                }
-            })
-            self.assertStatus(response, 201)
+            with assert_emit(on_new_potential_spam):
+                response = self.post(url_for('api.discussions'), {
+                    'title': 'title and blah',
+                    'comment': 'bla bla spam',
+                    'subject': {
+                        'class': 'Dataset',
+                        'id': dataset.id,
+                    }
+                })
+                self.assertStatus(response, 201)
 
         discussions = Discussion.objects(subject=dataset)
         self.assertEqual(len(discussions), 1)
@@ -388,10 +390,11 @@ def test_add_spam_comment_to_discussion(self):
 
         self.login()
         with assert_not_emit(on_new_discussion_comment):
-            response = self.post(url_for('api.discussion', id=discussion.id), {
-                'comment': 'spam new bla bla'
-            })
-            self.assert200(response)
+            with assert_emit(on_new_potential_spam):
+                response = self.post(url_for('api.discussion', id=discussion.id), {
+                    'comment': 'spam new bla bla'
+                })
+                self.assert200(response)
 
         discussion.reload()
         self.assertFalse(discussion.is_spam())
@@ -461,11 +464,12 @@ def test_close_discussion_with_spam(self):
         on_new_discussion.send(discussion)  # Updating metrics.
 
         with assert_not_emit(on_discussion_closed):
-            response = self.post(url_for('api.discussion', id=discussion.id), {
-                'comment': 'This is a suspicious, real suspicious message in english.',
-                'close': True,
-            })
-            self.assert200(response)
+            with assert_emit(on_new_potential_spam):
+                response = self.post(url_for('api.discussion', id=discussion.id), {
+                    'comment': 'This is a suspicious, real suspicious message in english.',
+                    'close': True,
+                })
+                self.assert200(response)
 
         discussion.reload()
         self.assertFalse(discussion.is_spam())