Merge pull request #3835 from mozilla/akismet-payload-1259173

Bug 1259173 - Adjust Akismet payload

kuma/wiki/admin.py

@@ -9,8 +9,9 @@
 from django.contrib.admin.views.decorators import staff_member_required
 from django.http import HttpResponse, HttpResponseRedirect
 from django.template.response import TemplateResponse
-from django.template.defaultfilters import truncatechars
+from django.template.defaultfilters import linebreaksbr, truncatechars
 from django.utils import timezone
+from django.utils.encoding import smart_text
 from django.utils.html import escape
 from django.utils.safestring import mark_safe
 from django.utils.text import Truncator
@@ -342,6 +343,8 @@ class DocumentTagAdmin(admin.ModelAdmin):
 class DocumentZoneAdmin(admin.ModelAdmin):
     raw_id_fields = ('document',)
 
+SUBMISSION_NOT_AVAILABLE = 'Akismet submission not available.'
+
 
 @admin.register(DocumentSpamAttempt)
 class DocumentSpamAttemptAdmin(admin.ModelAdmin):
@@ -437,13 +440,8 @@ def save_model(self, request, obj, form, change):
                 self.message_user(request, message, level=messages.INFO)
         obj.save()
 
-    SUBMISSION_NOT_AVAILABLE = 'Akismet submission not available.'
-
     def submitted_data(self, instance):
-        if instance.data:
-            return json.dumps(json.loads(instance.data), indent=4)
-        else:
-            return self.SUBMISSION_NOT_AVAILABLE
+        return instance.data or SUBMISSION_NOT_AVAILABLE
 
 
 @admin.register(Revision)
@@ -460,9 +458,14 @@ class RevisionAdmin(admin.ModelAdmin):
 
 @admin.register(RevisionIP)
 class RevisionIPAdmin(admin.ModelAdmin):
-    readonly_fields = ('revision', 'ip')
+    readonly_fields = ('revision', 'ip', 'user_agent', 'referrer',
+                       'submitted_data')
     list_display = ('revision', 'ip')
 
+    def submitted_data(self, obj):
+        """Display Akismet data, if saved at edit time."""
+        return obj.data or SUBMISSION_NOT_AVAILABLE
+
 
 @admin.register(RevisionAkismetSubmission)
 class RevisionAkismetSubmissionAdmin(DisabledDeletionMixin, admin.ModelAdmin):
@@ -483,10 +486,13 @@ def get_fields(self, request, obj=None):
 
     def revision_with_link(self, obj):
         """Admin link to the revision"""
-        admin_link = reverse('admin:wiki_revision_change',
-                             args=[obj.revision.id])
-        return ('<a target="_blank" href="%s">%s</a>' %
-                (admin_link, escape(obj.revision)))
+        if obj.revision_id:
+            admin_link = reverse('admin:wiki_revision_change',
+                                 args=[obj.revision.id])
+            return ('<a target="_blank" href="%s">%s</a>' %
+                    (admin_link, escape(obj.revision)))
+        else:
+            return 'None'
     revision_with_link.allow_tags = True
     revision_with_link.short_description = "Revision"
 
@@ -518,6 +524,40 @@ def __new__(cls, *args, **kwargs):
 
         return AdminFormWithRequest
 
+    def add_view(self, request, form_url='', extra_context=None):
+        extra_context = extra_context or {}
+        extra_context['submitted_data'] = self.submitted_data(request)
+        return super(RevisionAkismetSubmissionAdmin, self).add_view(
+            request, form_url, extra_context=extra_context)
+
+    def change_view(self, request, object_id, form_url='', extra_context=None):
+        extra_context = extra_context or {}
+        extra_context['submitted_data'] = self.submitted_data(
+            request, object_id)
+        return super(RevisionAkismetSubmissionAdmin, self).change_view(
+            request, object_id, form_url, extra_context=extra_context)
+
+    def submitted_data(self, request, obj_id=None):
+        """Display Akismet data, if saved at edit time."""
+        if obj_id:
+            obj = RevisionAkismetSubmission.objects.get(id=obj_id)
+        else:
+            obj = None
+
+        if obj and obj.revision_id:
+            revision_ip = obj.revision.revisionip_set.first()
+        else:
+            revision_id = request.GET.get('revision')
+            if revision_id:
+                revision_ip = RevisionIP.objects.filter(
+                    revision_id=revision_id).first()
+            else:
+                revision_ip = None
+        if revision_ip and revision_ip.data:
+            return linebreaksbr(smart_text(revision_ip.data))
+        else:
+            return SUBMISSION_NOT_AVAILABLE
+
 
 @admin.register(EditorToolbar)
 class EditorToolbarAdmin(admin.ModelAdmin):

kuma/wiki/forms.py

@@ -274,11 +274,16 @@ def __init__(self, revision, request=None):
         super(AkismetHistoricalData, self).__init__()
         revision_ip = revision.revisionip_set.first()
         if revision_ip:
-            self.parameters.update({
-                'user_ip': revision_ip.ip,
-                'user_agent': revision_ip.user_agent,
-                'referrer': revision_ip.referrer,
-            })
+            if revision_ip.data:
+                # Use captured Akismet submission
+                self.parameters = json.loads(revision_ip.data)
+                return
+            else:
+                self.parameters.update({
+                    'user_ip': revision_ip.ip,
+                    'user_agent': revision_ip.user_agent,
+                    'referrer': revision_ip.referrer,
+                })
         else:
             self.parameters.update({
                 'user_ip': '0.0.0.0',
@@ -687,7 +692,7 @@ def akismet_error(self, parameters, exception=None):
                 slug=self.cleaned_data['slug'],
                 user=self.request.user,
                 document=document,
-                data=json.dumps(dsa_params),
+                data=json.dumps(dsa_params, indent=2, sort_keys=True),
                 review=review
             )
         finally:
@@ -757,6 +762,8 @@ def save(self, document, **kwargs):
                 RevisionIP.objects.log(
                     revision=new_rev,
                     headers=self.request.META,
+                    data=json.dumps(self.akismet_parameters(),
+                                    indent=2, sort_keys=True)
                 )
 
             # send first edit emails

kuma/wiki/managers.py

@@ -235,7 +235,7 @@ def delete_old(self, days=30):
         old_rev_ips = self.filter(revision__created__lte=cutoff_date)
         old_rev_ips.delete()
 
-    def log(self, revision, headers):
+    def log(self, revision, headers, data):
         """
         Records the IP and some more data for the given revision and the
         request headers.
@@ -245,4 +245,5 @@ def log(self, revision, headers):
             ip=headers.get('REMOTE_ADDR'),
             user_agent=headers.get('HTTP_USER_AGENT', ''),
             referrer=headers.get('HTTP_REFERER', ''),
+            data=data
         )

kuma/wiki/models.py

@@ -1838,13 +1838,12 @@ class RevisionIP(models.Model):
         editable=False,
         blank=True,
     )
-    # TODO: Uncomment this after production database has the column
-    # data = models.TextField(
-    #    editable=False,
-    #    blank=True,
-    #    null=True,
-    #    verbose_name=_('Data submitted to Akismet')
-    # )
+    data = models.TextField(
+        editable=False,
+        blank=True,
+        null=True,
+        verbose_name=_('Data submitted to Akismet')
+    )
     objects = RevisionIPManager()
 
     def __unicode__(self):

kuma/wiki/templates/admin/wiki/revisionakismetsubmission/change_form.html

@@ -11,3 +11,16 @@ <h1>{% trans "Akismet submission details" %}</h1>
     {% endif %}
     {{ block.super }}
 {% endblock %}
+
+{% block after_field_sets %}
+{# Add submitted data, styled almost like it was a real field #}
+<fieldset class="module aligned">
+  <div class="form-row field-submitted-data">
+    <div>
+        <label>Submitted Data:</label>
+        <p>{{ submitted_data }}</p>
+    </div>
+  </div>
+</fieldset>
+{{ block.super }}
+{% endblock %}

kuma/wiki/tests/test_admin.py

@@ -15,8 +15,9 @@
 from kuma.spam.constants import HAM_URL, SPAM_SUBMISSIONS_FLAG, SPAM_URL, VERIFY_URL
 from kuma.users.tests import UserTestCase
 from kuma.users.models import User
-from kuma.wiki.admin import DocumentSpamAttemptAdmin
-from kuma.wiki.models import DocumentSpamAttempt, RevisionAkismetSubmission
+from kuma.wiki.admin import DocumentSpamAttemptAdmin, SUBMISSION_NOT_AVAILABLE
+from kuma.wiki.models import (DocumentSpamAttempt, RevisionAkismetSubmission,
+                              RevisionIP)
 from kuma.wiki.tests import document, revision
 
 
@@ -94,13 +95,10 @@ def test_doc_short_long_unicode(self):
 
     def test_submitted_data(self):
         dsa = DocumentSpamAttempt(data=None)
-        expected = self.admin.SUBMISSION_NOT_AVAILABLE
-        assert self.admin.submitted_data(dsa) == expected
-        dsa.data = '{"foo": "bar"}'
-        assert self.admin.submitted_data(dsa) == (
-            '{\n'
-            '    "foo": "bar"\n'
-            '}')
+        assert self.admin.submitted_data(dsa) == SUBMISSION_NOT_AVAILABLE
+        data = '{"foo": "bar"}'
+        dsa.data = data
+        assert self.admin.submitted_data(dsa) == data
 
     def assert_needs_review(self):
         dsa = DocumentSpamAttempt.objects.get()
@@ -312,3 +310,66 @@ def test_spam_submission_tags(self, mock_requests):
             'Orange'
         )
         self.assertEqual(submitted_data['comment_content'], expected_content)
+
+    def test_create_no_revision(self):
+        url = urlparams(
+            reverse('admin:wiki_revisionakismetsubmission_add'),
+            type='ham',
+        )
+        self.client.login(username='admin', password='testpass')
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200)
+        self.assertIn(SUBMISSION_NOT_AVAILABLE, response.content)
+
+    def test_view_change_existing(self):
+        admin = User.objects.get(username='admin')
+        flag, created = Flag.objects.get_or_create(name=SPAM_SUBMISSIONS_FLAG)
+        flag.users.add(admin)
+        revision = admin.created_revisions.all()[0]
+        submission = RevisionAkismetSubmission.objects.create(
+            sender=admin, revision=revision, type='ham')
+
+        self.client.login(username='admin', password='testpass')
+        url = reverse('admin:wiki_revisionakismetsubmission_change',
+                      args=(submission.id,))
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200)
+        self.assertIn(SUBMISSION_NOT_AVAILABLE, response.content)
+
+    def test_view_change_with_data(self):
+        admin = User.objects.get(username='admin')
+        flag, created = Flag.objects.get_or_create(name=SPAM_SUBMISSIONS_FLAG)
+        flag.users.add(admin)
+        revision = admin.created_revisions.all()[0]
+        submission = RevisionAkismetSubmission.objects.create(
+            sender=admin, revision=revision, type='spam')
+        RevisionIP.objects.create(revision=revision,
+                                  data='{"content": "spam"}')
+
+        self.client.login(username='admin', password='testpass')
+        url = reverse('admin:wiki_revisionakismetsubmission_change',
+                      args=(submission.id,))
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200)
+        self.assertIn('{"content": "spam"}',
+                      response.content)
+
+    def test_view_changelist_existing(self):
+        admin = User.objects.get(username='admin')
+        flag, created = Flag.objects.get_or_create(name=SPAM_SUBMISSIONS_FLAG)
+        flag.users.add(admin)
+        revision = admin.created_revisions.all()[0]
+        RevisionAkismetSubmission.objects.create(sender=admin,
+                                                 revision=revision,
+                                                 type='ham')
+        RevisionAkismetSubmission.objects.create(sender=admin,
+                                                 type='ham')
+
+        self.client.login(username='admin', password='testpass')
+        url = reverse('admin:wiki_revisionakismetsubmission_changelist')
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200)
+        revision_url = reverse('admin:wiki_revision_change',
+                               args=[revision.id])
+        self.assertIn(revision_url, response.content)
+        self.assertIn('None', response.content)

kuma/wiki/tests/test_forms.py

@@ -15,11 +15,81 @@
 from kuma.users.tests import UserTestCase, UserTransactionTestCase
 
 from ..constants import SPAM_EXEMPTED_FLAG, SPAM_TRAINING_FLAG
-from ..forms import RevisionForm, TreeMoveForm
-from ..models import DocumentSpamAttempt, Revision
+from ..forms import AkismetHistoricalData, RevisionForm, TreeMoveForm
+from ..models import DocumentSpamAttempt, Revision, RevisionIP
 from ..tests import document, normalize_html, revision
 
 
+class AkismetHistoricalDataTests(UserTestCase):
+    """Tests for AkismetHistoricalData."""
+    rf = RequestFactory()
+    base_akismet_payload = {
+        'blog_charset': 'UTF-8',
+        'blog_lang': u'en_us',
+        'comment_author': u'Test User',
+        'comment_author_email': u'testuser@test.com',
+        'comment_content': (
+            'Sample\n'
+            'SampleSlug\n'
+            'content\n'
+            'Comment'
+        ),
+        'comment_type': 'wiki-revision',
+        'referrer': '',
+        'user_agent': '',
+        'user_ip': '0.0.0.0'
+    }
+
+    def setUp(self):
+        super(AkismetHistoricalDataTests, self).setUp()
+        self.user = self.user_model.objects.get(username='testuser')
+        self.revision = revision(save=True, content='content', title='Sample',
+                                 slug='SampleSlug', comment='Comment',
+                                 summary='', tags='')
+
+    def test_no_revision_ip_no_request(self):
+        """
+        Test Akismet payload with no RevisionIP or request.
+
+        This is a possible payload from ./manage.py submit_deleted_documents.
+        """
+        params = AkismetHistoricalData(self.revision).parameters
+        assert params == self.base_akismet_payload
+
+    def test_revision_ip_no_data(self):
+        """
+        Test Akismet payload with a RevisionIP without data.
+
+        This is a possible payload from an April 2016 revision.
+        """
+        RevisionIP.objects.create(revision=self.revision, ip='127.0.0.1',
+                                  user_agent='Agent', referrer='Referrer')
+        request = self.rf.get('/en-US/dashboard/revisions')
+        params = AkismetHistoricalData(self.revision, request).parameters
+        expected = self.base_akismet_payload.copy()
+        expected.update({
+            'blog': 'http://testserver/',
+            'permalink': 'http://testserver/en-US/docs/SampleSlug',
+            'referrer': 'Referrer',
+            'user_agent': 'Agent',
+            'user_ip': '127.0.0.1',
+        })
+        assert params == expected
+
+    def test_revision_ip_with_data(self):
+        """
+        Test Akismet payload is the data from the RevisionIP.
+
+        This payload is from a revision after April 2016.
+        """
+        RevisionIP.objects.create(revision=self.revision, ip='127.0.0.1',
+                                  user_agent='Agent', referrer='Referrer',
+                                  data='{"content": "spammy"}')
+        request = self.rf.get('/en-US/dashboard/revisions')
+        params = AkismetHistoricalData(self.revision, request).parameters
+        assert params == {'content': 'spammy'}
+
+
 class RevisionFormTests(UserTransactionTestCase):
     """
     Generic tests for RevisionForm.