apps/ai_reports: connect with external API

- add a celery task which sends a comment to the XAI server and saves
  the response as AiReport
- connect comment post_save signal with celery task
- rename category to label to be in line with the XAI response
- change AiReport fields to JSONField for now
- add tests
- **BREAKING CHANGE** Reset migrations for the ai_reports app (see
  changelog)

adhocracy-plus/config/settings/base.py

@@ -547,3 +547,4 @@
 CELERY_RESULT_BACKEND = "redis://localhost:6379"
 CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True
 CELERY_RESULT_EXTENDED = True
+XAI_API_URL = ""

apps/ai_reports/admin.py

@@ -8,26 +8,26 @@ class AiReportAdmin(admin.ModelAdmin):
     model = AiReport
     list_display = (
         "__str__",
-        "category",
+        "label",
         "confidence",
         "is_pending",
         "comment",
     )
     list_filter = (
         "is_pending",
-        "category",
+        "label",
         "confidence",
         "comment",
     )
     search_fields = (
-        "category",
+        "label",
         "explanation",
         "confidence",
         "is_pending",
     )
     fields = (
         "is_pending",
-        "category",
+        "label",
         "explanation",
         "confidence",
         "comment",

apps/ai_reports/apps.py

@@ -3,3 +3,6 @@
 
 class AiReportsConfig(AppConfig):
     name = "apps.ai_reports"
+
+    def ready(self):
+        import apps.ai_reports.signals  # noqa:F401

apps/ai_reports/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 3.2.19 on 2023-07-27 15:31
+# Generated by Django 3.2.19 on 2024-05-30 14:30
 
 from django.db import migrations, models
 import django.db.models.deletion
@@ -40,9 +40,10 @@ class Migration(migrations.Migration):
                     ),
                 ),
                 ("is_pending", models.BooleanField(default=True)),
-                ("category", models.CharField(max_length=50)),
-                ("explanation", models.TextField()),
-                ("confidence", models.FloatField(default=0)),
+                ("label", models.JSONField()),
+                ("explanation", models.JSONField()),
+                ("confidence", models.JSONField()),
+                ("show_in_discussion", models.BooleanField(default=True)),
                 (
                     "comment",
                     models.OneToOneField(

apps/ai_reports/models.py

@@ -6,9 +6,9 @@
 
 class AiReport(TimeStampedModel):
     is_pending = models.BooleanField(default=True)
-    category = models.CharField(max_length=50)
-    explanation = models.TextField()
-    confidence = models.FloatField(default=0)
+    label = models.JSONField()
+    explanation = models.JSONField()
+    confidence = models.JSONField()
     show_in_discussion = models.BooleanField(default=True)
     comment = models.OneToOneField(
         Comment,

apps/ai_reports/serializers.py

@@ -1,16 +1,24 @@
+import json
+
 from rest_framework import serializers
 
 from apps.ai_reports.models import AiReport
 
 
 class AiReportSerializer(serializers.ModelSerializer):
+    explanation = serializers.SerializerMethodField()
+
     class Meta:
         model = AiReport
         fields = (
-            "category",
+            "label",
             "confidence",
             "explanation",
             "is_pending",
             "comment",
             "show_in_discussion",
         )
+
+    # FIXME: remove once frontend knows what to do with this
+    def get_explanation(self, ai_report: AiReport) -> str:
+        return json.dumps(ai_report.explanation)

apps/ai_reports/signals.py

@@ -0,0 +1,20 @@
+import httpx
+from django.conf import settings
+from django.db.models import signals
+from django.dispatch import receiver
+
+from adhocracy4.comments.models import Comment
+from apps.ai_reports.tasks import get_classification_for_comment
+
+client = httpx.Client()
+
+
+@receiver(signals.post_save, sender=Comment)
+def get_ai_classification(sender, instance, created, update_fields, **kwargs):
+    if getattr(settings, "XAI_API_URL"):
+        comment_text_changed = getattr(instance, "_former_comment") != getattr(
+            instance, "comment"
+        )
+        if created or comment_text_changed:
+            # FIXME: use delay_on_commit() once updated to celery 5.x
+            get_classification_for_comment.delay(instance.pk)

apps/ai_reports/tasks.py

@@ -0,0 +1,58 @@
+import ast
+
+import backoff
+import httpx
+from celery import shared_task
+from django.conf import settings
+
+from adhocracy4.comments.models import Comment
+from apps import logger
+from apps.ai_reports.models import AiReport
+
+client = httpx.Client()
+
+
+@shared_task
+def get_classification_for_comment(comment_pk: int) -> None:
+    try:
+        comment = Comment.objects.get(pk=comment_pk)
+        response = call_ai_api(comment=comment.comment)
+        if response.status_code == 200:
+            extract_and_save_ai_classifications(comment=comment, report=response.json())
+        else:
+            logger.error("Error: XAI server returned %s", response.status_code)
+    except httpx.HTTPError as e:
+        logger.error("Error connecting to %s: %s", settings.AI_API_URL, str(e))
+
+
+def skip_retry(e: Exception) -> bool:
+    if isinstance(e, httpx.HTTPStatusError):
+        return 400 <= e.response.status_code < 500
+    return False
+
+
+@backoff.on_exception(
+    backoff.expo, httpx.HTTPError, max_tries=4, factor=2, giveup=skip_retry
+)
+def call_ai_api(comment: str) -> httpx.Response:
+    response = client.post(
+        settings.XAI_API_URL,
+        json={"comment": comment},
+        headers={"Accept": "application/json", "Content-Type": "application/json"},
+        timeout=25.0,
+    )
+    response.raise_for_status()
+    return response
+
+
+def extract_and_save_ai_classifications(comment: Comment, report: dict) -> None:
+    # FIXME: the data returned from the api is not actually valid json, so we need
+    # to use ast to explicitly convert it. This should be fixed on their side.
+    confidence = ast.literal_eval((report["confidence"]))
+    label = ast.literal_eval(report["label"])
+    explanation = ast.literal_eval(report["explanation"])
+
+    ai_report = AiReport(
+        comment=comment, confidence=confidence, label=label, explanation=explanation
+    )
+    ai_report.save()

changelog/8164.md

@@ -0,0 +1,17 @@
+### Added
+
+- add celery task which sends a comment to the xai and stores the response as
+  AiReport
+- add comment post_save signal which connects comment creation / editing with
+  the xai celery task
+- added tests
+- add tests
+- add pytest-mock package to dev requirements
+- added backup and httpx to fork requirements
+
+### Changed
+
+- rename category to label to be in line with the XAI response
+- change AiReport fields to JSONField for now
+- **BREAKING CHANGE** Reset migrations for the ai_reports app as there's no
+  automatic way to convert the `confidence` field to JSONField

requirements/dev.txt

@@ -13,3 +13,4 @@ pytest==7.3.2
 pytest-cov==4.1.0
 pytest-django==4.5.2
 pytest-factoryboy==2.5.1
+pytest-mock==3.14.0

requirements/fork.txt

@@ -1 +1,3 @@
 # requirements needed in this fork, but not a+
+backoff==2.2.1
+httpx==0.27.0

tests/ai_reports/conftest.py

@@ -1,5 +1,8 @@
 from pytest_factoryboy import register
 
+from tests.ideas import factories as idea_factories
+
 from .factories import AiReportFactory
 
 register(AiReportFactory)
+register(idea_factories.IdeaFactory)

tests/ai_reports/factories.py

@@ -10,8 +10,8 @@ class AiReportFactory(factory.django.DjangoModelFactory):
     class Meta:
         model = AiReport
 
-    category = factory.Faker("word")
-    explanation = factory.Faker("sentence", nb_words=6)
-    confidence = random.random()
+    label = factory.List([factory.Faker("word") for _ in range(3)])
+    explanation = {"xai explanation": [["first word", 0.0123]]}
+    confidence = [random.random() for _ in range(3)]
     comment = factory.SubFactory(CommentFactory)
     show_in_discussion = True

tests/ai_reports/test_classification_signals.py

@@ -0,0 +1,51 @@
+import pytest
+from django.db.models import signals
+from django.test import override_settings
+
+from adhocracy4.comments.models import Comment
+from apps.ai_reports.signals import get_ai_classification
+
+
+@pytest.mark.django_db
+def test_comment_not_sent_to_xai_no_url(mocker, idea, comment_factory, caplog):
+    task_mock = mocker.patch(
+        "apps.ai_reports.tasks.get_classification_for_comment" ".delay"
+    )
+    assert get_ai_classification in signals.post_save._live_receivers(Comment)
+    comment_factory(content_object=idea, comment="lala")
+    task_mock.assert_not_called()
+
+
+@override_settings(XAI_API_URL="https://liqd.net")
+@pytest.mark.django_db
+def test_comment_sent_to_xai_on_comment_text_change(mocker, idea, comment_factory):
+    task_mock = mocker.patch(
+        "apps.ai_reports.tasks.get_classification_for_comment" ".delay"
+    )
+
+    assert get_ai_classification in signals.post_save._live_receivers(Comment)
+    comment = comment_factory(content_object=idea, comment="lala")
+    task_mock.assert_called_once_with(comment.pk)
+    task_mock.reset_mock()
+
+    comment.comment = "modified comment"
+    comment.save()
+    task_mock.assert_called_once_with(comment.pk)
+
+
+@override_settings(XAI_API_URL="https://liqd.net")
+@pytest.mark.django_db
+def test_comment_not_sent_to_xai_without_comment_text_change(
+    mocker, idea, comment_factory, caplog
+):
+    task_mock = mocker.patch(
+        "apps.ai_reports.tasks.get_classification_for_comment" ".delay"
+    )
+    assert get_ai_classification in signals.post_save._live_receivers(Comment)
+    comment = comment_factory(content_object=idea, comment="lala")
+    task_mock.assert_called_once_with(comment.pk)
+    task_mock.reset_mock()
+
+    comment.is_blocked = True
+    comment.save()
+    task_mock.assert_not_called()

tests/moderatorfeedback/test_moderator_comment_feedback_api.py

@@ -1,3 +1,5 @@
+import json
+
 import pytest
 from django.urls import reverse
 
@@ -33,7 +35,9 @@ def test_moderator_and_ai_report_added_in_comment(
     )
     assert comment.ai_report.explanation == ai_report.explanation
 
-    assert ai_report.explanation == response.data["ai_report"]["explanation"]
+    assert ai_report.explanation == json.loads(
+        response.data["ai_report"]["explanation"]
+    )
     assert response.status_code == 200
 
 

tests/userdashboard/test_serializers.py

@@ -1,3 +1,4 @@
+import json
 from datetime import timedelta
 
 import pytest
@@ -63,8 +64,8 @@ def test_num_reports(
                 comment["ai_report"]
                 and comment["ai_report"]["comment"] == ai.comment.pk
             ):
-                assert comment["ai_report"]["category"] == ai.category
-                assert comment["ai_report"]["explanation"] == ai.explanation
+                assert comment["ai_report"]["label"] == ai.label
+                assert json.loads(comment["ai_report"]["explanation"]) == ai.explanation
                 assert comment["ai_report"]["confidence"] == ai.confidence
                 assert comment["ai_report"]["is_pending"] == ai.is_pending