gitter: Messages, Analysis and Import

.ci/build.sh

@@ -25,6 +25,8 @@ python manage.py migrate
 python manage.py import_contributors_data
 python manage.py import_issues_data
 python manage.py import_merge_requests_data
+python manage.py download_nltk_data
+python manage.py import_messages_data
 python manage.py create_config_data
 python manage.py create_participants
 python manage.py update_participants_data

.gitignore

@@ -268,6 +268,9 @@ tags
 .idea/**/dictionaries
 .idea/**/shelf
 
+# Generated files
+.idea/**/contentModel.xml
+
 # Sensitive or high-churn files
 .idea/**/dataSources/
 .idea/**/dataSources.ids
@@ -483,10 +486,5 @@ cscope.po.out
 _site/
 /public/
 
-# Patch & backup files
-*.patch
-*.orig
-*.diff
-
 # Pytest profile files
 prof/

.moban.yaml

@@ -7,6 +7,7 @@ packages:
   - inactive_issues
   - data
   - gci
+  - gitter
   - gsoc
   - gamification
   - log

.nocover.yaml

@@ -8,6 +8,7 @@ nocover_file_globs:
   - community/git.py
   - gci/*.py
   - gsoc/*.py
+  - gitter/nlp/dict_tagger.py
   - log/*.py
   - meta_review/handler.py
   - model/*.py

community/settings.py

@@ -40,6 +40,7 @@
     'model',
     'gamification',
     'meta_review',
+    'gitter',
     'django.contrib.contenttypes',
     'django.contrib.staticfiles',
     'django_distill',

gamification/migrations/0002_auto_20180811_2208.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.15 on 2018-08-11 22:08
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gamification', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='activity',
+            name='answers_given',
+            field=models.IntegerField(default=0, null=True),
+        ),
+        migrations.AddField(
+            model_name='activity',
+            name='ignore_messages',
+            field=models.IntegerField(default=0, null=True),
+        ),
+        migrations.AddField(
+            model_name='activity',
+            name='questions_asked',
+            field=models.IntegerField(default=0, null=True),
+        ),
+    ]

gamification/models.py

@@ -22,6 +22,15 @@ class Activity(models.Model):
     # The date and time this activity was last updated
     updated_at = models.DateTimeField(null=True)
 
+    # Number of questions asked by the participant
+    questions_asked = models.IntegerField(default=0, null=True)
+
+    # Number of answers given by the participant
+    answers_given = models.IntegerField(default=0, null=True)
+
+    # Number of ignore messages send by the participant
+    ignore_messages = models.IntegerField(default=0, null=True)
+
     def __str__(self):
         return self.name
 
@@ -81,7 +90,8 @@ def __str__(self):
     class Meta:
         ordering = ['-score']
 
-    def add_points(self, points, activity_string, performed_at, updated_at):
+    def add_points(self, points, activity_string, performed_at, updated_at,
+                   questions_asked, answers_given, ignore_messages):
         """
         Update score, level and add activities peformed.
 
@@ -94,23 +104,36 @@ def add_points(self, points, activity_string, performed_at, updated_at):
                                 and time when this activity was performed.
         :param updated_at:      a datetime object representing the date and time when
                                 this activity was performed.
+        :param question_asked:  an integer representing the number of questions asked
+                                by the participant during performing this activity.
+        :param answers_given:   an integer representing the number of answers given
+                                by the participant during performing this activity.
+        :param ignore_messages: an integer representing the number of ignore messages
+                                sent by the participant during performing this activity.
         """
         self.update_score_and_level(points)
         self.add_activity(points,
                           activity_string,
                           performed_at,
                           updated_at,
+                          questions_asked,
+                          answers_given,
+                          ignore_messages,
                           )
 
     def deduct_points(self, points_to_deduct, activity_string,
-                      performed_at, updated_at):
+                      performed_at, updated_at, questions_asked,
+                      answers_given, ignore_messages):
         """
         Deduct points for performing some specific activities.
         """
         self.add_points(points_to_deduct,
                         activity_string,
                         performed_at,
                         updated_at,
+                        questions_asked,
+                        answers_given,
+                        ignore_messages,
                         )
 
     def find_level_for_score(self, score):
@@ -132,7 +155,8 @@ def update_score_and_level(self, points):
         self.level = self.find_level_for_score(self.score)
         self.save()
 
-    def add_activity(self, points, activity_string, performed_at, updated_at):
+    def add_activity(self, points, activity_string, performed_at, updated_at,
+                     questions_asked, answers_given, ignore_messages):
         """
         Add a new activity to the participant.
         """
@@ -141,7 +165,10 @@ def add_activity(self, points, activity_string, performed_at, updated_at):
             points=points,
             performer=self,
             performed_at=performed_at,
-            updated_at=updated_at)
+            updated_at=updated_at,
+            questions_asked=questions_asked,
+            answers_given=answers_given,
+            ignore_messages=ignore_messages)
         self.activities.add(activity)
 
     def find_badges_for_activity(self, activities):

gamification/process/update.py

@@ -10,6 +10,7 @@
 from gamification.labels import NEGATIVE_POINT_LABELS
 from gamification.data.points import MERGE_REQUEST_CLOSED_WITHOUT_MERGE
 from gamification.models import Participant
+from gitter.models import Message
 
 
 def get_mr_objects():
@@ -46,6 +47,18 @@ def update_participants_data_with_mr(mr):
     mr_author = Participant.objects.get(username=mr.author)
     mr_performed_at = mr.created_at
     mr_updated_at = mr.updated_at
+    mr_questions_asked = Message.objects.filter(
+        message_type='question',
+        sent_at__gte=mr_performed_at,
+        sent_at__lte=mr_updated_at).count()
+    mr_answers_given = Message.objects.filter(
+        message_type='answer',
+        sent_at__gte=mr_performed_at,
+        sent_at__lte=mr_updated_at).count()
+    mr_ignore_messages = Message.objects.filter(
+        message_type='ignore',
+        sent_at__gte=mr_performed_at,
+        sent_at__lte=mr_updated_at).count()
 
     if mr.state == 'merged':
         mr_labels = mr.labels.values('name')
@@ -59,6 +72,9 @@ def update_participants_data_with_mr(mr):
                              mr_activity_string,
                              mr_performed_at,
                              mr_updated_at,
+                             mr_questions_asked,
+                             mr_answers_given,
+                             mr_ignore_messages,
                              )
         # Get all the issues this mr is closing
         try:
@@ -70,6 +86,18 @@ def update_participants_data_with_mr(mr):
             issue_labels = issue.labels.values('name')
             issue_performed_at = issue.created_at
             issue_updated_at = issue.updated_at
+            issue_questions_asked = Message.objects.filter(
+                message_type='question',
+                sent_at__gte=issue_performed_at,
+                sent_at__lte=issue_updated_at).count()
+            issue_answers_given = Message.objects.filter(
+                message_type='answer',
+                sent_at__gte=issue_performed_at,
+                sent_at__lte=issue_updated_at).count()
+            issue_ignore_messages = Message.objects.filter(
+                message_type='ignore',
+                sent_at__gte=issue_performed_at,
+                sent_at__lte=issue_updated_at).count()
             # Get activity and points based on the labels on the issue
             issue_points, issue_activity_string = get_activity_with_points(
                 'issue', issue_labels)
@@ -82,6 +110,9 @@ def update_participants_data_with_mr(mr):
                                     issue_activity_string,
                                     issue_performed_at,
                                     issue_updated_at,
+                                    issue_questions_asked,
+                                    issue_answers_given,
+                                    issue_ignore_messages,
                                     )
 
     elif mr.state == 'closed':
@@ -91,6 +122,9 @@ def update_participants_data_with_mr(mr):
                                 mr_activity_string,
                                 mr_performed_at,
                                 mr_updated_at,
+                                mr_questions_asked,
+                                mr_answers_given,
+                                mr_ignore_messages,
                                 )
 
 
@@ -122,10 +156,25 @@ def update_participants_data_with_issue(issue):
         issue_points, issue_activity = get_activity_with_points('issue', labels)
         issue_performed_at = issue.created_at
         issue_updated_at = issue.updated_at
+        issue_questions_asked = Message.objects.filter(
+            message_type='question',
+            sent_at__gte=issue_performed_at,
+            sent_at__lte=issue_updated_at).count()
+        issue_answers_given = Message.objects.filter(
+            message_type='answer',
+            sent_at__gte=issue_performed_at,
+            sent_at__lte=issue_updated_at).count()
+        issue_ignore_messages = Message.objects.filter(
+            message_type='ignore',
+            sent_at__gte=issue_performed_at,
+            sent_at__lte=issue_updated_at).count()
         issue_author.deduct_points(issue_points,
                                    issue_activity,
                                    issue_performed_at,
                                    issue_updated_at,
+                                   issue_questions_asked,
+                                   issue_answers_given,
+                                   issue_ignore_messages,
                                    )
         issue_author.save()
 

gamification/tests/test_management_commands.py

@@ -81,6 +81,7 @@ class UpdateParticipantsTest(TestCase):
     def setUpTestData(cls):
         call_command('import_issues_data')
         call_command('import_merge_requests_data')
+        call_command('import_messages_data')
         call_command('create_config_data')
         call_command('create_participants')
         call_command('update_participants_data')
@@ -98,3 +99,17 @@ def test_command_update_particiapants_data(self):
 
         number_of_badges = participant.badges.all().count()
         self.assertEquals(number_of_badges, 2)
+
+        activity1 = participant.activities.all()[0]
+
+        # Number of questions aksed during performing this
+        # activity should be one
+        self.assertEquals(activity1.questions_asked, 1)
+
+        # Number of answers given during performing this
+        # activity should be one
+        self.assertEquals(activity1.answers_given, 1)
+
+        # Number of ignore messages send during performing this
+        # activity should be one
+        self.assertEquals(activity1.ignore_messages, 1)

gamification/tests/test_models.py

@@ -30,12 +30,21 @@ def test_field_label(self):
             'performed_at').verbose_name
         updated_at = activity._meta.get_field(
             'updated_at').verbose_name
+        questions_asked = activity._meta.get_field(
+            'questions_asked').verbose_name
+        answers_given = activity._meta.get_field(
+            'answers_given').verbose_name
+        ignore_messages = activity._meta.get_field(
+            'ignore_messages').verbose_name
         self.assertEquals(name, 'name')
         self.assertEquals(points, 'points')
         self.assertEquals(number_of_times, 'number of times')
         self.assertEquals(performer, 'performer')
         self.assertEquals(updated_at, 'updated at')
         self.assertEquals(performed_at, 'performed at')
+        self.assertEquals(questions_asked, 'questions asked')
+        self.assertEquals(answers_given, 'answers given')
+        self.assertEquals(ignore_messages, 'ignore messages')
 
     def test_object_name_is_activity_name(self):
         activity = Activity.objects.get(id=1)
@@ -212,10 +221,16 @@ def test_add_points_method(self):
         activity_string = 'Created a difficulty/newcomer bug issue'
         performed_at = '2017-08-24 05:59:31+00:00'
         updated_at = '2018-06-02 17:06:18+00:00'
+        questions_asked = 1
+        answers_given = 1
+        ignore_messages = 1
         participant.add_points(points,
                                activity_string,
                                performed_at,
                                updated_at,
+                               questions_asked,
+                               answers_given,
+                               ignore_messages,
                                )
         self.assertEquals(participant.score, 10)
         self.assertEquals(participant.level.name, 'Beginner-I')
@@ -229,10 +244,16 @@ def test_deduct_points_method(self):
         activity_string = 'Merge request was closed without merge'
         performed_at = '2017-08-24 05:59:31+00:00'
         updated_at = '2018-06-02 17:06:18+00:00'
+        questions_asked = 1
+        answers_given = 1
+        ignore_messages = 1
         participant.deduct_points(points,
                                   activity_string,
                                   performed_at,
                                   updated_at,
+                                  questions_asked,
+                                  answers_given,
+                                  ignore_messages,
                                   )
         self.assertEquals(participant.score, 0)
         self.assertEquals(participant.level.name, 'Fresher')
@@ -277,10 +298,16 @@ def test_add_activity_method(self):
         activity = 'Created a difficulty/newcomer type/bug issue'
         performed_at = '2017-08-24 05:59:31+00:00'
         updated_at = '2018-06-02 17:06:18+00:00'
+        questions_asked = 1
+        answers_given = 1
+        ignore_messages = 1
         participant.add_activity(points,
                                  activity,
                                  performed_at,
                                  updated_at,
+                                 questions_asked,
+                                 answers_given,
+                                 ignore_messages,
                                  )
 
         # After applying add_activity
@@ -294,6 +321,9 @@ def test_add_activity_method(self):
                                  activity,
                                  performed_at,
                                  updated_at,
+                                 questions_asked,
+                                 answers_given,
+                                 ignore_messages,
                                  )
 
         # A new activity is added

gitter/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class GitterConfig(AppConfig):
+    name = 'gitter'

gitter/management/commands/download_nltk_data.py

@@ -0,0 +1,11 @@
+from django.core.management.base import BaseCommand
+
+import nltk
+
+
+class Command(BaseCommand):
+    help = 'Download nltk data'
+
+    def handle(self, *args, **options):
+        nltk.download('averaged_perceptron_tagger')
+        nltk.download('punkt')