Rework tagging infrastructure

Solve getpatchwork#113 and getpatchwork#57 GitHub issues, fix up returning tags in the API,
keep track of tag origin to later be able to add tags to comments in
the API.

Use relations Tag-Patch and Tag-CoverLetter to avoid duplication of
tags for each patch in series, and use `series` attribute of
SubmissionTag as a notion of a tag which is related to each patch in the
series (because it comes from cover letter or it's comments)

Signed-off-by: Veronika Kabatova <vkabatov@redhat.com>

docs/usage/overview.rst

@@ -119,10 +119,11 @@ one delegate can be assigned to a patch.
 Tags
 ~~~~
 
-Tags are specially formatted metadata appended to the foot the body of a patch
-or a comment on a patch. Patchwork extracts these tags at parse time and
-associates them with the patch. You add extra tags to an email by replying to
-the email. The following tags are available on a standard Patchwork install:
+Tags are specially formatted metadata appended to the foot the body of a patch,
+cover letter or a comment related to them. Patchwork extracts these tags at
+parse time and associates them with patches. You add extra tags to an email by
+replying to the email. The following tags are available on a standard Patchwork
+install:
 
 ``Acked-by:``
   For example::

patchwork/api/comment.py

@@ -26,18 +26,34 @@
 from patchwork.api.base import PatchworkPermission
 from patchwork.api.embedded import PersonSerializer
 from patchwork.models import Comment
+from patchwork.models import SubmissionTag
 
 
 class CommentListSerializer(BaseHyperlinkedModelSerializer):
 
     subject = SerializerMethodField()
     headers = SerializerMethodField()
     submitter = PersonSerializer(read_only=True)
+    tags = SerializerMethodField()
 
     def get_subject(self, comment):
         return email.parser.Parser().parsestr(comment.headers,
                                               True).get('Subject', '')
 
+    def get_tags(self, instance):
+        if not instance.submission.project.use_tags:
+            return {}
+
+        tags = SubmissionTag.objects.filter(
+            comment=instance
+        ).values_list('tag__name', 'value')
+
+        result = {}
+        for name, value in tags:
+            result.setdefault(name, []).append(value)
+
+        return result
+
     def get_headers(self, comment):
         headers = {}
 
@@ -55,7 +71,7 @@ def get_headers(self, comment):
     class Meta:
         model = Comment
         fields = ('id', 'msgid', 'date', 'subject', 'submitter', 'content',
-                  'headers')
+                  'headers', 'tags')
         read_only_fields = fields
 
 

patchwork/api/cover.py

@@ -30,6 +30,7 @@
 from patchwork.api.embedded import ProjectSerializer
 from patchwork.api.embedded import SeriesSerializer
 from patchwork.models import CoverLetter
+from patchwork.models import SubmissionTag
 
 
 class CoverLetterListSerializer(BaseHyperlinkedModelSerializer):
@@ -39,6 +40,7 @@ class CoverLetterListSerializer(BaseHyperlinkedModelSerializer):
     mbox = SerializerMethodField()
     series = SeriesSerializer(many=True, read_only=True)
     comments = SerializerMethodField()
+    tags = SerializerMethodField()
 
     def get_mbox(self, instance):
         request = self.context.get('request')
@@ -48,13 +50,26 @@ def get_comments(self, cover):
         return self.context.get('request').build_absolute_uri(
             reverse('api-cover-comment-list', kwargs={'pk': cover.id}))
 
+    def get_tags(self, instance):
+        if not instance.project.use_tags:
+            return {}
+
+        tags = SubmissionTag.objects.filter(
+            submission=instance).values_list('tag__name', 'value').distinct()
+
+        result = {}
+        for name, value in tags:
+            result.setdefault(name, []).append(value)
+
+        return result
+
     class Meta:
         model = CoverLetter
         fields = ('id', 'url', 'project', 'msgid', 'date', 'name', 'submitter',
-                  'mbox', 'series', 'comments')
+                  'mbox', 'series', 'comments', 'tags')
         read_only_fields = fields
         versioned_fields = {
-            '1.1': ('mbox', 'comments'),
+            '1.1': ('mbox', 'comments', 'tags'),
         }
         extra_kwargs = {
             'url': {'view_name': 'api-cover-detail'},

patchwork/api/filters.py

@@ -21,9 +21,11 @@
 from django.core.exceptions import ValidationError
 from django.db.models import Q
 from django_filters.rest_framework import FilterSet
+from django_filters import Filter
 from django_filters import IsoDateTimeFilter
 from django_filters import ModelMultipleChoiceFilter
 from django.forms import ModelMultipleChoiceField as BaseMultipleChoiceField
+from django.forms.widgets import HiddenInput
 from django.forms.widgets import MultipleHiddenInput
 
 from patchwork.models import Bundle
@@ -35,6 +37,7 @@
 from patchwork.models import Project
 from patchwork.models import Series
 from patchwork.models import State
+from patchwork.models import SubmissionTag
 
 
 # custom fields, filters
@@ -136,6 +139,60 @@ class StateFilter(ModelMultipleChoiceFilter):
     field_class = StateChoiceField
 
 
+class TagNameFilter(Filter):
+
+    def filter(self, qs, tag_name):
+        submissions_series = SubmissionTag.objects.filter(
+            tag__name__iexact=tag_name
+        ).values_list('submission__id', 'series')
+
+        submission_list = []
+        series_list = []
+        for submission, series in submissions_series:
+            submission_list.append(submission)
+            series_list.append(series)
+
+        return qs.filter(Q(id__in=submission_list) | Q(series__in=series_list))
+
+
+class TagValueFilter(Filter):
+
+    def filter(self, qs, tag_value):
+        submissions_series = SubmissionTag.objects.filter(
+            value__icontains=tag_value
+        ).values_list('submission__id', 'series')
+
+        submission_list = []
+        series_list = []
+        for submission, series in submissions_series:
+            submission_list.append(submission)
+            series_list.append(series)
+
+        return qs.filter(Q(id__in=submission_list) | Q(series__in=series_list))
+
+
+class TagFilter(Filter):
+
+    def filter(self, qs, query):
+        try:
+            tag_name, tag_value = query.split(',', 1)
+        except ValueError:
+            raise ValidationError('Query in format `tag=name,value` expected!')
+
+        submissions_series = SubmissionTag.objects.filter(
+            tag__name__iexact=tag_name,
+            value__icontains=tag_value
+        ).values_list('submission__id', 'series')
+
+        submission_list = []
+        series_list = []
+        for submission, series in submissions_series:
+            submission_list.append(submission)
+            series_list.append(series)
+
+        return qs.filter(Q(id__in=submission_list) | Q(series__in=series_list))
+
+
 class UserChoiceField(ModelMultipleChoiceField):
 
     alternate_lookup = 'username__iexact'
@@ -173,10 +230,14 @@ class CoverLetterFilterSet(TimestampMixin, FilterSet):
     series = BaseFilter(queryset=Project.objects.all(),
                         widget=MultipleHiddenInput)
     submitter = PersonFilter(queryset=Person.objects.all())
+    tagname = TagNameFilter(label='Tag name')
+    tagvalue = TagValueFilter(widget=HiddenInput)
+    tag = TagFilter(widget=HiddenInput)
 
     class Meta:
         model = CoverLetter
-        fields = ('project', 'series', 'submitter')
+        fields = ('project', 'series', 'submitter', 'tagname', 'tagvalue',
+                  'tag')
 
 
 class PatchFilterSet(TimestampMixin, FilterSet):
@@ -189,11 +250,14 @@ class PatchFilterSet(TimestampMixin, FilterSet):
     submitter = PersonFilter(queryset=Person.objects.all())
     delegate = UserFilter(queryset=User.objects.all())
     state = StateFilter(queryset=State.objects.all())
+    tagname = TagNameFilter(label='Tag name')
+    tagvalue = TagValueFilter(widget=HiddenInput)
+    tag = TagFilter(widget=HiddenInput)
 
     class Meta:
         model = Patch
         fields = ('project', 'series', 'submitter', 'delegate',
-                  'state', 'archived')
+                  'state', 'archived', 'tagname', 'tagvalue', 'tag')
 
 
 class CheckFilterSet(TimestampMixin, FilterSet):

patchwork/api/patch.py

@@ -19,6 +19,7 @@
 
 import email.parser
 
+from django.db.models import Q
 from django.utils.translation import ugettext_lazy as _
 from rest_framework.generics import ListAPIView
 from rest_framework.generics import RetrieveUpdateAPIView
@@ -35,6 +36,7 @@
 from patchwork.api.embedded import UserSerializer
 from patchwork.models import Patch
 from patchwork.models import State
+from patchwork.models import SubmissionTag
 from patchwork.parser import clean_subject
 
 
@@ -104,9 +106,18 @@ def get_checks(self, instance):
             reverse('api-check-list', kwargs={'patch_id': instance.id}))
 
     def get_tags(self, instance):
-        # TODO(stephenfin): Make tags performant, possibly by reworking the
-        # model
-        return {}
+        if not instance.project.use_tags:
+            return {}
+
+        tags = SubmissionTag.objects.filter(
+            Q(submission=instance) | Q(series=instance.series.all())
+        ).values_list('tag__name', 'value').distinct()
+
+        result = {}
+        for name, value in tags:
+            result.setdefault(name, []).append(value)
+
+        return result
 
     class Meta:
         model = Patch
@@ -123,6 +134,9 @@ class Meta:
         extra_kwargs = {
             'url': {'view_name': 'api-patch-detail'},
         }
+        versioned_fields = {
+            '1.1': ('tags', ),
+        }
 
 
 class PatchDetailSerializer(PatchListSerializer):
@@ -155,6 +169,7 @@ class Meta:
             'headers', 'content', 'diff', 'prefixes')
         versioned_fields = PatchListSerializer.Meta.versioned_fields
         extra_kwargs = PatchListSerializer.Meta.extra_kwargs
+        versioned_fields = PatchListSerializer.Meta.versioned_fields
 
 
 class PatchList(ListAPIView):

patchwork/management/commands/retag.py

@@ -19,11 +19,13 @@
 
 from django.core.management.base import BaseCommand
 
+from patchwork.models import Cover
 from patchwork.models import Patch
+from patchwork.models import SeriesPatch
 
 
 class Command(BaseCommand):
-    help = 'Update the tag (Ack/Review/Test) counts on existing patches'
+    help = 'Update tags on existing patches'
     args = '[<patch_id>...]'
 
     def handle(self, *args, **options):
@@ -37,7 +39,17 @@ def handle(self, *args, **options):
         count = query.count()
 
         for i, patch in enumerate(query.iterator()):
-            patch.refresh_tag_counts()
+            patch.refresh_tags()
+            for comment in patch.comments.all():
+                comment.refresh_tags()
+
+            series_patches = SeriesPatch.objects.filter(patch_id=patch.id)
+            for series_patch in series_patches:
+                cover = series_patch.series.cover_letter
+                cover.refresh_tags()
+                for comment in cover.comments.all():
+                    comment.refresh_tags()
+
             if (i % 10) == 0:
                 self.stdout.write('%06d/%06d\r' % (i, count), ending='')
                 self.stdout.flush()