Add patch tag infrastructure

This change add patch 'tags', eg 'Acked-by' / 'Reviewed-by', etc., to
patchwork.

Tag parsing is implemented in the patch parser's extract_tags function,
which returns a Counter object of the tags in a comment. These are
stored in the PatchTag (keyed to Tag) objects associated with each
patch.

We need to ensure that the main patch lists do not cause per-patch
queries on the Patch.tags ManyToManyField (this would result in ~500
queries per page), so we introduce a new QuerySet (and Manager) for
Patch, adding a with_tag_counts() method to populate the tag counts in a
single query.

As users may be migrating from previous patchwork versions (ie, with no
tag counts in the database), we add a 'retag' management command.

Signed-off-by: Jeremy Kerr <jk@ozlabs.org>

Author: jk-ozlabs

docs/INSTALL

@@ -148,7 +148,12 @@ in brackets):
 
      PYTHONPATH=lib/python ./manage.py collectstatic
 
-    and add privileges for your mail and web users. This is only needed if
+    If you'd like to use the default tag set (Acked-by, Reviewed-by and
+    Tested-by), then load these default tags:
+
+     PYTHONPATH=lib/python ./manage.py loaddata default_tags
+
+    Finally, add privileges for your mail and web users. This is only needed if
     you use the ident-based approach. If you use password-based database
     authentication, you can skip this step.
 

lib/sql/grant-all.mysql.sql

@@ -23,13 +23,17 @@ GRANT SELECT, UPDATE, INSERT, DELETE ON patchwork_bundlepatch TO 'www-data'@loca
 GRANT SELECT, UPDATE, INSERT, DELETE ON patchwork_patch TO 'www-data'@localhost;
 GRANT SELECT, UPDATE, INSERT, DELETE ON patchwork_emailoptout TO 'www-data'@localhost;
 GRANT SELECT, UPDATE, INSERT, DELETE ON patchwork_patchchangenotification TO 'www-data'@localhost;
+GRANT SELECT, UPDATE, INSERT, DELETE ON patchwork_tag TO 'www-data'@localhost;
+GRANT SELECT, UPDATE, INSERT, DELETE ON patchwork_patchtag TO 'www-data'@localhost;
 
 -- allow the mail user (in this case, 'nobody') to add patches
 GRANT INSERT, SELECT ON patchwork_patch TO 'nobody'@localhost;
 GRANT INSERT, SELECT ON patchwork_comment TO 'nobody'@localhost;
 GRANT INSERT, SELECT ON patchwork_person TO 'nobody'@localhost;
+GRANT INSERT, SELECT, UPDATE, DELETE ON patchwork_patchtag TO 'nobody'@localhost;
 GRANT SELECT ON	patchwork_project TO 'nobody'@localhost;
 GRANT SELECT ON patchwork_state TO 'nobody'@localhost;
+GRANT SELECT ON patchwork_tag TO 'nobody'@localhost;
 
 COMMIT;
 

lib/sql/grant-all.postgres.sql

@@ -23,7 +23,9 @@ GRANT SELECT, UPDATE, INSERT, DELETE ON
 	patchwork_bundlepatch,
 	patchwork_patch,
 	patchwork_emailoptout,
-	patchwork_patchchangenotification
+	patchwork_patchchangenotification,
+	patchwork_tag,
+	patchwork_patchtag
 TO "www-data";
 GRANT SELECT, UPDATE ON
 	auth_group_id_seq,
@@ -44,7 +46,9 @@ GRANT SELECT, UPDATE ON
 	patchwork_state_id_seq,
 	patchwork_emailconfirmation_id_seq,
 	patchwork_userprofile_id_seq,
-	patchwork_userprofile_maintainer_projects_id_seq
+	patchwork_userprofile_maintainer_projects_id_seq,
+	patchwork_tag_id_seq,
+	patchwork_patchtag_id_seq
 TO "www-data";
 
 -- allow the mail user (in this case, 'nobody') to add patches
@@ -53,14 +57,19 @@ GRANT INSERT, SELECT ON
 	patchwork_comment,
 	patchwork_person
 TO "nobody";
+GRANT INSERT, SELECT, UPDATE, DELETE ON
+	patchwork_patchtag
+TO "nobody";
 GRANT SELECT ON
 	patchwork_project,
-	patchwork_state
+	patchwork_state,
+	patchwork_tag
 TO "nobody";
 GRANT UPDATE, SELECT ON
 	patchwork_patch_id_seq,
 	patchwork_person_id_seq,
-	patchwork_comment_id_seq
+	patchwork_comment_id_seq,
+	patchwork_patchtag_id_seq
 TO "nobody";
 
 COMMIT;

lib/sql/migration/015-add-patch-tags.sql

@@ -0,0 +1,19 @@
+BEGIN;
+ALTER TABLE patchwork_project ADD COLUMN use_tags boolean default true;
+
+CREATE TABLE "patchwork_tag" (
+    "id" serial NOT NULL PRIMARY KEY,
+    "name" varchar(20) NOT NULL,
+    "pattern" varchar(50) NOT NULL,
+    "abbrev" varchar(2) NOT NULL UNIQUE
+);
+
+CREATE TABLE "patchwork_patchtag" (
+    "id" serial NOT NULL PRIMARY KEY,
+    "patch_id" integer NOT NULL,
+    "tag_id" integer NOT NULL REFERENCES "patchwork_tag" ("id"),
+    "count" integer NOT NULL,
+    UNIQUE ("patch_id", "tag_id")
+);
+
+COMMIT;

patchwork/admin.py

@@ -1,6 +1,6 @@
 from django.contrib import admin
 from patchwork.models import Project, Person, UserProfile, State, Patch, \
-         Comment, Bundle
+         Comment, Bundle, Tag
 
 class ProjectAdmin(admin.ModelAdmin):
     list_display = ('name', 'linkname','listid', 'listemail')
@@ -48,3 +48,8 @@ class BundleAdmin(admin.ModelAdmin):
     list_filter = ('public', 'project')
     search_fields = ('name', 'owner')
 admin.site.register(Bundle, BundleAdmin)
+
+class TagAdmin(admin.ModelAdmin):
+    list_display = ('name',)
+admin.site.register(Tag, TagAdmin)
+

patchwork/fixtures/default_tags.xml

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="utf-8"?>
+<django-objects version="1.0">
+  <object pk="1" model="patchwork.tag">
+    <field type="CharField" name="name">Acked-by</field>
+    <field type="CharField" name="pattern">^Acked-by:</field>
+    <field type="CharField" name="abbrev">A</field>
+  </object>
+  <object pk="2" model="patchwork.tag">
+    <field type="CharField" name="name">Reviewed-by</field>
+    <field type="CharField" name="pattern">^Reviewed-by:</field>
+    <field type="CharField" name="abbrev">R</field>
+  </object>
+  <object pk="3" model="patchwork.tag">
+    <field type="CharField" name="name">Tested-by</field>
+    <field type="CharField" name="pattern">^Tested-by:</field>
+    <field type="CharField" name="abbrev">T</field>
+  </object>
+</django-objects>

patchwork/management/__init__.py

@@ -0,0 +1,26 @@
+
+from django.core.management.base import BaseCommand, CommandError
+from patchwork.models import Patch
+import sys
+
+class Command(BaseCommand):
+    help = 'Update the tag (Ack/Review/Test) counts on existing patches'
+    args = '[<patch_id>...]'
+
+    def handle(self, *args, **options):
+
+        qs = Patch.objects
+
+        if args:
+            qs = qs.filter(id__in = args)
+
+        count = qs.count()
+        i = 0
+
+        for patch in qs.iterator():
+            patch.refresh_tag_counts()
+            i += 1
+            if (i % 10) == 0 or i == count:
+                sys.stdout.write('%06d/%06d\r' % (i, count))
+                sys.stdout.flush()
+        sys.stderr.write('\ndone\n')

patchwork/management/commands/__init__.py

@@ -22,11 +22,13 @@
 from django.core.urlresolvers import reverse
 from django.contrib.sites.models import Site
 from django.conf import settings
-from patchwork.parser import hash_patch
+from django.utils.functional import cached_property
+from patchwork.parser import hash_patch, extract_tags
 
 import re
 import datetime, time
 import random
+from collections import Counter, OrderedDict
 
 class Person(models.Model):
     email = models.CharField(max_length=255, unique = True)
@@ -56,6 +58,7 @@ class Project(models.Model):
     scm_url = models.CharField(max_length=2000, blank=True)
     webscm_url = models.CharField(max_length=2000, blank=True)
     send_notifications = models.BooleanField(default=False)
+    use_tags = models.BooleanField(default=True)
 
     def __unicode__(self):
         return self.name
@@ -65,6 +68,12 @@ def is_editable(self, user):
             return False
         return self in user.profile.maintainer_projects.all()
 
+    @cached_property
+    def tags(self):
+        if not self.use_tags:
+            return []
+        return list(Tag.objects.all())
+
     class Meta:
         ordering = ['linkname']
 
@@ -165,9 +174,68 @@ def _construct(string = ''):
     def db_type(self, connection=None):
         return 'char(%d)' % self.n_bytes
 
+class Tag(models.Model):
+    name = models.CharField(max_length=20)
+    pattern = models.CharField(max_length=50,
+                help_text='A simple regex to match the tag in the content of '
+                    'a message. Will be used with MULTILINE and IGNORECASE '
+                    'flags. eg. ^Acked-by:')
+    abbrev = models.CharField(max_length=2, unique=True,
+                help_text='Short (one-or-two letter) abbreviation for the tag, '
+                    'used in table column headers')
+
+    def __unicode__(self):
+        return self.name
+
+    @property
+    def attr_name(self):
+        return 'tag_%d_count' % self.id
+
+    class Meta:
+        ordering = ['abbrev']
+
+class PatchTag(models.Model):
+    patch = models.ForeignKey('Patch')
+    tag = models.ForeignKey('Tag')
+    count = models.IntegerField(default=1)
+
+    class Meta:
+        unique_together = [('patch', 'tag')]
+
 def get_default_initial_patch_state():
     return State.objects.get(ordering=0)
 
+class PatchQuerySet(models.query.QuerySet):
+
+    def with_tag_counts(self, project):
+        if not project.use_tags:
+            return self
+
+        # We need the project's use_tags field loaded for Project.tags().
+        # Using prefetch_related means we'll share the one instance of
+        # Project, and share the project.tags cache between all patch.project
+        # references.
+        qs = self.prefetch_related('project')
+        select = OrderedDict()
+        select_params = []
+        for tag in project.tags:
+            select[tag.attr_name] = ("coalesce("
+                "(SELECT count FROM patchwork_patchtag "
+                "WHERE patchwork_patchtag.patch_id=patchwork_patch.id "
+                    "AND patchwork_patchtag.tag_id=%s), 0)")
+            select_params.append(tag.id)
+
+        return qs.extra(select=select, select_params=select_params)
+
+class PatchManager(models.Manager):
+    use_for_related_fields = True
+
+    def get_queryset(self):
+        return PatchQuerySet(self.model, using=self.db)
+
+    def with_tag_counts(self, project):
+        return self.get_queryset().with_tag_counts(project)
+
 class Patch(models.Model):
     project = models.ForeignKey(Project)
     msgid = models.CharField(max_length=255)
@@ -182,13 +250,34 @@ class Patch(models.Model):
     pull_url = models.CharField(max_length=255, null = True, blank = True)
     commit_ref = models.CharField(max_length=255, null = True, blank = True)
     hash = HashField(null = True, blank = True)
+    tags = models.ManyToManyField(Tag, through=PatchTag)
+
+    objects = PatchManager()
 
     def __unicode__(self):
         return self.name
 
     def comments(self):
         return Comment.objects.filter(patch = self)
 
+    def _set_tag(self, tag, count):
+        if count == 0:
+            self.patchtag_set.filter(tag=tag).delete()
+            return
+        (patchtag, _) = PatchTag.objects.get_or_create(patch=self, tag=tag)
+        if patchtag.count != count:
+            patchtag.count = count
+            patchtag.save()
+
+    def refresh_tag_counts(self):
+        tags = self.project.tags
+        counter = Counter()
+        for comment in self.comment_set.all():
+            counter = counter + extract_tags(comment.content, tags)
+
+        for tag in tags:
+            self._set_tag(tag, counter[tag])
+
     def save(self):
         try:
             s = self.state
@@ -239,6 +328,14 @@ def patch_responses(self):
         return ''.join([ match.group(0) + '\n' for match in
                                 self.response_re.finditer(self.content)])
 
+    def save(self, *args, **kwargs):
+        super(Comment, self).save(*args, **kwargs)
+        self.patch.refresh_tag_counts()
+
+    def delete(self, *args, **kwargs):
+        super(Comment, self).delete(*args, **kwargs)
+        self.patch.refresh_tag_counts()
+
     class Meta:
         ordering = ['date']
         unique_together = [('msgid', 'patch')]