fix bug 710722, fix bug 722467: Document tag migration, viewing, editing

* Allow all users to create tags, removing Kitsune restriction code
* Migrate tag data from MindTouch
* Display tag list in document, each linked to tag-filtered doc list
* Tags are edited in Revisions, Revisions update Document tags
* Documents are indexed by DocumentTag, instead of the general tag pool.
* South migrations to add new DocumentTag and TaggedDocument models
* Show tags in comparisons for Revision changes
* Bugfixes to document feeds by tag
* Hackjob on CSS to pull in Deki styles for tags (FIXME)
* Much quicker --wipe option in migration command
* Form validation tweaks
* Tests and test tweaks

apps/dekicompat/management/commands/migrate_to_kuma_wiki.py

@@ -77,6 +77,9 @@
 # See also these SQL queries:
 # https://bugzilla.mozilla.org/show_bug.cgi?id=710753#c3
 # https://bugzilla.mozilla.org/show_bug.cgi?id=710753#c4
+#
+# And, see also this MySQL transcript listing the content involved:
+# https://bugzilla.mozilla.org/attachment.cgi?id=590867
 
 USER_NS_EXCLUDED_CONTENT_HASHES = """
 7479e8f30d5ab0e9202195a1bddec69d
@@ -200,7 +203,7 @@ def handle_migration(self, rows):
         for r in rows:
 
             try:
-                
+
                 if ct < self.options['skip']:
                     # Skip rows until past the option value
                     continue
@@ -289,17 +292,13 @@ def handle_template_metrics(self, rows):
     @transaction.commit_on_success
     def wipe_documents(self):
         """Delete all documents"""
-        docs = Document.objects.all()
-        ct = 0
-        log.info("Deleting %s documents..." % len(docs))
-        for d in docs:
-            d.delete()
-            ct += 1
-            if 0 == (ct % 10):
-                log.debug("\t%s deleted" % ct)
-                # Clear query cache after each document. Lots of queries are
-                # bound to happen, there.
-                django.db.reset_queries()
+        log.info("Wiping all Kuma documents and revisions")
+        kc = self.kumadb.cursor()
+        kc.execute("""
+            SET FOREIGN_KEY_CHECKS = 0;
+            TRUNCATE wiki_revision;
+            TRUNCATE wiki_document;
+        """)
 
     def index_migrated_docs(self):
         """Build an index of Kuma docs already migrated, mapping Mindtouch page
@@ -465,12 +464,14 @@ def update_document(self, r):
         else:
             log.info("\t\tDocument already exists. (ID=%s)" % doc.pk)
 
-        self.update_past_revisions(r, doc)
-        self.update_current_revision(r, doc)
+        tags = self.get_tags_for_page(r)
+
+        self.update_past_revisions(r, doc, tags)
+        self.update_current_revision(r, doc, tags)
 
         return True
 
-    def update_past_revisions(self, r_page, doc):
+    def update_past_revisions(self, r_page, doc, tags):
         """Update past revisions for the given page row and document"""
         ct_saved, ct_skipped, ct_error = 0, 0, 0
 
@@ -519,7 +520,7 @@ def update_past_revisions(self, r_page, doc):
                 r['old_id'], 1,
                 doc.slug, doc.title,
                 True, SIGNIFICANCES[0][0],
-                '', '',
+                '', '', tags,
                 self.convert_page_text(r['old_text']),
                 r['old_comment'],
                 ts, self.get_django_user_id_for_deki_id(r['old_user']),
@@ -533,7 +534,7 @@ def update_past_revisions(self, r_page, doc):
 
             # Build SQL placeholders for the revisions
             row_placeholders = ",\n".join(
-                "(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
+                "(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
                 for x in revs)
 
             # Flatten list of revisions data in chronological order, so that we
@@ -550,7 +551,7 @@ def update_past_revisions(self, r_page, doc):
                      mindtouch_old_id, is_mindtouch_migration,
                      slug, title,
                      is_approved, significance,
-                     summary, keywords,
+                     summary, keywords, tags,
                      content, comment,
                      created, creator_id,
                      reviewed, reviewer_id)
@@ -563,7 +564,7 @@ def update_past_revisions(self, r_page, doc):
         log.info("\t\tPast revisions: %s saved, %s skipped, %s errors" %
                  (ct_saved, ct_skipped, ct_error))
 
-    def update_current_revision(self, r, doc):
+    def update_current_revision(self, r, doc, tags):
         # HACK: Using old_id of None to indicate the current MindTouch revision.
         # All revisions of a Kuma document have revision records, whereas
         # MindTouch only tracks "old" revisions.
@@ -584,6 +585,7 @@ def update_current_revision(self, r, doc):
         rev.created = rev.reviewed = page_ts
         rev.slug = doc.slug
         rev.title = doc.title
+        rev.tags = tags
         rev.content = self.convert_page_text(r['page_text'])
 
         # HACK: Some comments end up being too long, but just truncate.
@@ -620,6 +622,35 @@ def convert_redirect(self, pt):
             href = reverse('wiki.document', args=[title])
             pt = REDIRECT_CONTENT % dict(href=href, title=title)
         return pt
+
+    def get_tags_for_page(self, r):
+        """For a given page row, get the list of tags from MindTouch and build
+        a string representation for Kuma revisions."""
+        wc = self.wikidb.cursor()
+        wc.execute("""
+            SELECT t.tag_name
+            FROM tag_map AS tm, tags AS t, pages AS p 
+            WHERE
+                t.tag_id=tm.tagmap_tag_id AND
+                p.page_id=tm.tagmap_page_id AND
+                p.page_id=%s
+        """, (r['page_id'],))
+
+        # HACK: To prevent MySQL truncation warnings, constrain the imported
+        # tags to 100 chars. Who wants tags that long, anyway?
+        mt_tags = [row[0][:100] for row in wc]
+
+        # To build a string representation, we need to quote or not quote based
+        # on the presence of commas or spaces in the tag name.
+        quoted = []
+        if len(mt_tags):
+            for tag in mt_tags:
+                if u',' in tag or u' ' in tag:
+                    quoted.append('"%s"' % tag)
+                else:
+                    quoted.append(tag)
+
+        return u', '.join(quoted)
 
     def get_django_user_id_for_deki_id(self, deki_user_id):
         """Given a Deki user ID, come up with a Django user object whether we

apps/wiki/feeds.py

@@ -144,12 +144,19 @@ class DocumentsRecentFeed(DocumentsFeed):
 
     def get_object(self, request, format, tag=None, category=None):
         super(DocumentsRecentFeed, self).get_object(request, format)
-        self.link = self.request.build_absolute_uri(
-            reverse('wiki.views.list_documents'))
+        self.category = category
+        self.tag = tag
+        if tag:
+            self.title = _('MDN recent changes to documents tagged %s' % tag)
+            self.link = self.request.build_absolute_uri(
+                reverse('wiki.tag', args=(tag,)))
+        else:
+            self.link = self.request.build_absolute_uri(
+                reverse('wiki.views.list_documents'))
 
-    def items(self, tag=None, category=None):
+    def items(self):
         return (Document.objects
-                .filter_for_list(tag_name=tag, category=category)
+                .filter_for_list(tag_name=self.tag, category=self.category)
                 .filter(current_revision__isnull=False)
                 .order_by('-current_revision__created')
                 .all()[:MAX_FEED_ITEMS])

apps/wiki/forms.py

@@ -11,6 +11,8 @@
 from sumo.form_fields import StrippedCharField
 from tags import forms as tag_forms
 
+from taggit.utils import parse_tags, edit_string_for_tags
+
 import wiki.content
 from wiki.models import (Document, Revision, FirefoxVersion, OperatingSystem,
                      FIREFOX_VERSIONS, OPERATING_SYSTEMS, SIGNIFICANCES,
@@ -56,14 +58,6 @@
 
 class DocumentForm(forms.ModelForm):
     """Form to create/edit a document."""
-    def __init__(self, *args, **kwargs):
-        can_create_tags = kwargs.pop('can_create_tags', False)
-
-        super(DocumentForm, self).__init__(*args, **kwargs)
-
-        # Set up tags field, which is instantiated deep within taggit:
-        tags_field = self.fields['tags']
-        tags_field.widget.can_create_tags = can_create_tags
 
     title = StrippedCharField(min_length=5, max_length=255,
                               widget=forms.TextInput(attrs={'placeholder':TITLE_PLACEHOLDER}),
@@ -113,11 +107,6 @@ def __init__(self, *args, **kwargs):
                                  help_text=_lazy(u'Type of article'),
                                  widget=forms.HiddenInput())
 
-    tags = tag_forms.TagField(required=False, label=_lazy(u'Topics:'),
-                              help_text=_lazy(
-                                u'Popular articles in each topic '
-                                'are displayed on the front page'))
-
     locale = forms.CharField(widget=forms.HiddenInput())
 
     def clean_slug(self):
@@ -147,8 +136,7 @@ def clean_operating_systems(self):
 
     class Meta:
         model = Document
-        fields = ('title', 'slug', 'category', 'is_localizable', 'tags',
-                  'locale')
+        fields = ('title', 'slug', 'category', 'is_localizable', 'locale')
 
     def save(self, parent_doc, **kwargs):
         """Persist the Document form, and return the saved Document."""
@@ -189,6 +177,9 @@ class RevisionForm(forms.ModelForm):
                                              'min_length': SLUG_SHORT,
                                              'max_length': SLUG_LONG})
 
+    tags = StrippedCharField(required=False,
+                             label=_lazy(u'Tags:'))
+
     keywords = StrippedCharField(required=False,
                                  label=_lazy(u'Keywords:'),
                                  help_text=_lazy(u'Affects search results'))
@@ -230,8 +221,8 @@ class RevisionForm(forms.ModelForm):
 
     class Meta(object):
         model = Revision
-        fields = ('title', 'slug', 'keywords', 'summary', 'content', 'comment',
-                  'based_on')
+        fields = ('title', 'slug', 'tags', 'keywords', 'summary', 'content',
+                  'comment', 'based_on')
 
     def __init__(self, *args, **kwargs):