Merge pull request #239 from groovecoder/migrate-breadcrumbs-753581

Fix bug 753581 migrate breadcrumb parent documents

apps/dekicompat/management/commands/migrate_to_kuma_wiki.py

@@ -159,6 +159,9 @@ class Command(BaseCommand):
         make_option('--skip-translations', action="store_true",
                     dest="skip_translations", default=False,
                     help="Skip migrating translated children of documents"),
+        make_option('--skip-breadcrumbs', action="store_true",
+                    dest="skip_breadcrumbs", default=False,
+                    help="Skip migrating breadcrumb parents of documents"),
 
         make_option('--limit', dest="limit", type="int", default=99999,
                     help="Stop after a migrating a number of documents"),
@@ -207,6 +210,9 @@ def handle(self, *args, **options):
             if not options['skip_translations']:
                 rows = self.gather_pages()
                 self.make_languages_relationships(rows)
+            if not options['skip_breadcrumbs']:
+                rows = self.gather_pages()
+                self.make_breadcrumb_relationships(rows)
 
     def init(self, options):
         """Set up connections and options"""
@@ -335,6 +341,63 @@ def handle_syntax_metrics(self, rows):
                         print (u"%s\t%s\t%s" % (r['page_title'], block.name,
                                                     attr[1])).encode('utf-8')
 
+    def _get_mindtouch_pages_row(self, page_id):
+        sql = """
+            SELECT *
+            FROM pages
+            WHERE page_id = %d
+        """ % page_id
+        try:
+            rows = self._query(sql)
+        except Exception, e:
+            log.error("\tpage_id %s error %s" %
+                      (page_id, e))
+        single_row = None
+        for row in rows:
+            single_row = row
+        return single_row
+
+    def _migrate_necessary_mindtouch_pages(self, migrate_ids):
+        """ Given a list of mindtouch ids, migrate only the necessary pages."""
+        existing = [str(x['mindtouch_page_id']) for x in
+                    Document.objects.filter(mindtouch_page_id__in=migrate_ids)
+                                    .values('mindtouch_page_id')]
+        need_migrate_ids = [str(x) for x in migrate_ids
+                            if str(x) not in existing]
+        if need_migrate_ids:
+            sql = "SELECT * FROM pages WHERE page_id in (%s)" % (
+                ",".join(need_migrate_ids))
+            rows = self._query(sql)
+            self.handle_migration(rows)
+
+    def _add_parent_ids(self, r, bc_ids):
+        """ Recursively add parent ids to breadcrumb ids list. """
+        parent_row = self._get_mindtouch_pages_row(r['page_parent'])
+        if parent_row:
+            bc_ids.insert(0, r['page_parent'])
+            self._add_parent_ids(parent_row, bc_ids)
+        return bc_ids
+
+    @transaction.commit_on_success(using='default')
+    def make_breadcrumb_relationships(self, rows):
+        """Set the topic_parent for Kuma pages using parent_id"""
+        log.info("Building parent/child breadcrumb tree...")
+        for r in rows:
+            if not r['page_text'].strip():
+                continue
+            bc_ids = []
+            bc_ids.insert(0, r['page_id'])
+            if r['page_parent']:
+                bc_ids = self._add_parent_ids(r, bc_ids)
+            log.info("Migrating breadcrumb ids: %s" % bc_ids)
+            self._migrate_necessary_mindtouch_pages(bc_ids)
+            parent_doc = Document.objects.get(mindtouch_page_id=bc_ids.pop(0))
+            for id in bc_ids:
+                doc = Document.objects.get(mindtouch_page_id=id)
+                doc.parent_topic = parent_doc
+                doc.save()
+                parent_doc = doc
+
     @transaction.commit_manually(using='default')
     def make_languages_relationships(self, rows):
         """Set the parent_id of Kuma pages using wiki.languages params"""
@@ -407,15 +470,7 @@ def make_languages_relationships(self, rows):
                     continue
 
             # Migrate any child documents that haven't already been
-            existing = [str(x['mindtouch_page_id']) for x in 
-                        Document.objects.filter(mindtouch_page_id__in=children)
-                                        .values('mindtouch_page_id')]
-            need_migrate_ids = [str(x) for x in children if str(x) not in existing]
-            if need_migrate_ids:
-                sql = "SELECT * FROM pages WHERE page_id in (%s)" % (
-                    ",".join(need_migrate_ids))
-                rows = self._query(sql)
-                self.handle_migration(rows)
+            self._migrate_necessary_mindtouch_pages(children)
 
             # All parents and children migrated, now set parent_id
             # TODO: refactor this to source_id when we change to
@@ -667,8 +722,10 @@ def update_document(self, r):
 
         # Ensure that the document exists, and has the MindTouch page ID
         doc, created = Document.objects.get_or_create(
-            locale=locale, slug=slug,
-            title=r['page_display_name'], defaults=dict(
+            locale=locale,
+            slug=slug,
+            title=r['page_display_name'],
+            defaults=dict(
                 category=CATEGORIES[0][0],
             ))
         doc.mindtouch_page_id = r['page_id']

apps/wiki/admin.py

@@ -21,9 +21,10 @@ def dump_selected_documents(modeladmin, request, queryset):
 class DocumentAdmin(admin.ModelAdmin):
     actions = [dump_selected_documents, ]
     change_list_template = 'admin/wiki/document/change_list.html'
-    fields = ('title', 'slug', 'locale', 'parent', 'category')
+    fields = ('title', 'slug', 'locale', 'parent', 'parent_topic', 'category')
     list_display = ('id', 'locale', 'slug', 'title', 'is_localizable',
                     'modified', 'parent_document_link',
+                    'topic_parent_document_link',
                     'current_revision_link', 'related_revisions_link',)
     list_display_links = ('id', 'slug',)
     list_filter = ('is_template', 'is_localizable', 'category', 'locale')

apps/wiki/models.py

@@ -864,15 +864,25 @@ def current_revision_link(self):
     current_revision_link.short_description = "Current Revision"
 
     def parent_document_link(self):
-        """HTML link to the parent document for admin change list"""
+        """HTML link to the topical parent document for admin change list"""
         if not self.parent:
             return "None"
         url = reverse('admin:wiki_document_change', args=[self.parent.id])
         return '<a href="%s">Document #%s</a>' % (url, self.parent.id)
 
     parent_document_link.allow_tags = True
-    parent_document_link.short_description = "Parent Document"
+    parent_document_link.short_description = "Translation Parent"
+
+    def topic_parent_document_link(self):
+        """HTML link to the parent document for admin change list"""
+        if not self.parent_topic:
+            return "None"
+        url = reverse('admin:wiki_document_change',
+                      args=[self.parent_topic.id])
+        return '<a href="%s">Document #%s</a>' % (url, self.parent_topic.id)
 
+    topic_parent_document_link.allow_tags = True
+    topic_parent_document_link.short_description = "Parent Document"
 
 class ReviewTag(TagBase):
     """A tag indicating review status, mainly for revisions"""

apps/wiki/templates/wiki/document.html

@@ -50,14 +50,10 @@
   <!-- left crumb navigation -->
   <nav class="crumbs" role="navigation">
     <ol>
-      <!-- Sample... 
-      <li class="crumb-one"><a href="/">MDN</a></li>
-      <li class="crumb-two"><a href="/docs">Docs</a></li>
-      <li class="first"><a href="https://developer.mozilla.org/" class="deki-ns">MDN</a></li>
-      <li class="second"><a href="https://developer.mozilla.org/en" class="deki-ns">MDN</a></li>
-      <li><a href="https://developer.mozilla.org/en/HTML" class="deki-ns">HTML</a></li>
-      <li class="last"><a href="https://developer.mozilla.org/en/HTML/Using_the_application_cache" class="deki-ns current">Using the application cache</a></li> 
-      -->
+      {% for doc in document.parents %}
+      <li class="crumb"><a href="{{ url('wiki.document', doc.locale+'/'+doc.slug) }}">{{ doc.title }}</a></li>
+      {% endfor %}
+      <li class="crumb">{{ document.title }}</li>
     </ol>
   </nav>
 
@@ -222,6 +218,3 @@ <h2>{{ _('Tags') }}</h2>
 {% block side %}
   {% include 'wiki/includes/support_for_selectors.html' %}
 {% endblock %}
-
-{% block breadcrumbs %}
-{% endblock %}

apps/wiki/tests/__init__.py

@@ -165,3 +165,14 @@ def create_template_test_users():
         superuser.save()
 
     return (perms, groups, users, superuser)
+
+
+def create_topical_parents_docs():
+    d1 = document(title='HTML7')
+    d1.save()
+
+    d2 = document(title='Smellovision')
+    d2.parent_topic = d1
+    d2.save()
+    return d1, d2
+

apps/wiki/tests/test_models.py

@@ -20,7 +20,8 @@
                          get_current_or_latest_revision,
                          TaggedDocument)
 from wiki.tests import (document, revision, doc_rev, translated_revision,
-                        create_template_test_users)
+                        create_template_test_users,
+                        create_topical_parents_docs)
 
 
 def _objects_eq(manager, list_):
@@ -268,12 +269,7 @@ def test_other_translations(self):
         eq_(False, enfant in enfant.other_translations)
 
     def test_topical_parents(self):
-        d1 = document(title='HTML7')
-        d1.save()
-
-        d2 = document(title='Smellovision')
-        d2.parent_topic = d1
-        d2.save()
+        d1, d2 = create_topical_parents_docs()
         ok_(d2.parents == [d1])
 
         d3 = document(title='Smell accessibility')

apps/wiki/tests/test_templates.py

@@ -23,7 +23,8 @@
                          ApproveRevisionInLocaleEvent)
 from wiki.models import Document, Revision, HelpfulVote, SIGNIFICANCES
 from wiki.tasks import send_reviewed_notification
-from wiki.tests import TestCaseBase, document, revision, new_document_data
+from wiki.tests import (TestCaseBase, document, revision, new_document_data,
+                        create_topical_parents_docs)
 from devmo.tests import SkippedTestCase
 
 
@@ -76,6 +77,22 @@ def test_document_view(self):
         eq_(r.document.title, doc('article header h1.page-title').text())
         eq_(r.document.html, doc('div#wikiArticle').text())
 
+    @attr("breadcrumbs")
+    def test_document_breadcrumbs(self):
+        """Create docs with topical parent/child rel, verify breadcrumbs."""
+        d1, d2 = create_topical_parents_docs()
+        response = self.client.get(d1.get_absolute_url())
+        eq_(200, response.status_code)
+        doc = pq(response.content)
+        eq_(d1.title, doc('article header h1.page-title').text())
+        eq_(d1.title, doc('nav.crumbs').text())
+        response = self.client.get(d2.get_absolute_url())
+        eq_(200, response.status_code)
+        doc = pq(response.content)
+        eq_(d2.title, doc('article header h1.page-title').text())
+        crumbs = "%s %s" % (d1.title, d2.title)
+        eq_(crumbs, doc('nav.crumbs').text())
+
     def test_english_document_no_approved_content(self):
         """Load an English document with no approved content."""
         r = revision(save=True, content='Some text.', is_approved=False)