Skip to content
This repository has been archived by the owner on Aug 26, 2022. It is now read-only.

Commit

Permalink
Merge remote-tracking branch 'upstream/master'
Browse files Browse the repository at this point in the history
  • Loading branch information
ubernostrum committed Apr 3, 2012
2 parents 1375dca + 125ba6f commit f41128e
Show file tree
Hide file tree
Showing 11 changed files with 384 additions and 53 deletions.
84 changes: 77 additions & 7 deletions apps/dekicompat/management/commands/migrate_to_kuma_wiki.py
Expand Up @@ -67,7 +67,8 @@
MT_NS_NAME_TO_ID = dict(MT_NAMESPACES)
MT_NS_ID_TO_NAME = dict((x[1], x[0]) for x in MT_NAMESPACES)
MT_MIGRATED_NS_IDS = (MT_NS_NAME_TO_ID[x] for x in (
'', 'Talk:', 'User:', 'User_talk:', 'Project:', 'Project_talk:'
'', 'Talk:', 'User:', 'User_talk:', 'Project:', 'Project_talk:',
'Template:', 'Template_talk:',
))

# NOTE: These are MD5 hashes of garbage User page content. The criteria is that
Expand Down Expand Up @@ -143,6 +144,8 @@ class Command(BaseCommand):
help="Migrate # of documents with syntax blocks"),
make_option('--withscripts', dest="withscripts", type="int", default=0,
help="Migrate # of documents that use scripts"),
make_option('--withtemplates', dest="withtemplates", type="int", default=0,
help="Migrate # of template documents"),
make_option('--syntax-metrics', action="store_true",
dest="syntax_metrics", default=False,
help="Measure syntax highlighter usage, skip migration"),
Expand Down Expand Up @@ -474,6 +477,18 @@ def gather_pages(self):
LIMIT %s
""" % (ns_list, '%s'), self.options['withscripts']))

if self.options['withtemplates'] > 0:
log.info("Gathering %s templates" %
self.options['withtemplates'])
iters.append(self._query("""
SELECT *
FROM pages
WHERE page_namespace=%s
ORDER BY page_timestamp DESC
LIMIT %s
""", MT_NS_NAME_TO_ID['Template:'],
self.options['withtemplates']))

return itertools.chain(*iters)

@transaction.commit_on_success
Expand Down Expand Up @@ -511,6 +526,12 @@ def update_document(self, r):
(locale, slug, r['page_display_name']))
return False

# Skip migrating Template:MindTouch/* templates
if slug.startswith('Template:MindTouch'):
log.debug("\t%s/%s (%s) skipped, was a MindTouch default template" %
(locale, slug, r['page_display_name']))
return False

# Check to see if this page's content is too long, skip if so.
if len(r['page_text']) > self.options['maxlength']:
log.debug("\t%s/%s (%s) skipped, page too long (%s > %s max)" %
Expand Down Expand Up @@ -596,7 +617,7 @@ def update_past_revisions(self, r_page, doc, tags):
significance=SIGNIFICANCES[0][0],
summary='',
keywords='',
content=self.convert_page_text(r['old_text']),
content=self.convert_page_text(r_page, r['old_text']),
comment=r['old_comment'],
created=ts,
creator_id=self.get_django_user_id_for_deki_id(r['old_user']),
Expand Down Expand Up @@ -657,7 +678,7 @@ def update_current_revision(self, r, doc, tags):
rev.slug = doc.slug
rev.title = doc.title
rev.tags = tags
rev.content = self.convert_page_text(r['page_text'])
rev.content = self.convert_page_text(r, r['page_text'])

# HACK: Some comments end up being too long, but just truncate.
rev.comment = r['page_comment'][:255]
Expand All @@ -666,20 +687,31 @@ def update_current_revision(self, r, doc, tags):
rev.save()
rev.make_current()

# If this is a template, set it as in need of template review
if doc.slug.startswith('Template:'):
rev.review_tags.set('template')

if created:
log.info("\t\tCurrent revision created. (ID=%s)" % rev.pk)
else:
log.info("\t\tCurrent revision updated. (ID=%s)" % rev.pk)

def convert_page_text(self, pt):
def convert_page_text(self, r, pt):
"""Given a page row from MindTouch, do whatever needs doing to convert
the page content for Kuma."""

# If this is a redirect, just convert the redirect.
if pt.startswith('#REDIRECT'):
pt = self.convert_redirect(pt)
return self.convert_redirect(pt)

# If this is a template, just do template conversion
ns_name = MT_NS_ID_TO_NAME.get(r['page_namespace'], '')
if ns_name == 'Template:':
return self.convert_dekiscript_template(pt)

# Otherwise, run through the rest of the conversions.
pt = self.convert_code_blocks(pt)
pt = self.convert_dekiscript_template_calls(pt)
pt = self.convert_dekiscript_calls(pt)
# TODO: bug 710726 - Convert intra-wiki links?

return pt
Expand All @@ -699,10 +731,48 @@ def convert_code_blocks(self, pt):
pt = ContentSectionTool(pt).filter(CodeSyntaxFilter).serialize()
return pt

def convert_dekiscript_template_calls(self, pt):
def convert_dekiscript_calls(self, pt):
return (wiki.content.parse(pt).filter(DekiscriptMacroFilter)
.serialize())

def convert_dekiscript_template(self, pt):
"""Do what we can to convert DekiScript templates into EJS templates.
This is an incomplete process, but it tries to take care off as much as
it can so that human intervention is minimized."""

# Many templates start with this prefix, which corresponds to {% in EJS
pre = '<pre class="script">'
if pt.startswith(pre):
pt = "{%%\n%s" % pt[len(pre):]

# Many templates end with this postfix, which corresponds to %} in EJS
post = '</pre>'
if pt.endswith(post):
pt = "%s\n%%}" % pt[:0-len(post)]

# Template source is usually HTML encoded inside the <pre>
pt = (pt.replace('&amp;', '&')
.replace('&lt;', '<')
.replace('&gt;', '>')
.replace('&quot;', '"'))

# String concatenation is '..' in DS, '+' in EJS
pt = pt.replace('..', '+')

# ?? in DS is pretty much || in EJS
pt = pt.replace('??', '||')

# No need for DS 'let' in EJS
pt = pt.replace('let ', '')

# This is a common sequence at the start of many templates. It clobbers
# the url API, and needs correcting.
pt = (pt.replace('var uri =', 'var u =')
.replace('uri.path[', 'u.path['))

return pt

def get_tags_for_page(self, r):
"""For a given page row, get the list of tags from MindTouch and build
a string representation for Kuma revisions."""
Expand Down
2 changes: 1 addition & 1 deletion apps/demos/__init__.py
Expand Up @@ -153,7 +153,7 @@
"summary": _("The HTML5 audio element lets you embed sound in webpages without requiring your users to rely on plug-ins."),
"description": _("The HTML5 audio element lets you embed sound in webpages without requiring your users to rely on plug-ins."),
"learn_more": [],
"tab_copy": _("<p>The HTML5 &lt;audio&gt; element lets you embed sound in Web pages. More importantly, it lets you do so without requiring your users to rely on plug-ins. This means sound for everyone, everywhere, in the most open way possible. In particular, you can play sounds in games with very low latency, making for a responsive, immersive game experience.</p><p>What else can you do with the audio element? Show us by submitting to the Dev Derby today.</p>"),
"tab_copy": _("<p>The <a href=\"https://developer.mozilla.org/en/HTML/Element/audio\">HTML5 &lt;audio&gt;</a> element lets you embed sound in Web pages. More importantly, it lets you do so without requiring your users to rely on plug-ins. This means sound for everyone, everywhere, in the most open way possible. In particular, you can play sounds in games with <a href=\"http://robert.ocallahan.org/2011/11/latency-of-html5-sounds.html\">very low latency</a>, making for a responsive, immersive game experience.</p><p>What else can you do with the audio element? Show us by submitting to the Dev Derby today.</p>"),
},
{
"tag_name": "challenge:2012:may",
Expand Down
48 changes: 43 additions & 5 deletions apps/wiki/admin.py
@@ -1,19 +1,57 @@
from datetime import datetime

from django.contrib import admin
from django.core import serializers
from django.http import HttpResponse

from smuggler.settings import SMUGGLER_FORMAT
from smuggler.utils import serialize_to_response

from wiki.models import Document, Revision, EditorToolbar


def dump_selected(modeladmin, request, queryset):
objects = []
for doc in queryset.all():
rev = Revision.objects.get(id=doc.current_revision.id)
doc.current_revision = None
objects.append(doc)
objects.append(rev)
serializers.get_serializer('json')
filename = "documents_%s.%s" % (
datetime.now().isoformat(), SMUGGLER_FORMAT)
response = HttpResponse(mimetype="text/plain")
response['Content-Disposition'] = 'attachment; filename=%s' % filename
return serialize_to_response(objects, response)

dump_selected.short_description = "Dump selected objects as JSON data"


class DocumentAdmin(admin.ModelAdmin):
exclude = ('tags',)
list_display = ('id', 'locale', 'slug', 'title', 'category',
'is_localizable')
actions = [dump_selected, ]
change_list_template = 'admin/wiki/document/change_list.html'
fields = ('title', 'slug', 'locale', 'parent', 'category')
list_display = ('id', 'locale', 'slug', 'title', 'is_localizable',
'modified', 'parent_document_link',
'current_revision_link', 'related_revisions_link',)
list_display_links = ('id', 'slug',)
list_filter = ('is_template', 'is_localizable', 'category', 'locale')
raw_id_fields = ('parent',)
readonly_fields = ('id', 'current_revision')
search_fields = ('title',)
search_fields = ('title', 'slug', 'html')


class RevisionAdmin(admin.ModelAdmin):
fields = ('title', 'slug', 'summary', 'content', 'keywords', 'tags',
'reviewed', 'comment', 'is_approved')
list_display = ('id', 'slug', 'title', 'is_approved', 'created',
'creator',)
list_display_links = ('id', 'slug')
list_filter = ('is_approved', )
ordering = ('-created',)
search_fields = ('title', 'slug', 'summary', 'content', 'tags')


admin.site.register(Document, DocumentAdmin)
admin.site.register(Revision, admin.ModelAdmin)
admin.site.register(Revision, RevisionAdmin)
admin.site.register(EditorToolbar, admin.ModelAdmin)
130 changes: 130 additions & 0 deletions apps/wiki/fixtures/wiki/templates.json
@@ -0,0 +1,130 @@
[
{
"pk": 8,
"model": "wiki.document",
"fields": {
"category": 10,
"parent": null,
"title": "Template:test",
"locale": "en-US",
"mindtouch_page_id": null,
"html": "<p>{%= new Date() %}</p>",
"current_revision": 24,
"modified": "2012-03-20 10:09:16",
"is_template": true,
"is_localizable": false,
"slug": "Template:test"
}
},
{
"pk": 9,
"model": "wiki.document",
"fields": {
"category": 10,
"parent": null,
"title": "test-template",
"locale": "en-US",
"mindtouch_page_id": null,
"html": "<p>{{ test() }}</p>",
"current_revision": 25,
"modified": "2012-03-20 10:09:38",
"is_template": false,
"is_localizable": false,
"slug": "test-template"
}
},
{
"pk": 1,
"model": "wiki.reviewtag",
"fields": {
"name": "technical",
"slug": "technical"
}
},
{
"pk": 2,
"model": "wiki.reviewtag",
"fields": {
"name": "editorial",
"slug": "editorial"
}
},
{
"pk": 1,
"model": "wiki.reviewtaggedrevision",
"fields": {
"content_object": 24,
"tag": 1
}
},
{
"pk": 2,
"model": "wiki.reviewtaggedrevision",
"fields": {
"content_object": 24,
"tag": 2
}
},
{
"pk": 3,
"model": "wiki.reviewtaggedrevision",
"fields": {
"content_object": 25,
"tag": 1
}
},
{
"pk": 4,
"model": "wiki.reviewtaggedrevision",
"fields": {
"content_object": 25,
"tag": 2
}
},
{
"pk": 24,
"model": "wiki.revision",
"fields": {
"comment": "",
"based_on": null,
"is_approved": true,
"reviewed": null,
"title": "Template:test",
"tags": "",
"summary": "",
"content": "<p>{%= new Date() %}</p>",
"mindtouch_old_id": null,
"is_mindtouch_migration": false,
"created": "2012-03-20 10:09:16",
"significance": null,
"keywords": "",
"reviewer": null,
"document": 8,
"creator": 7,
"slug": "Template:test"
}
},
{
"pk": 25,
"model": "wiki.revision",
"fields": {
"comment": "",
"based_on": null,
"is_approved": true,
"reviewed": null,
"title": "test-template",
"tags": "",
"summary": "",
"content": "<p>{{ test() }}</p>",
"mindtouch_old_id": null,
"is_mindtouch_migration": false,
"created": "2012-03-20 10:09:38",
"significance": null,
"keywords": "",
"reviewer": null,
"document": 9,
"creator": 7,
"slug": "test-template"
}
}
]
12 changes: 7 additions & 5 deletions apps/wiki/forms.py
Expand Up @@ -251,11 +251,13 @@ def __init__(self, *args, **kwargs):
self.initial['slug'] = self.instance.document.slug

content = self.instance.content
tool = wiki.content.parse(content)
tool.injectSectionIDs()
if self.section_id:
tool.extractSection(self.section_id)
self.initial['content'] = tool.serialize()
if not self.instance.document.is_template:
tool = wiki.content.parse(content)
tool.injectSectionIDs()
if self.section_id:
tool.extractSection(self.section_id)
content = tool.serialize()
self.initial['content'] = content

self.initial['review_tags'] = [x.name
for x in self.instance.review_tags.all()]
Expand Down

0 comments on commit f41128e

Please sign in to comment.