Skip to content

Bug 730707 kumascript #164

Merged
merged 16 commits into from Apr 20, 2012
+415 −140
View
10 apps/dekicompat/management/commands/migrate_to_kuma_wiki.py
@@ -741,15 +741,15 @@ def convert_dekiscript_template(self, pt):
This is an incomplete process, but it tries to take care off as much as
it can so that human intervention is minimized."""
- # Many templates start with this prefix, which corresponds to {% in EJS
+ # Many templates start with this prefix, which corresponds to <% in EJS
pre = '<pre class="script">'
if pt.startswith(pre):
- pt = "{%%\n%s" % pt[len(pre):]
+ pt = "<%%\n%s" % pt[len(pre):]
- # Many templates end with this postfix, which corresponds to %} in EJS
+ # Many templates end with this postfix, which corresponds to %> in EJS
post = '</pre>'
if pt.endswith(post):
- pt = "%s\n%%}" % pt[:0-len(post)]
+ pt = "%s\n%%>" % pt[:0-len(post)]
# Template source is usually HTML encoded inside the <pre>
pt = (pt.replace('&amp;', '&')
@@ -817,7 +817,7 @@ def get_kuma_locale_and_slug_for_page(self, r):
if '/' in title:
# Treat the first part of the slug path as locale and snip it off.
mt_language, new_title = title.split('/', 1)
- if mt_language in MT_TO_KUMA_LOCALE_MAP:
+ if mt_language.lower() in MT_TO_KUMA_LOCALE_MAP:
# If it's a known language, then rebuild the slug
slug = '%s%s' % (ns_name, new_title)
else:
View
121 apps/wiki/content.py
@@ -1,8 +1,12 @@
+import logging
import re
from urllib import urlencode
+from xml.sax.saxutils import quoteattr
+
import html5lib
from html5lib.filters._base import Filter as html5lib_Filter
+from pyquery import PyQuery as pq
from tower import ugettext as _
@@ -27,6 +31,16 @@ def parse(src):
return ContentSectionTool(src)
+def filter_out_noinclude(src):
+ """Quick and dirty filter to remove <div class="noinclude"> blocks"""
+ # NOTE: This started as an html5lib filter, but it started getting really
+ # complex. Seems like pyquery works well enough without corrupting
+ # character encoding.
+ doc = pq(src)
+ doc.remove('*[class=noinclude]')
+ return doc.html()
+
+
class ContentSectionTool(object):
def __init__(self, src=None):
@@ -58,7 +72,7 @@ def parse(self, src):
def serialize(self, stream=None):
if stream is None:
stream = self.stream
- return "".join(self.serializer.serialize(stream))
+ return u"".join(self.serializer.serialize(stream))
def __unicode__(self):
return self.serialize()
@@ -102,6 +116,10 @@ def gen_id(self):
self.known_ids.add(id)
return id
+ def slugify(self, text):
+ """Turn the text content of a header into a slug for use in an ID"""
+ return (text.replace(' ', '_'))
@ubernostrum
ubernostrum added a note Apr 19, 2012

Is there any chance these IDs ever end up as part of a URL (not just a fragment identifier)? Looks like they do further down and if so, that's a potential Unicode issue -- we might want to do something like Django's own built-in slugify template filter, which has a little Unicode-normalization song-and-dance to produce a readable but URL-safe result.

@lmorchard
Mozilla member
lmorchard added a note Apr 19, 2012

Yeah, these will probably end up in section editing URLs. :/ Need to look at this some more, because I want to make sure it matches up with existing anchor links from MindTouch. I don't think it quite does that all the way, either.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
+
def __iter__(self):
input = html5lib_Filter.__iter__(self)
@@ -113,17 +131,63 @@ def __iter__(self):
attrs = dict(token['data'])
if 'id' in attrs:
self.known_ids.add(attrs['id'])
+ if 'name' in attrs:
+ self.known_ids.add(attrs['name'])
- # Pass 2: Sprinkle in IDs where they're missing
- for token in buffer:
- if ('StartTag' == token['type'] and
+ # Pass 2: Sprinkle in IDs where they're needed
+ while len(buffer):
+ token = buffer.pop(0)
+
+ if not ('StartTag' == token['type'] and
token['name'] in SECTION_TAGS):
+ yield token
+ else:
attrs = dict(token['data'])
- id = attrs.get('id', None)
- if not id:
+
+ # Treat a name attribute as a human-specified ID override
+ name = attrs.get('name', None)
+ if name:
+ attrs['id'] = name
+ token['data'] = attrs.items()
+ yield token
+ continue
+
+ # If this is not a header, then generate a section ID.
+ if token['name'] not in HEAD_TAGS:
attrs['id'] = self.gen_id()
token['data'] = attrs.items()
- yield token
+ yield token
+ continue
+
+ # If this is a header, then scoop up the rest of the header and
+ # gather the text it contains.
+ start, text, tmp = token, [], []
+ while len(buffer):
+ token = buffer.pop(0)
+ tmp.append(token)
+ if token['type'] in ('Characters', 'SpaceCharacters'):
+ text.append(token['data'])
+ elif ('EndTag' == token['type'] and
+ start['name'] == token['name']):
+ # Note: This is naive, and doesn't track other
+ # start/end tags nested in the header. Odd things might
+ # happen in a case like <h1><h1></h1></h1>. But, that's
+ # invalid markup and the worst case should be a
+ # truncated ID because all the text wasn't accumulated.
+ break
@ubernostrum
ubernostrum added a note Apr 20, 2012

This may be a silly question, but the comment here made me think of it: is there any mechanism enforcing uniqueness of IDs within the document? What happens if IDs end up colliding?

@lmorchard
Mozilla member
lmorchard added a note Apr 20, 2012

I kind of punted on that... There is a mechanism for uniqueness, but only for auto-generated IDs (eg. sect1, sect2, etc). For IDs based on element text or the name attribute, no uniqueness is enforced.

This is really a half-baked feature, ugh. :/

@lmorchard
Mozilla member
lmorchard added a note Apr 20, 2012

FWIW, I just filed bug 747403 to remember to put more work into this feature

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
+
+ # Slugify the text we found inside the header, generate an ID
+ # as a last resort.
+ slug = self.slugify(u''.join(text))
+ if not slug:
+ slug = self.gen_id()
+ attrs['id'] = slug
+ start['data'] = attrs.items()
+
+ # Finally, emit the tokens we scooped up for the header.
+ yield start
+ for t in tmp:
+ yield t
class SectionEditLinkFilter(html5lib_Filter):
@@ -152,17 +216,18 @@ def __iter__(self):
'title': _('Edit section'),
'class': 'edit-section',
'data-section-id': id,
- 'data-section-src-url': '%s?%s' % (
+ 'data-section-src-url': u'%s?%s' % (
reverse('wiki.document',
args=[self.full_path],
locale=self.locale),
- urlencode({'section': id, 'raw': 'true'})
+ urlencode({'section': id.encode('utf-8'),
+ 'raw': 'true'})
),
- 'href': '%s?%s' % (
+ 'href': u'%s?%s' % (
reverse('wiki.edit_document',
args=[self.full_path],
locale=self.locale),
- urlencode({'section': id,
+ urlencode({'section': id.encode('utf-8'),
'edit_links': 'true'})
)
}},
@@ -385,12 +450,26 @@ def __iter__(self):
continue
ds_call = []
- while len(buffer) and 'EndTag' != token['type']:
+ while len(buffer):
token = buffer.pop(0)
- if 'Characters' == token['type']:
+ if token['type'] in ('Characters', 'SpaceCharacters'):
ds_call.append(token['data'])
-
- ds_call = ''.join(ds_call).strip()
+ elif 'StartTag' == token['type']:
+ attrs = token['data']
+ if attrs:
+ a_out = (u' %s' % u' '.join(
+ (u'%s=%s' %
+ (name, quoteattr(val))
+ for name, val in attrs)))
+ else:
+ a_out = u''
+ ds_call.append(u'<%s%s>' % (token['name'], a_out))
+ elif 'EndTag' == token['type']:
+ if 'span' == token['name']:
+ break
+ ds_call.append('</%s>' % token['name'])
+
+ ds_call = u''.join(ds_call).strip()
# Snip off any "template." prefixes
strip_prefixes = ('template.', 'wiki.')
@@ -417,7 +496,11 @@ def __iter__(self):
if m:
ds_call = '%s()' % (m.group(1))
- yield dict(
- type="Characters",
- data='{{ %s }}' % ds_call
- )
+ # HACK: This is dirty, but seems like the easiest way to
+ # reconstitute the token stream, including what gets parsed as
+ # markup in the middle of macro parameters.
+ #
+ # eg. {{ Note("This is <strong>strongly</strong> discouraged") }}
+ parsed = parse('{{ %s }}' % ds_call)
+ for token in parsed.stream:
+ yield token
View
11 apps/wiki/forms.py
@@ -49,7 +49,6 @@
COMMENT_LONG = _lazy(u'Please keep the length of the comment to '
u'%(limit_value)s characters or less. It is currently '
u'%(show_value)s characters.')
-TITLE_COLLIDES = _lazy(u'Another document with this title already exists.')
SLUG_COLLIDES = _lazy(u'Another document with this slug already exists.')
OTHER_COLLIDES = _lazy(u'Another document with this metadata already exists.')
@@ -162,7 +161,7 @@ def save(self, parent_doc, **kwargs):
class RevisionForm(forms.ModelForm):
"""Form to create new revisions."""
- title = StrippedCharField(min_length=5, max_length=255,
+ title = StrippedCharField(min_length=2, max_length=255,
required=False,
widget=forms.TextInput(
attrs={'placeholder': TITLE_PLACEHOLDER}),
@@ -204,7 +203,7 @@ class RevisionForm(forms.ModelForm):
c in GROUPED_FIREFOX_VERSIONS]}
content = StrippedCharField(
- min_length=5, max_length=100000,
+ min_length=5, max_length=300000,
label=_lazy(u'Content:'),
widget=forms.Textarea(attrs={'data-showfor':
json.dumps(showfor_data)}),
@@ -274,8 +273,7 @@ def _clean_collidable(self, name):
# to them are ignored for an iframe submission
return getattr(self.instance.document, name)
- error_message = {'title': TITLE_COLLIDES,
- 'slug': SLUG_COLLIDES}.get(name, OTHER_COLLIDES)
+ error_message = {'slug': SLUG_COLLIDES}.get(name, OTHER_COLLIDES)
try:
existing_doc = Document.uncached.get(
locale=self.instance.document.locale,
@@ -297,9 +295,6 @@ def _clean_collidable(self, name):
return value
- def clean_title(self):
- return self._clean_collidable('title')
-
def clean_slug(self):
return self._clean_collidable('slug')
View
29 apps/wiki/models.py
@@ -36,7 +36,7 @@
ALLOWED_TAGS = bleach.ALLOWED_TAGS + [
'div', 'span', 'p', 'br', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
'pre', 'code',
- 'dl', 'dt', 'dd', 'small', 'sup',
+ 'dl', 'dt', 'dd', 'small', 'sup', 'u',
'img',
'input',
'table', 'tbody', 'thead', 'tr', 'th', 'td',
@@ -46,13 +46,14 @@
'address'
]
ALLOWED_ATTRIBUTES = bleach.ALLOWED_ATTRIBUTES
-ALLOWED_ATTRIBUTES['div'] = ['class', 'id']
-ALLOWED_ATTRIBUTES['pre'] = ['class', 'id']
-ALLOWED_ATTRIBUTES['span'] = ['style', ]
+ALLOWED_ATTRIBUTES['div'] = ['style', 'class', 'id']
+ALLOWED_ATTRIBUTES['p'] = ['style', 'class', 'id']
+ALLOWED_ATTRIBUTES['pre'] = ['style', 'class', 'id']
+ALLOWED_ATTRIBUTES['span'] = ['style', 'title', ]
ALLOWED_ATTRIBUTES['img'] = ['src', 'id', 'align', 'alt', 'class', 'is',
'title', 'style']
-ALLOWED_ATTRIBUTES['a'] = ['id', 'class', 'href', 'title', ]
-ALLOWED_ATTRIBUTES.update(dict((x, ['style', ]) for x in
+ALLOWED_ATTRIBUTES['a'] = ['style', 'id', 'class', 'href', 'title', ]
+ALLOWED_ATTRIBUTES.update(dict((x, ['style', 'name', ]) for x in
('h1', 'h2', 'h3', 'h4', 'h5', 'h6')))
ALLOWED_ATTRIBUTES.update(dict((x, ['id', ]) for x in (
'p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'code', 'dl', 'dt', 'dd',
@@ -61,6 +62,16 @@
'progress', 'audio', 'video', 'details', 'datagrid', 'datalist', 'table',
'address'
)))
+ALLOWED_STYLES = [
+ 'border', 'float', 'overflow', 'min-height', 'vertical-align',
+ 'white-space',
+ 'margin', 'margin-left', 'margin-top', 'margin-bottom', 'margin-right',
+ 'padding', 'padding-left', 'padding-top', 'padding-bottom', 'padding-right',
+ 'background', # TODO: Maybe not this one, it can load URLs
+ 'background-color',
+ 'font', 'font-size', 'font-weight', 'text-align', 'text-transform',
+ '-moz-column-width', '-webkit-columns', 'columns',
+]
# Disruptiveness of edits to translated versions. Numerical magnitude indicate
# the relative severity.
@@ -549,10 +560,10 @@ def locale_and_slug_from_path(path, request=None):
if '/' in path:
locale, slug = path.split('/', 1)
- if locale in settings.MT_TO_KUMA_LOCALE_MAP:
+ if locale.lower() in settings.MT_TO_KUMA_LOCALE_MAP:
# If this looks like a MindTouch locale, remap it.
old_locale = locale
- locale = settings.MT_TO_KUMA_LOCALE_MAP[locale]
+ locale = settings.MT_TO_KUMA_LOCALE_MAP[locale.lower()]
# But, we only need a redirect if the locale actually changed.
needs_redirect = (locale != old_locale)
@@ -938,7 +949,7 @@ def content_cleaned(self):
return self.content
return bleach.clean(
self.content, attributes=ALLOWED_ATTRIBUTES, tags=ALLOWED_TAGS,
- strip_comments=False
+ styles=ALLOWED_STYLES, strip_comments=False
)
def get_previous(self):
View
18 apps/wiki/templates/wiki/document.html
@@ -1,9 +1,7 @@
{# vim: set ts=2 et sts=2 sw=2: #}
{% extends "wiki/base.html" %}
{% from "wiki/includes/sidebar_modules.html" import document_tabs, document_notifications %}
-{# L10n: {t} is the title of the document. {c} is the category. #}
-{% set title = _('{t} | {c}')|f(t=document.title, c=document.get_category_display()) %}
-{% block title %}{{ page_title(title) }}{% endblock %}
+{% block title %}{{ page_title(document.title) }}{% endblock %}
{% set classes = 'document' %}
{% block bodyclass %}document{% endblock %}
{% if document.parent %}
@@ -38,19 +36,7 @@ <h1 class="page-title">{{ document.title }}</h1>
{% endif %}
</ul>
{% if kumascript_errors %}
- <div class="warning" id="kumascript-errors">
- <p>{{ _("There are scripting errors on this page:") }}</p>
- <ul>
- {% for error in kumascript_errors %}
- <li class="error error-{{ error.level }}">
- {# <span class="level">{{ error.level }}</span> #}
- {% if error.args %}<span class="type">{{ error.args[0] }}</span>{% endif %}
- &#8212;
- <span class="message">{{ error.message }}</span>
- </li>
- {% endfor %}
- </ul>
- </div>
+ {% include 'wiki/includes/kumascript_errors.html' %}
{% endif %}
</header>
{% if redirected_from %}
View
2 apps/wiki/templates/wiki/edit_document.html
@@ -2,7 +2,7 @@
{% extends "wiki/base.html" %}
{% from "layout/errorlist.html" import errorlist %}
{% from "wiki/includes/sidebar_modules.html" import document_tabs %}
-{% set title = _('Edit Article | {document}')|f(document=document.title) %}
+{% set title = _('{document} | Edit Article')|f(document=document.title) %}
{% block title %}{{ page_title(title) }}{% endblock %}
{# TODO: Change KB url to landing page when we have one #}
{% set crumbs = [(url('wiki.category', document.category), document.get_category_display()),
View
45 apps/wiki/templates/wiki/includes/kumascript_errors.html
@@ -0,0 +1,45 @@
+<div class="warning" id="kumascript-errors">
+<p>{{ _("There are scripting errors on this page:") }}</p>
+<ul>
+ {% for error in kumascript_errors %}
+ <li class="error error-{{ error.level }}">
+ {% if error.args %}
+ {% set err_type = error.args[0] %}
+ <strong class="type">{{ err_type }}</strong>
+ {% if err_type == 'TemplateExecutionError' %}
+ {% set options = error.args[2] %}
+ {% set token = options.token %}
+ {% set template_name = token.name %}
+ {% set template_args = token.args %}
+ {% set template_slug = 'Template:{name}' | f(name=template_name) %}
+ {% set template_path = ('{locale}/{slug}' | f(locale='en-US', slug=template_slug)) %}
+ {% set edit_url = url('wiki.edit_document', template_path) %}
+ <span>
+ at document offset {{ token['offset'] }}
+ in macro <code>{{ template_name }} ({{ template_args }})</code>
+ ( <a href="{{ edit_url }}">edit</a> ):
+ </span>
+ {% endif %}
+ {% if err_type == 'TemplateLoadingError' %}
+ {% set options = error.args[2] %}
+ {% set template_name = options.name %}
+ {% set template_slug = 'Template:{name}' | f(name=template_name) %}
+ {% set template_path = ('{locale}/{slug}' | f(locale='en-US', slug=template_slug)) %}
+ {% set edit_url = url('wiki.edit_document', template_path) %}
+ {% set new_url = url('wiki.new_document') %}
+ <span>
+ for <code>{{ template_slug }}</code> (
+ {% if 'status 404' in error.message %}
+ <a href="{{ new_url }}?slug={{ template_slug }}">new</a>
+ {% else %}
+ <a href="{{ edit_url }}">edit</a>
+ {% endif %}
+ ):
+ </span>
+ {% endif %}
+ {% endif %}
+ <pre class="message brush: text">{{ error.message }}</pre>
+ </li>
+ {% endfor %}
+</ul>
+</div>
View
48 apps/wiki/tests/test_content.py
@@ -1,5 +1,6 @@
# This Python file uses the following encoding: utf-8
# see also: http://www.python.org/dev/peps/pep-0263/
+import logging
from nose.tools import eq_, ok_
from nose.plugins.attrib import attr
@@ -19,18 +20,18 @@ class ContentSectionToolTests(TestCase):
def test_section_ids(self):
doc_src = """
- <h1>head</h1>
+ <h1 class="header1">Header One</h1>
<p>test</p>
<section>
- <h1>head</h1>
+ <h1 class="header2">Header Two</h1>
<p>test</p>
</section>
- <h2>head</h2>
+ <h2 name="Constants" class="hasname">This title does not match the name</h2>
<p>test</p>
- <h1 id="i-already-have-an-id" class="hasid">head</h1>
+ <h1 id="i-already-have-an-id" class="hasid">This text clobbers the ID</h1>
- <h1>head</h1>
+ <h1 class="header3">Header Three</h1>
<p>test</p>
"""
@@ -40,8 +41,14 @@ def test_section_ids(self):
.serialize())
result_doc = pq(result_src)
- # First, ensure an existing ID hasn't been disturbed
- eq_('i-already-have-an-id', result_doc.find('.hasid').attr('id'))
+ expected = (
+ ('header1', 'Header_One'),
+ ('header2', 'Header_Two'),
+ ('hasname', 'Constants'),
+ ('hasid', 'This_text_clobbers_the_ID'),
+ )
+ for cls, id in expected:
+ eq_(id, result_doc.find('.%s' % cls).attr('id'))
# Then, ensure all elements in need of an ID now all have unique IDs.
ok_(len(SECTION_TAGS) > 0)
@@ -366,11 +373,12 @@ def test_generate_toc(self):
.filter(SectionTOCFilter).serialize())
eq_(normalize_html(expected), normalize_html(result))
- @attr('current')
def test_dekiscript_macro_conversion(self):
doc_src = u"""
<span>Just a span</span>
<span class="notascript">Hi there</span>
+ <li><span class="script">Warning("Performing synchronous IO on the main thread can cause serious performance problems. As a result, this method of modifying the database is <strong>strongly</strong> discouraged!")</span></li>
+ <li><span class="script">Note("Performing synchronous IO on the main thread can cause serious performance problems. As a result, this method of modifying the database is <strong class="important">strongly</strong> discouraged!")</span></li>
<li><span class="script">MixedCaseName('parameter1', 'parameter2')</span></li>
<li><span class="script">bug(689641)</span></li>
<li><span class="script">template.lowercasename('border')</span></li>
@@ -383,6 +391,8 @@ def test_dekiscript_macro_conversion(self):
expected = u"""
<span>Just a span</span>
<span class="notascript">Hi there</span>
+ <li>{{ Warning("Performing synchronous IO on the main thread can cause serious performance problems. As a result, this method of modifying the database is <strong>strongly</strong> discouraged!") }}</li>
+ <li>{{ Note("Performing synchronous IO on the main thread can cause serious performance problems. As a result, this method of modifying the database is <strong class="important">strongly</strong> discouraged!") }}</li>
<li>{{ MixedCaseName('parameter1', 'parameter2') }}</li>
<li>{{ bug("689641") }}</li>
<li>{{ lowercasename('border') }}</li>
@@ -408,6 +418,28 @@ def test_dekiscript_macro_conversion(self):
.filter(DekiscriptMacroFilter).serialize())
eq_(normalize_html(expected), normalize_html(result))
+ def test_noinclude(self):
+ doc_src = u"""
+ <div class="noinclude">{{ XULRefAttr() }}</div>
+ <dl>
+ <dt>{{ XULAttr(&quot;maxlength&quot;) }}</dt>
+ <dd>Type: <em>integer</em></dd>
+ <dd>Przykłady 例 예제 示例</dd>
+ </dl>
+ <div class="noinclude">
+ <p>{{ languages( { &quot;ja&quot;: &quot;ja/XUL/Attribute/maxlength&quot; } ) }}</p>
+ </div>
+ """
+ expected = u"""
+ <dl>
+ <dt>{{ XULAttr(&quot;maxlength&quot;) }}</dt>
+ <dd>Type: <em>integer</em></dd>
+ <dd>Przykłady 例 예제 示例</dd>
+ </dl>
+ """
+ result = (wiki.content.filter_out_noinclude(doc_src))
+ eq_(normalize_html(expected), normalize_html(result))
+
class AllowedHTMLTests(TestCase):
simple_tags = (
View
20 apps/wiki/tests/test_forms.py
@@ -26,20 +26,20 @@ def test_form_loaded_with_section(self):
"""RevisionForm given section_id should load initial content for only
one section"""
d, r = doc_rev("""
- <h1 id="s1">Head 1</h1>
+ <h1 id="s1">s1</h1>
<p>test</p>
<p>test</p>
- <h1 id="s2">Head 2</h1>
+ <h1 id="s2">s2</h1>
<p>test</p>
<p>test</p>
- <h1 id="s3">Head 3</h1>
+ <h1 id="s3">s3</h1>
<p>test</p>
<p>test</p>
""")
expected = """
- <h1 id="s2">Head 2</h1>
+ <h1 id="s2">s2</h1>
<p>test</p>
<p>test</p>
"""
@@ -49,15 +49,15 @@ def test_form_loaded_with_section(self):
def test_form_save_section(self):
d, r = doc_rev("""
- <h1 id="s1">Head 1</h1>
+ <h1 id="s1">s1</h1>
<p>test</p>
<p>test</p>
- <h1 id="s2">Head 2</h1>
+ <h1 id="s2">s2</h1>
<p>test</p>
<p>test</p>
- <h1 id="s3">Head 3</h1>
+ <h1 id="s3">s3</h1>
<p>test</p>
<p>test</p>
""")
@@ -66,14 +66,14 @@ def test_form_save_section(self):
<p>new stuff</p>
"""
expected = """
- <h1 id="s1">Head 1</h1>
+ <h1 id="s1">s1</h1>
<p>test</p>
<p>test</p>
- <h1 id="s2">New stuff</h1>
+ <h1 id="New_stuff">New stuff</h1>
<p>new stuff</p>
- <h1 id="s3">Head 3</h1>
+ <h1 id="s3">s3</h1>
<p>test</p>
<p>test</p>
"""
View
160 apps/wiki/tests/test_views.py
@@ -1,6 +1,9 @@
+# This Python file uses the following encoding: utf-8
+# see also: http://www.python.org/dev/peps/pep-0263/
import logging
import json
import base64
+import time
from django.conf import settings
from django.contrib.sites.models import Site
@@ -229,6 +232,7 @@ def setUp(self):
super(KumascriptIntegrationTests, self).setUp()
self.d, self.r = doc_rev()
+ self.d.tags.set('foo', 'bar', 'baz')
self.url = reverse('wiki.document',
args=['%s/%s' % (self.d.locale, self.d.slug)],
locale=settings.WIKI_DEFAULT_LANGUAGE)
@@ -459,6 +463,41 @@ def my_requests_get(url, headers=None, timeout=None):
for error in expected_errors['logs']:
ok_(error['message'] in response.content)
+ @mock.patch('requests.get')
+ def test_env_vars(self, mock_requests_get):
+ """Kumascript reports errors in HTTP headers, Kuma should display them"""
+
+ # Now, trap the request from the view.
+ trap = {}
+ def my_requests_get(url, headers=None, timeout=None):
+ trap['headers'] = headers
+ return FakeResponse(
+ status_code=200,
+ body='HELLO WORLD',
+ headers={}
+ )
+ mock_requests_get.side_effect = my_requests_get
@groovecoder
Mozilla member
groovecoder added a note Apr 19, 2012

why side_effect here?

@lmorchard
Mozilla member
lmorchard added a note Apr 19, 2012

That's how my_requests_get gets called, and the headers trapped.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
+
+ # Ensure kumascript is enabled
+ constance.config.KUMASCRIPT_TIMEOUT = 1.0
+ constance.config.KUMASCRIPT_MAX_AGE = 600
+
+ # Fire off the request, and capture the env vars that would have been
+ # sent to kumascript
+ response = self.client.get(self.url)
+ pfx = 'x-kumascript-env-'
+ vars = dict(
+ (k[len(pfx):], json.loads(base64.b64decode(v)))
+ for k,v in trap['headers'].items()
+ if k.startswith(pfx))
+
+ # Ensure the env vars intended for kumascript match expected values.
+ for n in ('title', 'slug', 'locale'):
+ eq_(getattr(self.d, n), vars[n])
+ eq_(self.d.get_absolute_url(), vars['path'])
+ eq_(time.mktime(self.d.modified.timetuple()), vars['modified'])
+ eq_(sorted([u'foo', u'bar', u'baz']), sorted(vars['tags']))
+
class DocumentEditingTests(TestCaseBase):
"""Tests for the document-editing view"""
@@ -484,50 +523,48 @@ def test_retitling(self):
locale=d.locale).title)
assert "REDIRECT" in Document.uncached.get(title=old_title).html
- def test_retitling_ignored_for_iframe(self):
+ def test_slug_change_ignored_for_iframe(self):
"""When the title of an article is edited in an iframe, the change is
ignored."""
client = LocalizingClient()
client.login(username='admin', password='testpass')
- new_title = 'Some New Title'
+ new_slug = 'some_new_slug'
d, r = doc_rev()
- old_title = d.title
+ old_slug = d.slug
data = new_document_data()
- data.update({'title': new_title,
- 'slug': d.slug,
+ data.update({'title': d.title,
+ 'slug': new_slug,
'form': 'rev'})
client.post('%s?iframe=1' % reverse('wiki.edit_document',
args=[d.full_path]), data)
- eq_(old_title, Document.uncached.get(slug=d.slug,
- locale=d.locale).title)
- assert "REDIRECT" not in Document.uncached.get(title=old_title).html
+ eq_(old_slug, Document.uncached.get(slug=d.slug,
+ locale=d.locale).slug)
+ assert "REDIRECT" not in Document.uncached.get(slug=old_slug).html
@attr('clobber')
- def test_title_slug_collision_errors(self):
+ def test_slug_collision_errors(self):
"""When an attempt is made to retitle an article and another with that
title already exists, there should be form errors"""
client = LocalizingClient()
client.login(username='admin', password='testpass')
- exist_title = "Existing doc"
exist_slug = "existing-doc"
# Create a new doc.
data = new_document_data()
- data.update({ "title": exist_title, "slug": exist_slug })
+ data.update({"slug": exist_slug})
resp = client.post(reverse('wiki.new_document'), data)
eq_(302, resp.status_code)
# Create another new doc.
data = new_document_data()
- data.update({ "title": 'Some new title', "slug": 'some-new-title' })
+ data.update({"slug": 'some-new-title'})
resp = client.post(reverse('wiki.new_document'), data)
eq_(302, resp.status_code)
- # Now, post an update with duplicate slug and title
+ # Now, post an update with duplicate slug
data.update({
'form': 'rev',
- 'title': exist_title,
'slug': exist_slug
})
resp = client.post(reverse('wiki.edit_document',
@@ -537,7 +574,6 @@ def test_title_slug_collision_errors(self):
p = pq(resp.content)
ok_(p.find('.errorlist').length > 0)
- ok_(p.find('.errorlist a[href="#id_title"]').length > 0)
ok_(p.find('.errorlist a[href="#id_slug"]').length > 0)
@attr('clobber')
@@ -926,28 +962,28 @@ def test_raw_source(self):
client = LocalizingClient()
client.login(username='admin', password='testpass')
d, r = doc_rev("""
- <h1 id="s1">Head 1</h1>
+ <h1 id="s1">s1</h1>
<p>test</p>
<p>test</p>
- <h1 id="s2">Head 2</h1>
+ <h1 id="s2">s2</h1>
<p>test</p>
<p>test</p>
- <h1 id="s3">Head 3</h1>
+ <h1 id="s3">s3</h1>
<p>test</p>
<p>test</p>
""")
expected = """
- <h1 id="s1">Head 1</h1>
+ <h1 id="s1">s1</h1>
<p>test</p>
<p>test</p>
- <h1 id="s2">Head 2</h1>
+ <h1 id="s2">s2</h1>
<p>test</p>
<p>test</p>
- <h1 id="s3">Head 3</h1>
+ <h1 id="s3">s3</h1>
<p>test</p>
<p>test</p>
"""
@@ -962,26 +998,26 @@ def test_raw_with_editing_links_source(self):
client = LocalizingClient()
client.login(username='admin', password='testpass')
d, r = doc_rev("""
- <h1 id="s1">Head 1</h1>
+ <h1 id="s1">s1</h1>
<p>test</p>
<p>test</p>
- <h1 id="s2">Head 2</h1>
+ <h1 id="s2">s2</h1>
<p>test</p>
<p>test</p>
- <h1 id="s3">Head 3</h1>
+ <h1 id="s3">s3</h1>
<p>test</p>
<p>test</p>
""")
expected = """
- <h1 id="s1"><a class="edit-section" data-section-id="s1" data-section-src-url="/en-US/docs/%(full_path)s?raw=true&amp;section=s1" href="/en-US/docs/%(full_path)s$edit?section=s1&amp;edit_links=true" title="Edit section">Edit</a>Head 1</h1>
+ <h1 id="s1"><a class="edit-section" data-section-id="s1" data-section-src-url="/en-US/docs/%(full_path)s?raw=true&amp;section=s1" href="/en-US/docs/%(full_path)s$edit?section=s1&amp;edit_links=true" title="Edit section">Edit</a>s1</h1>
<p>test</p>
<p>test</p>
- <h1 id="s2"><a class="edit-section" data-section-id="s2" data-section-src-url="/en-US/docs/%(full_path)s?raw=true&amp;section=s2" href="/en-US/docs/%(full_path)s$edit?section=s2&amp;edit_links=true" title="Edit section">Edit</a>Head 2</h1>
+ <h1 id="s2"><a class="edit-section" data-section-id="s2" data-section-src-url="/en-US/docs/%(full_path)s?raw=true&amp;section=s2" href="/en-US/docs/%(full_path)s$edit?section=s2&amp;edit_links=true" title="Edit section">Edit</a>s2</h1>
<p>test</p>
<p>test</p>
- <h1 id="s3"><a class="edit-section" data-section-id="s3" data-section-src-url="/en-US/docs/%(full_path)s?raw=true&amp;section=s3" href="/en-US/docs/%(full_path)s$edit?section=s3&amp;edit_links=true" title="Edit section">Edit</a>Head 3</h1>
+ <h1 id="s3"><a class="edit-section" data-section-id="s3" data-section-src-url="/en-US/docs/%(full_path)s?raw=true&amp;section=s3" href="/en-US/docs/%(full_path)s$edit?section=s3&amp;edit_links=true" title="Edit section">Edit</a>s3</h1>
<p>test</p>
<p>test</p>
""" % {'full_path': d.full_path}
@@ -995,20 +1031,20 @@ def test_raw_section_source(self):
client = LocalizingClient()
client.login(username='admin', password='testpass')
d, r = doc_rev("""
- <h1 id="s1">Head 1</h1>
+ <h1 id="s1">s1</h1>
<p>test</p>
<p>test</p>
- <h1 id="s2">Head 2</h1>
+ <h1 id="s2">s2</h1>
<p>test</p>
<p>test</p>
- <h1 id="s3">Head 3</h1>
+ <h1 id="s3">s3</h1>
<p>test</p>
<p>test</p>
""")
expected = """
- <h1 id="s2">Head 2</h1>
+ <h1 id="s2">s2</h1>
<p>test</p>
<p>test</p>
"""
@@ -1023,24 +1059,24 @@ def test_raw_section_edit(self):
client = LocalizingClient()
client.login(username='admin', password='testpass')
d, r = doc_rev("""
- <h1 id="s1">Head 1</h1>
+ <h1 id="s1">s1</h1>
<p>test</p>
<p>test</p>
- <h1 id="s2">Head 2</h1>
+ <h1 id="s2">s2</h1>
<p>test</p>
<p>test</p>
- <h1 id="s3">Head 3</h1>
+ <h1 id="s3">s3</h1>
<p>test</p>
<p>test</p>
""")
replace = """
- <h1 id="s2">Replace</h1>
+ <h1 id="s2">s2</h1>
<p>replace</p>
"""
expected = """
- <h1 id="s2">Replace</h1>
+ <h1 id="s2">s2</h1>
<p>replace</p>
"""
response = client.post('%s?section=s2&raw=true' %
@@ -1052,14 +1088,14 @@ def test_raw_section_edit(self):
normalize_html(response.content))
expected = """
- <h1 id="s1">Head 1</h1>
+ <h1 id="s1">s1</h1>
<p>test</p>
<p>test</p>
- <h1 id="s2">Replace</h1>
+ <h1 id="s2">s2</h1>
<p>replace</p>
- <h1 id="s3">Head 3</h1>
+ <h1 id="s3">s3</h1>
<p>test</p>
<p>test</p>
"""
@@ -1077,34 +1113,34 @@ def test_midair_section_merge(self):
client.login(username='admin', password='testpass')
doc, rev = doc_rev("""
- <h1 id="s1">Head 1</h1>
+ <h1 id="s1">s1</h1>
<p>test</p>
<p>test</p>
- <h1 id="s2">Head 2</h1>
+ <h1 id="s2">s2</h1>
<p>test</p>
<p>test</p>
- <h1 id="s3">Head 3</h1>
+ <h1 id="s3">s3</h1>
<p>test</p>
<p>test</p>
""")
replace_1 = """
- <h1 id="s1">replace</h1>
+ <h1 id="s1">replace1</h1>
<p>replace</p>
"""
replace_2 = """
- <h1 id="s2">replace</h1>
+ <h1 id="s2">replace2</h1>
<p>replace</p>
"""
expected = """
- <h1 id="s1">replace</h1>
+ <h1 id="replace1">replace1</h1>
<p>replace</p>
- <h1 id="s2">replace</h1>
+ <h1 id="replace2">replace2</h1>
<p>replace</p>
- <h1 id="s3">Head 3</h1>
+ <h1 id="s3">s3</h1>
<p>test</p>
<p>test</p>
"""
@@ -1169,15 +1205,15 @@ def test_midair_section_collision(self):
client.login(username='admin', password='testpass')
doc, rev = doc_rev("""
- <h1 id="s1">Head 1</h1>
+ <h1 id="s1">s1</h1>
<p>test</p>
<p>test</p>
- <h1 id="s2">Head 2</h1>
+ <h1 id="s2">s2</h1>
<p>test</p>
<p>test</p>
- <h1 id="s3">Head 3</h1>
+ <h1 id="s3">s3</h1>
<p>test</p>
<p>test</p>
""")
@@ -1229,6 +1265,30 @@ def test_midair_section_collision(self):
# With the raw API, we should get a 409 Conflict on collision.
eq_(409, resp.status_code)
+ def test_raw_include_option(self):
+ doc_src = u"""
+ <div class="noinclude">{{ XULRefAttr() }}</div>
+ <dl>
+ <dt>{{ XULAttr(&quot;maxlength&quot;) }}</dt>
+ <dd>Type: <em>integer</em></dd>
+ <dd>Przykłady 例 예제 示例</dd>
+ </dl>
+ <div class="noinclude">
+ <p>{{ languages( { &quot;ja&quot;: &quot;ja/XUL/Attribute/maxlength&quot; } ) }}</p>
+ </div>
+ """
+ doc, rev = doc_rev(doc_src)
+ expected = u"""
+ <dl>
+ <dt>{{ XULAttr(&quot;maxlength&quot;) }}</dt>
+ <dd>Type: <em>integer</em></dd>
+ <dd>Przykłady 例 예제 示例</dd>
+ </dl>
+ """
+ client = LocalizingClient()
+ resp = client.get('%s?raw&include' % reverse('wiki.document', args=[doc.full_path]))
+ eq_(normalize_html(expected), normalize_html(resp.content.decode('utf-8')))
+
@attr('kumawiki')
def test_kumawiki_waffle_flag(self):
View
39 apps/wiki/views.py
@@ -1,4 +1,5 @@
from datetime import datetime
+import time
import json
from collections import defaultdict
import base64
@@ -47,7 +48,8 @@
OPERATING_SYSTEMS, GROUPED_OPERATING_SYSTEMS,
FIREFOX_VERSIONS, GROUPED_FIREFOX_VERSIONS,
REVIEW_FLAG_TAGS_DEFAULT, ALLOWED_ATTRIBUTES,
- ALLOWED_TAGS, get_current_or_latest_revision)
+ ALLOWED_TAGS, ALLOWED_STYLES,
+ get_current_or_latest_revision)
from wiki.tasks import send_reviewed_notification, schedule_rebuild_kb
import wiki.content
@@ -127,7 +129,7 @@ def process(request, document_path=None, *args, **kwargs):
@waffle_flag('kumawiki')
-@require_GET
+@require_http_methods(['GET', 'HEAD'])
@process_document_path
def document(request, document_slug, document_locale):
"""View a wiki document."""
@@ -222,6 +224,7 @@ def set_common_headers(r):
# Grab some parameters that affect output
section_id = request.GET.get('section', None)
show_raw = request.GET.get('raw', False) is not False
+ is_include = request.GET.get('include', False) is not False
no_macros = request.GET.get('nomacros', False) is not False
force_macros = request.GET.get('macros', False) is not False
need_edit_links = request.GET.get('edit_links', False) is not False
@@ -240,7 +243,7 @@ def set_common_headers(r):
# * The request *has* asked for macro evaluation
# (eg. ?raw&macros)
resp_body, resp_errors = _perform_kumascript_request(
- request, response_headers, document_locale, document_slug)
+ request, response_headers, doc, document_locale, document_slug)
if resp_body:
doc_html = resp_body
if resp_errors:
@@ -270,6 +273,10 @@ def set_common_headers(r):
doc_html = tool.serialize()
+ # If this is an include, filter out the class="noinclude" blocks.
+ if is_include:
+ doc_html = (wiki.content.filter_out_noinclude(doc_html))
+
# if ?raw parameter is supplied, then we respond with raw page source
# without template wrapping or edit links. This is also permissive for
# iframe inclusion
@@ -313,8 +320,8 @@ def _invalidate_kumascript_cache(document):
document.locale))
-def _perform_kumascript_request(request, response_headers, document_locale,
- document_slug):
+def _perform_kumascript_request(request, response_headers, document,
+ document_locale, document_slug):
"""Perform a kumascript GET request for a document locale and slug.
This is broken out into its own utility function, both to make the view
@@ -358,6 +365,26 @@ def _perform_kumascript_request(request, response_headers, document_locale,
'Cache-Control': cache_control
}
+ # Assemble some KumaScript env vars
+ # TODO: See dekiscript vars for future inspiration
+ # http://developer.mindtouch.com/en/docs/DekiScript/Reference/Wiki_Functions_and_Variables
+ path = document.get_absolute_url()
+ env_vars = dict(
+ path=path,
+ url=request.build_absolute_uri(path),
+ id=document.pk,
+ locale=document.locale,
+ title=document.title,
+ slug=document.slug,
+ tags=[x.name for x in document.tags.all()],
+ modified=time.mktime(document.modified.timetuple()),
+ )
+ # Encode the vars as kumascript headers, as base64 JSON-encoded values.
+ headers.update(dict(
+ ('x-kumascript-env-%s' % k,
+ base64.b64encode(json.dumps(v)))
+ for k, v in env_vars.items()))
+
# Set up for conditional GET, if we have the details cached.
c_meta = cache.get_many([ck_etag, ck_modified])
if ck_etag in c_meta:
@@ -416,7 +443,7 @@ def _perform_kumascript_request(request, response_headers, document_locale,
# want sanitation, so it finally gets picked up here.
resp_body = bleach.clean(
resp_body, attributes=ALLOWED_ATTRIBUTES, tags=ALLOWED_TAGS,
- strip_comments=False
+ styles=ALLOWED_STYLES, strip_comments=False
)
# Cache the request for conditional GET, but use the max_age for
2 kumascript
@@ -1 +1 @@
-Subproject commit ceab707180771c479994b38328a29310d3130473
+Subproject commit fa32b79430d62db881bd30d47f4f3ed81429b177
View
24 kumascript_settings_local.json-dist
@@ -2,16 +2,32 @@
"log": {
"console": true,
"file": {
- "filename": "./kumascript.log",
+ "filename": "/home/vagrant/logs/kumascript.log",
"maxsize": 500000
}
},
"server": {
"port": 9080,
"numWorkers": 4,
"workerTimeout": 10000,
- "document_url_template": "https://developer.mozilla.org/en-US/docs/{path}",
- "template_url_template": "https://developer.mozilla.org/en-US/docs/en-US/Template:{path}",
- "template_class": "KumaEJSTemplate"
+ "document_url_template": "http://localhost/en-US/docs/{path}?raw=1",
+ "template_url_template": "http://localhost/en-US/docs/en-US/Template:{name}?raw=1",
+ "template_class": "EJSTemplate",
+ "autorequire": {
+ "mdn": "MDN:Common",
+ "Culture": "DekiScript:Culture",
+ "Date": "DekiScript:Date",
+ "Json": "DekiScript:Json",
+ "List": "DekiScript:List",
+ "Map": "DekiScript:Map",
+ "Meta": "DekiScript:Meta",
+ "Num": "DekiScript:Num",
+ "Page": "DekiScript:Page",
+ "String": "DekiScript:String",
+ "Uri": "DekiScript:Uri",
+ "Web": "DekiScript:Web",
+ "Wiki": "DekiScript:Wiki",
+ "Xml": "DekiScript:Xml"
+ }
}
}
View
1 media/js/libs/django/prepopulate.js
@@ -34,6 +34,7 @@
});
s = values.join(' ');
+ s = s.replace(' ', '_');
// "$" is used for verb delimiter in URLs
s = s.replace(/\$/g, '');
// trim to first num_chars chars
View
6 media/js/wiki.js
@@ -14,7 +14,9 @@
function init() {
$('select.enable-if-js').removeAttr('disabled');
- initPrepopulatedSlugs();
+ if ($('body').is('.new')) {
+ initPrepopulatedSlugs();
+ }
initDetailsTags();
if ($('body').is('.document') || $('body').is('.home')) { // Document page
@@ -39,7 +41,7 @@
initMetadataEditButton();
initSaveAndEditButtons();
initArticlePreview();
- initTitleAndSlugCheck();
+ // initTitleAndSlugCheck();
@groovecoder
Mozilla member
groovecoder added a note Apr 19, 2012

just comment? should we delete altogether?

@lmorchard
Mozilla member
lmorchard added a note Apr 19, 2012

Might be worth deleting... I just wanted to turn it off for now, and look into properly fixing it in the future. It's not updated for the Kuma world where locale + slug are unique rather than title + slug

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
// initDrafting();
}
if ($('body').is('.edit.is-template') ||
View
18 puppet/files/vagrant/kumascript_settings_local.json
@@ -12,6 +12,22 @@
"workerTimeout": 10000,
"document_url_template": "http://localhost/en-US/docs/{path}?raw=1",
"template_url_template": "http://localhost/en-US/docs/en-US/Template:{name}?raw=1",
- "template_class": "KumaEJSTemplate"
+ "template_class": "EJSTemplate",
+ "autorequire": {
+ "mdn": "MDN:Common",
+ "Culture": "DekiScript:Culture",
+ "Date": "DekiScript:Date",
+ "Json": "DekiScript:Json",
+ "List": "DekiScript:List",
+ "Map": "DekiScript:Map",
+ "Meta": "DekiScript:Meta",
+ "Num": "DekiScript:Num",
+ "Page": "DekiScript:Page",
+ "String": "DekiScript:String",
+ "Uri": "DekiScript:Uri",
+ "Web": "DekiScript:Web",
+ "Wiki": "DekiScript:Wiki",
+ "Xml": "DekiScript:Xml"
+ }
}
}
View
1 settings.py
@@ -653,6 +653,7 @@ def JINJA_CONFIG():
'syntaxhighlighter/scripts/shBrushJScript.js',
'syntaxhighlighter/scripts/shBrushPhp.js',
'syntaxhighlighter/scripts/shBrushXml.js',
+ 'syntaxhighlighter/scripts/shBrushPlain.js',
'js/wiki.js',
'js/main.js',
),
Something went wrong with that request. Please try again.