bug 730707: Bugfix for macro migration

* Bugfix for macro migration, where markup appears in the middle of a
  parameter. This happens in a lot of `note` and `warning` macros.

apps/wiki/content.py

@@ -1,6 +1,9 @@
+import logging
 import re
 from urllib import urlencode
 
+from xml.sax.saxutils import quoteattr
+
 import html5lib
 from html5lib.filters._base import Filter as html5lib_Filter
 
@@ -385,10 +388,24 @@ def __iter__(self):
                 continue
 
             ds_call = []
-            while len(buffer) and 'EndTag' != token['type']:
+            while len(buffer):
                 token = buffer.pop(0)
-                if 'Characters' == token['type']:
+                if token['type'] in ('Characters', 'SpaceCharacters'):
                     ds_call.append(token['data'])
+                elif 'StartTag' == token['type']:
+                    attrs = token['data']
+                    if attrs:
+                        a_out = (u' %s' % u' '.join(
+                            (u'%s=%s' % 
+                             (name, quoteattr(val))
+                             for name, val in attrs)))
+                    else:
+                        a_out = u''
+                    ds_call.append(u'<%s%s>' % (token['name'], a_out))
+                elif 'EndTag' == token['type']:
+                    if 'span' == token['name']:
+                        break
+                    ds_call.append('</%s>' % token['name'])
 
             ds_call = ''.join(ds_call).strip()
 
@@ -417,7 +434,11 @@ def __iter__(self):
             if m:
                 ds_call = '%s()' % (m.group(1))
 
-            yield dict(
-                type="Characters",
-                data='{{ %s }}' % ds_call
-            )
+            # HACK: This is dirty, but seems like the easiest way to
+            # reconstitute the token stream, including what gets parsed as
+            # markup in the middle of macro parameters.
+            #
+            # eg. {{ Note("This is <strong>strongly</strong> discouraged") }}
+            parsed = parse('{{ %s }}' % ds_call)
+            for token in parsed.stream:
+                yield token

apps/wiki/tests/test_content.py

@@ -371,6 +371,8 @@ def test_dekiscript_macro_conversion(self):
         doc_src = u"""
             <span>Just a span</span>
             <span class="notascript">Hi there</span>
+            <li><span class="script">Warning("Performing synchronous IO on the main thread can cause serious performance problems. As a result, this method of modifying the database is <strong>strongly</strong> discouraged!")</span></li>
+            <li><span class="script">Note("Performing synchronous IO on the main thread can cause serious performance problems. As a result, this method of modifying the database is <strong class="important">strongly</strong> discouraged!")</span></li>
             <li><span class="script">MixedCaseName('parameter1', 'parameter2')</span></li>
             <li><span class="script">bug(689641)</span></li>
             <li><span class="script">template.lowercasename('border')</span></li>
@@ -383,6 +385,8 @@ def test_dekiscript_macro_conversion(self):
         expected = u"""
             <span>Just a span</span>
             <span class="notascript">Hi there</span>
+            <li>{{ Warning("Performing synchronous IO on the main thread can cause serious performance problems. As a result, this method of modifying the database is <strong>strongly</strong> discouraged!") }}</li>
+            <li>{{ Note("Performing synchronous IO on the main thread can cause serious performance problems. As a result, this method of modifying the database is <strong class="important">strongly</strong> discouraged!") }}</li>
             <li>{{ MixedCaseName('parameter1', 'parameter2') }}</li>
             <li>{{ bug("689641") }}</li>
             <li>{{ lowercasename('border') }}</li>