Added tower to extract normalized msgids from Django templates. Bug 647352.

Author: Fred Wenzel

.gitignore

@@ -1,6 +1,6 @@
 *.py[co]
-*.pot
 *.mo
+*.[mp]o~
 ffenv
 
 ffdemo/settings_local.py

README.md

@@ -117,7 +117,26 @@ Apache, do this:
      AddType image/svg+xml svg
 
 
-## Javascript
+## Javascript (Minification)
 
-[Jim][Jim] is used for managing the javascript in Mozilla MarkUp. More information is available [here][Jim].
+[Jim][Jim] is used for managing the JavaScript in Mozilla MarkUp. More information is available [here][Jim].
 [Jim]: https://github.com/quirkey/jim
+
+
+## L10n
+
+Markup uses the [tower][tower] library for string extraction. To extract
+strings from template files, run something like:
+
+    ./manage.py extract
+    ./manage.py verbatimize --rename
+    ./manage.py merge
+
+For more information, consult the tower docs.
+
+To compile .po files into .mo, run:
+
+    cd locale
+    ./compile-mo.sh .
+
+[tower]: https://github.com/clouserw/tower

ffdemo/lib/__init__.py

@@ -0,0 +1,19 @@
+"""Monkeypatch to normalize blocktrans msgids to work with tower."""
+
+from tower import strip_whitespace
+
+from django.templatetags import i18n
+
+
+OldBlockTranslateNode = i18n.BlockTranslateNode
+
+class ShoehornedBlockTranslateNode(OldBlockTranslateNode):
+    def render_token_list(self, tokens):
+        """Strip whitespace from msgid before letting gettext touch it."""
+        rendered = super(ShoehornedBlockTranslateNode, self).render_token_list(
+            tokens)
+        return strip_whitespace(rendered[0]), rendered[1]
+
+
+def monkeypatch():
+    i18n.BlockTranslateNode = ShoehornedBlockTranslateNode

ffdemo/lib/shoehorn_l10n/__init__.py

@@ -0,0 +1,101 @@
+"""Modified Django templatize utility to translate Django L10n into something almost usable by Jinja2."""
+
+from cStringIO import StringIO
+import re
+
+
+dot_re = re.compile(r'\S')
+def blankout(src, char):
+    """
+    Changes every non-whitespace character to the given char.
+    Used in the templatize function.
+    """
+    return dot_re.sub(char, src)
+
+inline_re = re.compile(r"""^\s*trans\s+((?:".*?")|(?:'.*?'))\s*""")
+block_re = re.compile(r"""^\s*blocktrans(?:\s+|$)""")
+endblock_re = re.compile(r"""^\s*endblocktrans$""")
+plural_re = re.compile(r"""^\s*plural$""")
+constant_re = re.compile(r"""_\(((?:".*?")|(?:'.*?'))\)""")
+
+def templatize(src):
+    """
+    Turns a Django template into something that is understood by xgettext. It
+    does so by translating the Django translation tags into standard gettext
+    function invocations.
+    """
+    from django.template import Lexer, TOKEN_TEXT, TOKEN_VAR, TOKEN_BLOCK
+    out = StringIO()
+    intrans = False
+    inplural = False
+    singular = []
+    plural = []
+    for t in Lexer(src, None).tokenize():
+        if intrans:
+            if t.token_type == TOKEN_BLOCK:
+                endbmatch = endblock_re.match(t.contents)
+                pluralmatch = plural_re.match(t.contents)
+                if endbmatch:
+                    if inplural:
+                        out.write(' {{ ngettext(%r,%r,count) }} ' % (''.join(singular), ''.join(plural)))
+                        for part in singular:
+                            out.write(blankout(part, 'S'))
+                        for part in plural:
+                            out.write(blankout(part, 'P'))
+                    else:
+                        out.write(' {{ gettext(%r) }} ' % ''.join(singular))
+                        for part in singular:
+                            out.write(blankout(part, 'S'))
+                    intrans = False
+                    inplural = False
+                    singular = []
+                    plural = []
+                elif pluralmatch:
+                    inplural = True
+                else:
+                    raise SyntaxError("Translation blocks must not include other block tags: %s" % t.contents)
+            elif t.token_type == TOKEN_VAR:
+                if inplural:
+                    plural.append('%%(%s)s' % t.contents)
+                else:
+                    singular.append('%%(%s)s' % t.contents)
+            elif t.token_type == TOKEN_TEXT:
+                if inplural:
+                    plural.append(t.contents)
+                else:
+                    singular.append(t.contents)
+        else:
+            if t.token_type == TOKEN_BLOCK:
+                imatch = inline_re.match(t.contents)
+                bmatch = block_re.match(t.contents)
+                cmatches = constant_re.findall(t.contents)
+                if imatch:
+                    g = imatch.group(1)
+                    if g[0] == '"': g = g.strip('"')
+                    elif g[0] == "'": g = g.strip("'")
+                    out.write(' {{ gettext(%r) }} ' % g)
+                elif bmatch:
+                    for fmatch in constant_re.findall(t.contents):
+                        out.write(' {{ _(%s) }} ' % fmatch)
+                    intrans = True
+                    inplural = False
+                    singular = []
+                    plural = []
+                elif cmatches:
+                    for cmatch in cmatches:
+                        out.write(' {{ _(%s) }} ' % cmatch)
+                else:
+                    out.write(blankout(t.contents, 'B'))
+            elif t.token_type == TOKEN_VAR:
+                parts = t.contents.split('|')
+                cmatch = constant_re.match(parts[0])
+                if cmatch:
+                    out.write(' {{ _(%s) }} ' % cmatch.group(1))
+                for p in parts[1:]:
+                    if p.find(':_(') >= 0:
+                        out.write(' %s ' % p.split(':',1)[1])
+                    else:
+                        out.write(blankout(p, 'F'))
+            else:
+                out.write(blankout(t.contents, 'X'))
+    return out.getvalue()

ffdemo/lib/shoehorn_l10n/templatetag.py

@@ -0,0 +1,12 @@
+"""Teach tower L10n library to hopefully extract from django templates."""
+from StringIO import StringIO
+
+from tower.management.commands.extract import (extract_tower_template,
+                                               tweak_message)
+from lib.shoehorn_l10n.templatize import templatize
+
+
+def extract_django_template(fileobj, keywords, comment_tags, options):
+    src = fileobj.read()
+    templatized = StringIO(templatize(src))
+    return extract_tower_template(templatized, keywords, comment_tags, options)

ffdemo/lib/shoehorn_l10n/templatize.py

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# syntax:
+# compile-mo.sh locale-dir/
+
+function usage() {
+    echo "syntax:"
+    echo "compile.sh locale-dir/"
+    exit 1
+}
+
+# check if file and dir are there
+if [[ ($# -ne 1) || (! -d "$1") ]]; then usage; fi
+
+for lang in `find $1 -type f -name "*.po"`; do
+    dir=`dirname $lang`
+    stem=`basename $lang .po`
+    msgfmt -o ${dir}/${stem}.mo $lang
+done