Added tower to extract normalized msgids from Django templates. Bug 647352.

Author: Fred Wenzel

.gitignore

@@ -1,6 +1,6 @@
 *.py[co]
-*.pot
 *.mo
+*.[mp]o~
 ffenv
 
 ffdemo/settings_local.py

ffdemo/lib/__init__.py

@@ -0,0 +1,19 @@
+"""Monkeypatch to normalize blocktrans msgids to work with tower."""
+
+from tower import strip_whitespace
+
+from django.templatetags import i18n
+
+
+OldBlockTranslateNode = i18n.BlockTranslateNode
+
+class ShoehornedBlockTranslateNode(OldBlockTranslateNode):
+    def render_token_list(self, tokens):
+        """Strip whitespace from msgid before letting gettext touch it."""
+        rendered = super(ShoehornedBlockTranslateNode, self).render_token_list(
+            tokens)
+        return strip_whitespace(rendered[0]), rendered[1]
+
+
+def monkeypatch():
+    i18n.BlockTranslateNode = ShoehornedBlockTranslateNode

ffdemo/lib/shoehorn_l10n/__init__.py

@@ -0,0 +1,101 @@
+"""Modified Django templatize utility to translate Django L10n into something almost usable by Jinja2."""
+
+from cStringIO import StringIO
+import re
+
+
+dot_re = re.compile(r'\S')
+def blankout(src, char):
+    """
+    Changes every non-whitespace character to the given char.
+    Used in the templatize function.
+    """
+    return dot_re.sub(char, src)
+
+inline_re = re.compile(r"""^\s*trans\s+((?:".*?")|(?:'.*?'))\s*""")
+block_re = re.compile(r"""^\s*blocktrans(?:\s+|$)""")
+endblock_re = re.compile(r"""^\s*endblocktrans$""")
+plural_re = re.compile(r"""^\s*plural$""")
+constant_re = re.compile(r"""_\(((?:".*?")|(?:'.*?'))\)""")
+
+def templatize(src):
+    """
+    Turns a Django template into something that is understood by xgettext. It
+    does so by translating the Django translation tags into standard gettext
+    function invocations.
+    """
+    from django.template import Lexer, TOKEN_TEXT, TOKEN_VAR, TOKEN_BLOCK
+    out = StringIO()
+    intrans = False
+    inplural = False
+    singular = []
+    plural = []
+    for t in Lexer(src, None).tokenize():
+        if intrans:
+            if t.token_type == TOKEN_BLOCK:
+                endbmatch = endblock_re.match(t.contents)
+                pluralmatch = plural_re.match(t.contents)
+                if endbmatch:
+                    if inplural:
+                        out.write(' {{ ngettext(%r,%r,count) }} ' % (''.join(singular), ''.join(plural)))
+                        for part in singular:
+                            out.write(blankout(part, 'S'))
+                        for part in plural:
+                            out.write(blankout(part, 'P'))
+                    else:
+                        out.write(' {{ gettext(%r) }} ' % ''.join(singular))
+                        for part in singular:
+                            out.write(blankout(part, 'S'))
+                    intrans = False
+                    inplural = False
+                    singular = []
+                    plural = []
+                elif pluralmatch:
+                    inplural = True
+                else:
+                    raise SyntaxError("Translation blocks must not include other block tags: %s" % t.contents)
+            elif t.token_type == TOKEN_VAR:
+                if inplural:
+                    plural.append('%%(%s)s' % t.contents)
+                else:
+                    singular.append('%%(%s)s' % t.contents)
+            elif t.token_type == TOKEN_TEXT:
+                if inplural:
+                    plural.append(t.contents)
+                else:
+                    singular.append(t.contents)
+        else:
+            if t.token_type == TOKEN_BLOCK:
+                imatch = inline_re.match(t.contents)
+                bmatch = block_re.match(t.contents)
+                cmatches = constant_re.findall(t.contents)
+                if imatch:
+                    g = imatch.group(1)
+                    if g[0] == '"': g = g.strip('"')
+                    elif g[0] == "'": g = g.strip("'")
+                    out.write(' {{ gettext(%r) }} ' % g)
+                elif bmatch:
+                    for fmatch in constant_re.findall(t.contents):
+                        out.write(' {{ _(%s) }} ' % fmatch)
+                    intrans = True
+                    inplural = False
+                    singular = []
+                    plural = []
+                elif cmatches:
+                    for cmatch in cmatches:
+                        out.write(' {{ _(%s) }} ' % cmatch)
+                else:
+                    out.write(blankout(t.contents, 'B'))
+            elif t.token_type == TOKEN_VAR:
+                parts = t.contents.split('|')
+                cmatch = constant_re.match(parts[0])
+                if cmatch:
+                    out.write(' {{ _(%s) }} ' % cmatch.group(1))
+                for p in parts[1:]:
+                    if p.find(':_(') >= 0:
+                        out.write(' %s ' % p.split(':',1)[1])
+                    else:
+                        out.write(blankout(p, 'F'))
+            else:
+                out.write(blankout(t.contents, 'X'))
+    return out.getvalue()

ffdemo/lib/shoehorn_l10n/templatetag.py

@@ -0,0 +1,12 @@
+"""Teach tower L10n library to hopefully extract from django templates."""
+from StringIO import StringIO
+
+from tower.management.commands.extract import (extract_tower_template,
+                                               tweak_message)
+from lib.shoehorn_l10n.templatize import templatize
+
+
+def extract_django_template(fileobj, keywords, comment_tags, options):
+    src = fileobj.read()
+    templatized = StringIO(templatize(src))
+    return extract_tower_template(templatized, keywords, comment_tags, options)

ffdemo/lib/shoehorn_l10n/templatize.py

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# syntax:
+# compile-mo.sh locale-dir/
+
+function usage() {
+    echo "syntax:"
+    echo "compile.sh locale-dir/"
+    exit 1
+}
+
+# check if file and dir are there
+if [[ ($# -ne 1) || (! -d "$1") ]]; then usage; fi
+
+for lang in `find $1 -type f -name "*.po"`; do
+    dir=`dirname $lang`
+    stem=`basename $lang .po`
+    msgfmt -o ${dir}/${stem}.mo $lang
+done