moved trans extension from jinja2.i18n to jinja2.ext and fixed some m…

…ore unittests

--HG--
branch : trunk

jinja2/environment.py

@@ -23,7 +23,7 @@
 _spontaneous_environments = LRUCache(10)
 
 
-def _get_spontaneous_environment(*args):
+def get_spontaneous_environment(*args):
     """Return a new spontaneus environment.  A spontaneus environment is an
     unnamed and unaccessable (in theory) environment that is used for
     template generated from a string and not from the file system.
@@ -85,11 +85,21 @@ def __init__(self,
                  comment_end_string='#}',
                  line_statement_prefix=None,
                  trim_blocks=False,
+                 extensions=(),
                  optimized=True,
                  undefined=Undefined,
-                 loader=None,
-                 extensions=(),
-                 finalize=unicode):
+                 finalize=unicode,
+                 loader=None):
+        # !!Important notice!!
+        #   The constructor accepts quite a few arguments that should be
+        #   passed by keyword rather than position.  However it's important to
+        #   not change the order of arguments because it's used at least
+        #   internally in those cases:
+        #       -   spontaneus environments (i18n extension and Template)
+        #       -   unittests
+        #   If parameter changes are required only add parameters at the end
+        #   and don't change the arguments (or the defaults!) of the arguments
+        #   up to (but excluding) loader.
         """Here the possible initialization parameters:
 
         ========================= ============================================
@@ -109,15 +119,15 @@ def __init__(self,
         `trim_blocks`             If this is set to ``True`` the first newline
                                   after a block is removed (block, not
                                   variable tag!). Defaults to ``False``.
+        `extensions`              List of Jinja extensions to use.
         `optimized`               should the optimizer be enabled?  Default is
                                   ``True``.
         `undefined`               a subclass of `Undefined` that is used to
                                   represent undefined variables.
-        `loader`                  the loader which should be used.
-        `extensions`              List of Jinja extensions to use.
         `finalize`                A callable that finalizes the variable.  Per
                                   default this is `unicode`, other useful
                                   builtin finalizers are `escape`.
+        `loader`                  the loader which should be used.
         ========================= ============================================
         """
 
@@ -138,14 +148,6 @@ def __init__(self,
         self.line_statement_prefix = line_statement_prefix
         self.trim_blocks = trim_blocks
 
-        # load extensions
-        self.extensions = []
-        for extension in extensions:
-            if isinstance(extension, basestring):
-                extension = import_string(extension)
-            # extensions are instanciated early but initalized later.
-            self.extensions.append(object.__new__(extension))
-
         # runtime information
         self.undefined = undefined
         self.optimized = optimized
@@ -162,9 +164,12 @@ def __init__(self,
         # create lexer
         self.lexer = Lexer(self)
 
-        # initialize extensions
-        for extension in self.extensions:
-            extension.__init__(self)
+        # load extensions
+        self.extensions = []
+        for extension in extensions:
+            if isinstance(extension, basestring):
+                extension = import_string(extension)
+            self.extensions.append(extension(self))
 
     def subscribe(self, obj, argument):
         """Get an item or attribute of an object."""
@@ -282,17 +287,15 @@ def __new__(cls, source,
                 comment_end_string='#}',
                 line_statement_prefix=None,
                 trim_blocks=False,
+                extensions=(),
                 optimized=True,
                 undefined=Undefined,
-                extensions=(),
                 finalize=unicode):
-        # make sure extensions are hashable
-        extensions = tuple(extensions)
-        env = _get_spontaneous_environment(
+        env = get_spontaneous_environment(
             block_start_string, block_end_string, variable_start_string,
             variable_end_string, comment_start_string, comment_end_string,
-            line_statement_prefix, trim_blocks, optimized, undefined,
-            None, extensions, finalize)
+            line_statement_prefix, trim_blocks, tuple(extensions), optimized,
+            undefined, finalize)
         return env.from_string(source, template_class=cls)
 
     def render(self, *args, **kwargs):
@@ -402,11 +405,9 @@ def generator():
 
             while 1:
                 try:
-                    while 1:
+                    while c_size < size:
                         push(next())
                         c_size += 1
-                        if c_size >= size:
-                            raise StopIteration()
                 except StopIteration:
                     if not c_size:
                         raise

jinja2/ext.py

@@ -3,17 +3,26 @@
     jinja2.ext
     ~~~~~~~~~~
 
-    Jinja extensions (EXPERIMENAL)
-
-    The plan: i18n and caching becomes a parser extension. cache/endcache
-    as well as trans/endtrans are not keyword and don't have nodes but
-    translate into regular jinja nodes so that the person who writes such
-    custom tags doesn't have to generate python code himself.
+    Jinja extensions allow to add custom tags similar to the way django custom
+    tags work.  By default two example extensions exist: an i18n and a cache
+    extension.
 
     :copyright: Copyright 2008 by Armin Ronacher.
     :license: BSD.
 """
+from collections import deque
 from jinja2 import nodes
+from jinja2.environment import get_spontaneous_environment
+from jinja2.runtime import Undefined
+from jinja2.parser import statement_end_tokens
+from jinja2.exceptions import TemplateAssertionError
+from jinja2.utils import import_string
+
+
+# the only real useful gettext functions for a Jinja template.  Note
+# that ugettext must be assigned to gettext as Jinja doesn't support
+# non unicode strings.
+GETTEXT_FUNCTIONS = ('_', 'gettext', 'ngettext')
 
 
 class Extension(object):
@@ -51,3 +60,235 @@ def parse(self, parser):
             nodes.Call(nodes.Name('cache_support', 'load'), args, [], None, None),
             [], [], body
         )
+
+
+class TransExtension(Extension):
+    """This extension adds gettext support to Jinja."""
+    tags = set(['trans'])
+
+    def __init__(self, environment):
+        Extension.__init__(self, environment)
+        environment.globals.update({
+            '_':        lambda x: x,
+            'gettext':  lambda x: x,
+            'ngettext': lambda s, p, n: (s, p)[n != 1]
+        })
+
+    def parse(self, parser):
+        """Parse a translatable tag."""
+        lineno = parser.stream.next().lineno
+
+        # skip colon for python compatibility
+        if parser.stream.current.type is 'colon':
+            parser.stream.next()
+
+        # find all the variables referenced.  Additionally a variable can be
+        # defined in the body of the trans block too, but this is checked at
+        # a later state.
+        plural_expr = None
+        variables = {}
+        while parser.stream.current.type is not 'block_end':
+            if variables:
+                parser.stream.expect('comma')
+            name = parser.stream.expect('name')
+            if name.value in variables:
+                raise TemplateAssertionError('translatable variable %r defined '
+                                             'twice.' % name.value, name.lineno,
+                                             parser.filename)
+
+            # expressions
+            if parser.stream.current.type is 'assign':
+                parser.stream.next()
+                variables[name.value] = var = parser.parse_expression()
+            else:
+                variables[name.value] = var = nodes.Name(name.value, 'load')
+            if plural_expr is None:
+                plural_expr = var
+        parser.stream.expect('block_end')
+
+        plural = plural_names = None
+        have_plural = False
+        referenced = set()
+
+        # now parse until endtrans or pluralize
+        singular_names, singular = self._parse_block(parser, True)
+        if singular_names:
+            referenced.update(singular_names)
+            if plural_expr is None:
+                plural_expr = nodes.Name(singular_names[0], 'load')
+
+        # if we have a pluralize block, we parse that too
+        if parser.stream.current.test('name:pluralize'):
+            have_plural = True
+            parser.stream.next()
+            if parser.stream.current.type is not 'block_end':
+                plural_expr = parser.parse_expression()
+            parser.stream.expect('block_end')
+            plural_names, plural = self._parse_block(parser, False)
+            parser.stream.next()
+            referenced.update(plural_names)
+        else:
+            parser.stream.next()
+
+        # register free names as simple name expressions
+        for var in referenced:
+            if var not in variables:
+                variables[var] = nodes.Name(var, 'load')
+
+        # no variables referenced?  no need to escape
+        if not referenced:
+            singular = singular.replace('%%', '%')
+            if plural:
+                plural = plural.replace('%%', '%')
+
+        if not have_plural:
+            plural_expr = None
+        elif plural_expr is None:
+            raise TemplateAssertionError('pluralize without variables',
+                                         lineno, parser.filename)
+
+        if variables:
+            variables = nodes.Dict([nodes.Pair(nodes.Const(x, lineno=lineno), y)
+                                    for x, y in variables.items()])
+        else:
+            variables = None
+
+        node = self._make_node(singular, plural, variables, plural_expr)
+        node.set_lineno(lineno)
+        return node
+
+    def _parse_block(self, parser, allow_pluralize):
+        """Parse until the next block tag with a given name."""
+        referenced = []
+        buf = []
+        while 1:
+            if parser.stream.current.type is 'data':
+                buf.append(parser.stream.current.value.replace('%', '%%'))
+                parser.stream.next()
+            elif parser.stream.current.type is 'variable_begin':
+                parser.stream.next()
+                name = parser.stream.expect('name').value
+                referenced.append(name)
+                buf.append('%%(%s)s' % name)
+                parser.stream.expect('variable_end')
+            elif parser.stream.current.type is 'block_begin':
+                parser.stream.next()
+                if parser.stream.current.test('name:endtrans'):
+                    break
+                elif parser.stream.current.test('name:pluralize'):
+                    if allow_pluralize:
+                        break
+                    raise TemplateSyntaxError('a translatable section can '
+                                              'have only one pluralize '
+                                              'section',
+                                              parser.stream.current.lineno,
+                                              parser.filename)
+                raise TemplateSyntaxError('control structures in translatable'
+                                          ' sections are not allowed.',
+                                          parser.stream.current.lineno,
+                                          parser.filename)
+            else:
+                assert False, 'internal parser error'
+
+        return referenced, u''.join(buf)
+
+    def _make_node(self, singular, plural, variables, plural_expr):
+        """Generates a useful node from the data provided."""
+        # singular only:
+        if plural_expr is None:
+            gettext = nodes.Name('gettext', 'load')
+            node = nodes.Call(gettext, [nodes.Const(singular)],
+                              [], None, None)
+            if variables:
+                node = nodes.Mod(node, variables)
+
+        # singular and plural
+        else:
+            ngettext = nodes.Name('ngettext', 'load')
+            node = nodes.Call(ngettext, [
+                nodes.Const(singular),
+                nodes.Const(plural),
+                plural_expr
+            ], [], None, None)
+            if variables:
+                node = nodes.Mod(node, variables)
+        return nodes.Output([node])
+
+
+def extract_from_ast(node, gettext_functions=GETTEXT_FUNCTIONS):
+    """Extract localizable strings from the given template node.
+
+    For every string found this function yields a ``(lineno, function,
+    message)`` tuple, where:
+
+    * ``lineno`` is the number of the line on which the string was found,
+    * ``function`` is the name of the ``gettext`` function used (if the
+      string was extracted from embedded Python code), and
+    *  ``message`` is the string itself (a ``unicode`` object, or a tuple
+       of ``unicode`` objects for functions with multiple string arguments).
+    """
+    for node in node.find_all(nodes.Call):
+        if not isinstance(node.node, nodes.Name) or \
+           node.node.name not in gettext_functions:
+            continue
+
+        strings = []
+        for arg in node.args:
+            if isinstance(arg, nodes.Const) and \
+               isinstance(arg.value, basestring):
+                strings.append(arg.value)
+            else:
+                strings.append(None)
+
+        if len(strings) == 1:
+            strings = strings[0]
+        else:
+            strings = tuple(strings)
+        yield node.lineno, node.node.name, strings
+
+
+def babel_extract(fileobj, keywords, comment_tags, options):
+    """Babel extraction method for Jinja templates.
+
+    :param fileobj: the file-like object the messages should be extracted from
+    :param keywords: a list of keywords (i.e. function names) that should be
+                     recognized as translation functions
+    :param comment_tags: a list of translator tags to search for and include
+                         in the results.  (Unused)
+    :param options: a dictionary of additional options (optional)
+    :return: an iterator over ``(lineno, funcname, message, comments)`` tuples.
+             (comments will be empty currently)
+    """
+    encoding = options.get('encoding', 'utf-8')
+
+    have_trans_extension = False
+    extensions = []
+    for extension in options.get('extensions', '').split(','):
+        extension = extension.strip()
+        if not extension:
+            continue
+        extension = import_string(extension)
+        if extension is TransExtension:
+            have_trans_extension = True
+        extensions.append(extension)
+    if not have_trans_extension:
+        extensions.append(TransExtension)
+
+    environment = get_spontaneous_environment(
+        options.get('block_start_string', '{%'),
+        options.get('block_end_string', '%}'),
+        options.get('variable_start_string', '{{'),
+        options.get('variable_end_string', '}}'),
+        options.get('comment_start_string', '{#'),
+        options.get('comment_end_string', '#}'),
+        options.get('line_statement_prefix') or None,
+        options.get('trim_blocks', '').lower() in ('1', 'on', 'yes', 'true'),
+        tuple(extensions),
+        # fill with defaults so that environments are shared
+        # with other spontaneus environments.
+        True, Undefined, unicode
+    )
+
+    node = environment.parse(fileobj.read().decode(encoding))
+    for lineno, func, message in extract_from_ast(node, keywords):
+        yield lineno, func, message, []