Skip to content
This repository
Browse code

added support for translator comments if extracted via babel.

--HG--
branch : trunk
  • Loading branch information...
commit 531578d5aae0fbb42afaa868e63e42ea460bfacb 1 parent ac7adf2
Armin Ronacher authored

Showing 3 changed files with 72 additions and 2 deletions. Show diff stats Hide diff stats

  1. +1 0  CHANGES
  2. +54 2 jinja2/ext.py
  3. +17 0 tests/test_i18n.py
1  CHANGES
@@ -18,6 +18,7 @@ Version 2.3
18 18 - implicit tuple expressions can no longer be totally empty.
19 19 This change makes ``{% if %}...{% endif %}`` a syntax error
20 20 now. (#364)
  21 +- added support for translator comments if extracted via babel.
21 22
22 23 Version 2.2.1
23 24 -------------
56 jinja2/ext.py
@@ -367,6 +367,10 @@ def extract_from_ast(node, gettext_functions=GETTEXT_FUNCTIONS,
367 367 string was extracted from embedded Python code), and
368 368 * ``message`` is the string itself (a ``unicode`` object, or a tuple
369 369 of ``unicode`` objects for functions with multiple string arguments).
  370 +
  371 + This extraction function operates on the AST and is because of that unable
  372 + to extract any comments. For comment support you have to use the babel
  373 + extraction interface or extract comments yourself.
370 374 """
371 375 for node in node.find_all(nodes.Call):
372 376 if not isinstance(node.node, nodes.Name) or \
@@ -400,14 +404,59 @@ def extract_from_ast(node, gettext_functions=GETTEXT_FUNCTIONS,
400 404 yield node.lineno, node.node.name, strings
401 405
402 406
  407 +class _CommentFinder(object):
  408 + """Helper class to find comments in a token stream. Can only
  409 + find comments for gettext calls forwards. Once the comment
  410 + from line 4 is found, a comment for line 1 will not return a
  411 + usable value.
  412 + """
  413 +
  414 + def __init__(self, tokens, comment_tags):
  415 + self.tokens = tokens
  416 + self.comment_tags = comment_tags
  417 + self.offset = 0
  418 + self.last_lineno = 0
  419 +
  420 + def find_backwards(self, offset):
  421 + try:
  422 + for _, token_type, token_value in \
  423 + reversed(self.tokens[self.offset:offset]):
  424 + if token_type in ('comment', 'linecomment'):
  425 + try:
  426 + prefix, comment = token_value.split(None, 1)
  427 + except ValueError:
  428 + continue
  429 + if prefix in self.comment_tags:
  430 + return [comment.rstrip()]
  431 + return []
  432 + finally:
  433 + self.offset = offset
  434 +
  435 + def find_comments(self, lineno):
  436 + if not self.comment_tags or self.last_lineno > lineno:
  437 + return []
  438 + for idx, (token_lineno, _, _) in enumerate(self.tokens[self.offset:]):
  439 + if token_lineno > lineno:
  440 + return self.find_backwards(self.offset + idx)
  441 + return self.find_backwards(len(self.tokens))
  442 +
  443 +
403 444 def babel_extract(fileobj, keywords, comment_tags, options):
404 445 """Babel extraction method for Jinja templates.
405 446
  447 + .. versionchanged:: 2.3
  448 + Basic support for translation comments was added. If `comment_tags`
  449 + is now set to a list of keywords for extraction, the extractor will
  450 + try to find the best preceeding comment that begins with one of the
  451 + keywords. For best results, make sure to not have more than one
  452 + gettext call in one line of code and the matching comment in the
  453 + same line or the line before.
  454 +
406 455 :param fileobj: the file-like object the messages should be extracted from
407 456 :param keywords: a list of keywords (i.e. function names) that should be
408 457 recognized as translation functions
409 458 :param comment_tags: a list of translator tags to search for and include
410   - in the results. (Unused)
  459 + in the results.
411 460 :param options: a dictionary of additional options (optional)
412 461 :return: an iterator over ``(lineno, funcname, message, comments)`` tuples.
413 462 (comments will be empty currently)
@@ -444,11 +493,14 @@ def babel_extract(fileobj, keywords, comment_tags, options):
444 493 source = fileobj.read().decode(options.get('encoding', 'utf-8'))
445 494 try:
446 495 node = environment.parse(source)
  496 + tokens = list(environment.lex(environment.preprocess(source)))
447 497 except TemplateSyntaxError, e:
448 498 # skip templates with syntax errors
449 499 return
  500 +
  501 + finder = _CommentFinder(tokens, comment_tags)
450 502 for lineno, func, message in extract_from_ast(node, keywords):
451   - yield lineno, func, message, []
  503 + yield lineno, func, message, finder.find_comments(lineno)
452 504
453 505
454 506 #: nicer import names
17 tests/test_i18n.py
@@ -95,3 +95,20 @@ def test_extract():
95 95 (3, 'gettext', u'Hello World', []),
96 96 (4, 'ngettext', (u'%(users)s user', u'%(users)s users', None), [])
97 97 ]
  98 +
  99 +
  100 +def test_comment_extract():
  101 + from jinja2.ext import babel_extract
  102 + from StringIO import StringIO
  103 + source = StringIO('''
  104 + {# trans first #}
  105 + {{ gettext('Hello World') }}
  106 + {% trans %}Hello World{% endtrans %}{# trans second #}
  107 + {#: third #}
  108 + {% trans %}{{ users }} user{% pluralize %}{{ users }} users{% endtrans %}
  109 + ''')
  110 + assert list(babel_extract(source, ('gettext', 'ngettext', '_'), ['trans', ':'], {})) == [
  111 + (3, 'gettext', u'Hello World', ['first']),
  112 + (4, 'gettext', u'Hello World', ['second']),
  113 + (6, 'ngettext', (u'%(users)s user', u'%(users)s users', None), ['third'])
  114 + ]

0 comments on commit 531578d

Please sign in to comment.
Something went wrong with that request. Please try again.