Added support for line-based comments.

--HG--
branch : trunk

CHANGES

@@ -19,6 +19,7 @@ Version 2.2
   variable and regular variable *after* the loop if that variable was unused
   *before* the loop.  (#331)
 - Added support for optional `scoped` modifier to blocks.
+- Added support for line-comments.
 
 Version 2.1.1
 -------------

docs/templates.rst

@@ -252,6 +252,14 @@ precedes it.  For better readability statements that start a block (such as
         # endfor
         </ul>
 
+Since Jinja 2.2 line-based comments are available as well.  For example if
+the line-comment prefix is configured to be ``##`` everything from ``##`` to
+the end of the line is ignored (excluding the newline sign)::
+
+    # for item in seq:
+        <li>{{ item }}</li>     ## this comment is ignored
+    # endfor
+
 
 .. _template-inheritance:
 

jinja2/defaults.py

@@ -19,6 +19,7 @@
 COMMENT_START_STRING = '{#'
 COMMENT_END_STRING = '#}'
 LINE_STATEMENT_PREFIX = None
+LINE_COMMENT_PREFIX = None
 TRIM_BLOCKS = False
 NEWLINE_SEQUENCE = '\n'
 

jinja2/environment.py

@@ -118,6 +118,12 @@ class Environment(object):
             If given and a string, this will be used as prefix for line based
             statements.  See also :ref:`line-statements`.
 
+        `line_comment_prefix`
+            If given and a string, this will be used as prefix for line based
+            based comments.  See also :ref:`line-statements`.
+
+            .. versionadded:: 2.2
+
         `trim_blocks`
             If this is set to ``True`` the first newline after a block is
             removed (block, not variable tag!).  Defaults to `False`.
@@ -198,6 +204,7 @@ def __init__(self,
                  comment_start_string=COMMENT_START_STRING,
                  comment_end_string=COMMENT_END_STRING,
                  line_statement_prefix=LINE_STATEMENT_PREFIX,
+                 line_comment_prefix=LINE_COMMENT_PREFIX,
                  trim_blocks=TRIM_BLOCKS,
                  newline_sequence=NEWLINE_SEQUENCE,
                  extensions=(),
@@ -228,6 +235,7 @@ def __init__(self,
         self.comment_start_string = comment_start_string
         self.comment_end_string = comment_end_string
         self.line_statement_prefix = line_statement_prefix
+        self.line_comment_prefix = line_comment_prefix
         self.trim_blocks = trim_blocks
         self.newline_sequence = newline_sequence
 
@@ -266,10 +274,10 @@ def extend(self, **attributes):
     def overlay(self, block_start_string=missing, block_end_string=missing,
                 variable_start_string=missing, variable_end_string=missing,
                 comment_start_string=missing, comment_end_string=missing,
-                line_statement_prefix=missing, trim_blocks=missing,
-                extensions=missing, optimized=missing, undefined=missing,
-                finalize=missing, autoescape=missing, loader=missing,
-                cache_size=missing, auto_reload=missing,
+                line_statement_prefix=missing, line_comment_prefix=missing,
+                trim_blocks=missing, extensions=missing, optimized=missing,
+                undefined=missing, finalize=missing, autoescape=missing,
+                loader=missing, cache_size=missing, auto_reload=missing,
                 bytecode_cache=missing):
         """Create a new overlay environment that shares all the data with the
         current environment except of cache and the overriden attributes.
@@ -560,6 +568,7 @@ def __new__(cls, source,
                 comment_start_string=COMMENT_START_STRING,
                 comment_end_string=COMMENT_END_STRING,
                 line_statement_prefix=LINE_STATEMENT_PREFIX,
+                line_comment_prefix=LINE_COMMENT_PREFIX,
                 trim_blocks=TRIM_BLOCKS,
                 newline_sequence=NEWLINE_SEQUENCE,
                 extensions=(),
@@ -570,9 +579,9 @@ def __new__(cls, source,
         env = get_spontaneous_environment(
             block_start_string, block_end_string, variable_start_string,
             variable_end_string, comment_start_string, comment_end_string,
-            line_statement_prefix, trim_blocks, newline_sequence,
-            frozenset(extensions), optimized, undefined, finalize,
-            autoescape, None, 0, False, None)
+            line_statement_prefix, line_comment_prefix, trim_blocks,
+            newline_sequence, frozenset(extensions), optimized, undefined,
+            finalize, autoescape, None, 0, False, None)
         return env.from_string(source, template_class=cls)
 
     @classmethod

jinja2/ext.py

@@ -429,6 +429,7 @@ def babel_extract(fileobj, keywords, comment_tags, options):
         options.get('comment_start_string', COMMENT_START_STRING),
         options.get('comment_end_string', COMMENT_END_STRING),
         options.get('line_statement_prefix') or LINE_STATEMENT_PREFIX,
+        options.get('line_comment_prefix') or LINE_COMMENT_PREFIX,
         str(options.get('trim_blocks', TRIM_BLOCKS)).lower() in \
             ('1', 'on', 'yes', 'true'),
         NEWLINE_SEQUENCE, frozenset(extensions),

jinja2/lexer.py

@@ -78,6 +78,9 @@
 TOKEN_COMMENT = intern('comment')
 TOKEN_LINESTATEMENT_BEGIN = intern('linestatement_begin')
 TOKEN_LINESTATEMENT_END = intern('linestatement_end')
+TOKEN_LINECOMMENT_BEGIN = intern('linecomment_begin')
+TOKEN_LINECOMMENT_END = intern('linecomment_end')
+TOKEN_LINECOMMENT = intern('linecomment')
 TOKEN_DATA = intern('data')
 TOKEN_INITIAL = intern('initial')
 TOKEN_EOF = intern('eof')
@@ -117,6 +120,11 @@
 operator_re = re.compile('(%s)' % '|'.join(re.escape(x) for x in
                          sorted(operators, key=lambda x: -len(x))))
 
+ignored_tokens = frozenset([TOKEN_COMMENT_BEGIN, TOKEN_COMMENT,
+                            TOKEN_COMMENT_END, TOKEN_WHITESPACE,
+                            TOKEN_WHITESPACE, TOKEN_LINECOMMENT_BEGIN,
+                            TOKEN_LINECOMMENT_END, TOKEN_LINECOMMENT])
+
 
 def count_newlines(value):
     """Count the number of newline characters in the string.  This is
@@ -125,6 +133,28 @@ def count_newlines(value):
     return len(newline_re.findall(value))
 
 
+def compile_rules(environment):
+    """Compiles all the rules from the environment into a list of rules."""
+    e = re.escape
+    rules = [
+        (len(environment.comment_start_string), 'comment',
+         e(environment.comment_start_string)),
+        (len(environment.block_start_string), 'block',
+         e(environment.block_start_string)),
+        (len(environment.variable_start_string), 'variable',
+         e(environment.variable_start_string)),
+    ]
+
+    if environment.line_statement_prefix is not None:
+        rules.append((len(environment.line_statement_prefix), 'linestatement',
+                      '^\\s*' + e(environment.line_statement_prefix)))
+    if environment.line_comment_prefix is not None:
+        rules.append((len(environment.line_comment_prefix), 'linecomment',
+                      '\\s*' + e(environment.line_comment_prefix)))
+
+    return [x[1:] for x in sorted(rules, reverse=True)]
+
+
 class Failure(object):
     """Class that raises a `TemplateSyntaxError` if called.
     Used by the `Lexer` to specify known errors.
@@ -302,6 +332,7 @@ def get_lexer(environment):
            environment.comment_start_string,
            environment.comment_end_string,
            environment.line_statement_prefix,
+           environment.line_comment_prefix,
            environment.trim_blocks,
            environment.newline_sequence)
     lexer = _lexer_cache.get(key)
@@ -340,22 +371,7 @@ def __init__(self, environment):
         # <%= for variables. (if someone wants asp like syntax)
         # variables are just part of the rules if variable processing
         # is required.
-        root_tag_rules = [
-            ('comment',     environment.comment_start_string),
-            ('block',       environment.block_start_string),
-            ('variable',    environment.variable_start_string)
-        ]
-        root_tag_rules.sort(key=lambda x: -len(x[1]))
-
-        # now escape the rules.  This is done here so that the escape
-        # signs don't count for the lengths of the tags.
-        root_tag_rules = [(a, e(b)) for a, b in root_tag_rules]
-
-        # if we have a line statement prefix we need an extra rule for
-        # that.  We add this rule *after* all the others.
-        if environment.line_statement_prefix is not None:
-            prefix = e(environment.line_statement_prefix)
-            root_tag_rules.insert(0, ('linestatement', '^\s*' + prefix))
+        root_tag_rules = compile_rules(environment)
 
         # block suffix if trimming is enabled
         block_suffix_re = environment.trim_blocks and '\\n?' or ''
@@ -416,7 +432,11 @@ def __init__(self, environment):
             # line statements
             TOKEN_LINESTATEMENT_BEGIN: [
                 (c(r'\s*(\n|$)'), TOKEN_LINESTATEMENT_END, '#pop')
-            ] + tag_rules
+            ] + tag_rules,
+            # line comments
+            TOKEN_LINECOMMENT_BEGIN: [
+                (c(r'.*?(?=\n|$)'), TOKEN_LINECOMMENT_END, '#pop')
+            ]
         }
 
     def _normalize_newlines(self, value):
@@ -434,8 +454,7 @@ def wrap(self, stream, name=None, filename=None):
         every token in a :class:`Token` and converts the value.
         """
         for lineno, token, value in stream:
-            if token in ('comment_begin', 'comment', 'comment_end',
-                         'whitespace'):
+            if token in ignored_tokens:
                 continue
             elif token == 'linestatement_begin':
                 token = 'block_begin'

tests/test_old_bugs.py

@@ -60,3 +60,9 @@ def test_weird_inline_comment():
     env = Environment(line_statement_prefix='%')
     raises(TemplateSyntaxError, env.from_string,
            '% for item in seq {# missing #}\n...% endfor')
+
+
+def test_old_macro_loop_scoping_bug(env):
+    tmpl = env.from_string('{% for i in (1, 2) %}{{ i }}{% endfor %}'
+                           '{% macro i() %}3{% endmacro %}{{ i() }}')
+    assert tmpl.render() == '123'

tests/test_parser.py

@@ -28,17 +28,32 @@
 <!--- endfor -->'''
 
 MAKO_SYNTAX = '''\
+<%# regular comment %>
 % for item in seq:
     ${item}
 % endfor'''
 
+MAKO_SYNTAX_LINECOMMENTS = '''\
+<%# regular comment %>
+% for item in seq:
+    ${item} ## the rest of the stuff
+% endfor'''
+
 BALANCING = '''{{{'foo':'bar'}.foo}}'''
 
 STARTCOMMENT = '''{# foo comment
 and bar comment #}
 {% macro blub() %}foo{% endmacro %}
 {{ blub() }}'''
 
+LINE_SYNTAX_PRIORITY = '''\
+/* ignore me.
+   I'm a multiline comment */
+# for item in seq:
+* ${item}          ## this is just extra stuff
+# endfor
+'''
+
 
 def test_php_syntax():
     env = Environment('<?', '?>', '<?=', '?>', '<!--', '-->')
@@ -72,4 +87,15 @@ def test_line_syntax():
     env = Environment('<%', '%>', '${', '}', '<%#', '%>', '%')
     tmpl = env.from_string(MAKO_SYNTAX)
     assert [int(x.strip()) for x in tmpl.render(seq=range(5)).split()] == \
-           range(5)
+            range(5)
+
+    env = Environment('<%', '%>', '${', '}', '<%#', '%>', '%', '##')
+    tmpl = env.from_string(MAKO_SYNTAX_LINECOMMENTS)
+    assert [int(x.strip()) for x in tmpl.render(seq=range(5)).split()] == \
+            range(5)
+
+
+def test_line_syntax_priority():
+    env = Environment('{%', '%}', '${', '}', '/*', '*/', '#', '##')
+    tmpl = env.from_string(LINE_SYNTAX_PRIORITY)
+    assert tmpl.render(seq=[1, 2]).strip() == '* 1\n* 2'