Fix tokens line number calculation when whitespace stripping is used

pallets · Mar 30, 2020 · 77a212b · 77a212b
1 parent 07b5c01
commit 77a212b
Show file tree

Hide file tree

Showing 3 changed files with 26 additions and 2 deletions.
diff --git a/CHANGES.rst b/CHANGES.rst
@@ -21,6 +21,8 @@ Unreleased
     when using Pytest. Due to the difficulty in supporting Python 2 and
     :pep:`451` simultaneously, the changes are reverted until 3.0.
     :pr:`1182`
+-   Fix line numbers in error messages when newlines are stripped.
+    :pr:`1178`
 
 
 Version 2.11.1

diff --git a/src/jinja2/lexer.py b/src/jinja2/lexer.py
@@ -681,6 +681,7 @@ def tokeniter(self, source, name, filename=None, state=None):
         source_length = len(source)
         balancing_stack = []
         lstrip_unless_re = self.lstrip_unless_re
+        newlines_stripped = 0
 
         while 1:
             # tokenizer loop
@@ -717,7 +718,9 @@ def tokeniter(self, source, name, filename=None, state=None):
 
                         if strip_sign == "-":
                             # Strip all whitespace between the text and the tag.
-                            groups = (text.rstrip(),) + groups[1:]
+                            stripped = text.rstrip()
+                            newlines_stripped = text[len(stripped) :].count("\n")
+                            groups = (stripped,) + groups[1:]
                         elif (
                             # Not marked for preserving whitespace.
                             strip_sign != "+"
@@ -758,7 +761,8 @@ def tokeniter(self, source, name, filename=None, state=None):
                             data = groups[idx]
                             if data or token not in ignore_if_empty:
                                 yield lineno, token, data
-                            lineno += data.count("\n")
+                            lineno += data.count("\n") + newlines_stripped
+                            newlines_stripped = 0
 
                 # strings as token just are yielded as it.
                 else:

diff --git a/tests/test_lexnparse.py b/tests/test_lexnparse.py
@@ -178,6 +178,24 @@ def test_name(self, env, name, valid2, valid3):
         else:
             pytest.raises(TemplateSyntaxError, env.from_string, t)
 
+    def test_lineno_with_strip(self, env):
+        tokens = env.lex(
+            """\
+<html>
+    <body>
+    {%- block content -%}
+        <hr>
+        {{ item }}
+    {% endblock %}
+    </body>
+</html>"""
+        )
+        for tok in tokens:
+            lineno, token_type, value = tok
+            if token_type == "name" and value == "item":
+                assert lineno == 5
+                break
+
 
 class TestParser(object):
     def test_php_syntax(self, env):