parser: don't skip lines containing jinja expressions

They can still be parsed as an option=value line, which makes option
available for the section.

server/CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog][keepachangelog],
 and this project adheres to [Semantic Versioning][semver].
 
+## [Unreleased]
+
+### Fixed:
+
+- don't skip lines containing jinja expressions. This was causing some missing options when jinja was used in option
+
 ## [0.3.0] - 2020-02-23
 
 ### Added:
@@ -54,3 +60,4 @@ and this project adheres to [Semantic Versioning][semver].
 [0.2.0]: https://github.com/perrinjerome/vscode-zc-buildout/compare/v0.1.1...v0.2.0
 [0.2.1]: https://github.com/perrinjerome/vscode-zc-buildout/compare/v0.2.0...v0.2.1
 [0.3.0]: https://github.com/perrinjerome/vscode-zc-buildout/compare/v0.2.1...v0.3.0
+[unreleased]: https://github.com/perrinjerome/vscode-zc-buildout/compare/v0.3.0...master

server/buildoutls/buildout.py

@@ -946,6 +946,7 @@ async def _parse(
     jinja_parser.feed(line)
     if jinja_parser.is_in_jinja:
       continue
+    line = jinja_parser.line
 
     if line[0] in '#;':
       continue  # comment

server/buildoutls/jinja.py

@@ -28,25 +28,33 @@ class JinjaStatement(str, enum.Enum):
 }
 
 statement_re = re.compile(r'.*\{%[\-\+\s]*(?P<statement>[\w]+).*%\}')
-expression_re = re.compile(r'.*\{\{.*\}\}')
+expression_re = re.compile(r'\{\{.*?\}\}')
 
 
 class JinjaParser:
   """A very simple jinja parser which allow skipping lines containing jinja blocks.
   """
+  # a replacement for jinja expressions, so that we
+  # can still parse them as buildout
+  jinja_value = "JINJA_EXPRESSION"
+
   def __init__(self) -> None:
-    self.is_in_expression = False
+    self.has_expression = False
     self.is_in_comment = False
     self.is_error = False
     self._stack: List[JinjaStatement] = []
     self._current_line_was_in_jinja = False
     self._in_comment = False
+    self.line = ""
 
   def feed(self, line: str) -> None:
     """Feeds a line and update the state.
     """
     self._current_line_was_in_jinja = False
-    self.is_in_expression = bool(expression_re.match(line))
+    self.has_expression = bool(expression_re.search(line))
+    if expression_re.search(line):
+      line = expression_re.sub(self.jinja_value, line)
+    self.line = line
     self.is_error = False
 
     if '{#' in line or self._in_comment:
@@ -67,5 +75,4 @@ def feed(self, line: str) -> None:
 
   @property
   def is_in_jinja(self) -> bool:
-    return (bool(self._stack) or self.is_in_expression
-            or self._current_line_was_in_jinja)
+    return (bool(self._stack) or self._current_line_was_in_jinja)

server/buildoutls/tests/test_buildout_parser.py

@@ -100,6 +100,42 @@ async def test_parse() -> None:
   ]
 
 
+@pytest.mark.asyncio
+async def test_parse_jinja_option() -> None:
+  parsed = await _parse(
+      fp=io.StringIO(
+          textwrap.dedent("""\
+                [section]
+                option = {{ jinja_expression }}
+                {# ignored #}
+                {% jinja_key = value %}
+                {{ jinja_expression }} = value
+
+                [section{{ jinja_expression }}]
+                option = value
+
+                [another_section]
+                {{ jinja_expression }} = {{ jinja_expression }}
+                """)),
+      uri="file:///buildout.cfg",
+      allow_errors=False,
+  )
+  assert list(parsed.keys()) == [
+      'buildout', 'section', 'sectionJINJA_EXPRESSION', 'another_section'
+  ]
+  assert sorted(parsed['section'].keys()) == [
+      'JINJA_EXPRESSION',
+      '_buildout_section_name_',
+      '_profile_base_location_',
+      'option',
+  ]
+  assert parsed['section']['option'].value == 'JINJA_EXPRESSION'
+  assert parsed['section']['JINJA_EXPRESSION'].value == 'value'
+  assert parsed['sectionJINJA_EXPRESSION']['option'].value == 'value'
+  assert parsed['another_section'][
+      'JINJA_EXPRESSION'].value == 'JINJA_EXPRESSION'
+
+
 @pytest.mark.asyncio
 async def test_BuildoutProfile_getSymbolAtPosition_BuildoutOptionKey(
     buildout: BuildoutProfile) -> None:

server/buildoutls/tests/test_jinja.py

@@ -12,7 +12,9 @@ def test_is_in_jinja() -> None:
       ('not ${jinja', False),
 
       # expressions
-      ('{{ jinja }}', True),
+      ('{{ jinja }}', False),
+      ('[section{{ jinja }}]', False),
+      ('key = {{ jinja }}', False),
 
       # one line statements
       ('{% set something = True %}', True),