Reimplement jinja[spacing] to avoid use of regex

This change rewrites jinja2 rule to make it use jinaja lex parser and
avoid use of regexes for parsing, as they cause too many false
positives.

Fixes: #2301 #2285 #2241 #2209 #2208 #2120

.github/workflows/tox.yml

@@ -165,7 +165,7 @@ jobs:
       WSLENV: FORCE_COLOR:PYTEST_REQPASS:TOXENV:TOX_PARALLEL_NO_SPINNER
       # Number of expected test passes, safety measure for accidental skip of
       # tests. Update value if you add/remove tests.
-      PYTEST_REQPASS: 661
+      PYTEST_REQPASS: 674
 
     steps:
       - name: Activate WSL1

examples/playbooks/jinja-spacing.yml

@@ -1,6 +1,7 @@
 ---
 # Should raise jinja[spacing] at tasks line 23, 26, 29, 54, 65
-- hosts: all
+- name: Fixture for testing jinja2[spacing]
+  hosts: all
   tasks:
     - name: Good variable format
       ansible.builtin.debug:
@@ -56,10 +57,10 @@
       vars:
         cases:
           case1: >-
-            http://example.com/{{
+            http://foo.com/{{
               case1 }}
           case2: >-
-            http://example.com/{{
+            http://bar.com/{{
             case2 }}
       ansible.builtin.debug:
         var: cases

examples/playbooks/vars/jinja-spacing.yml

@@ -1,5 +1,5 @@
 ---
-# Should raise jinja[spacing] at line [12, 13, 14, 27, 33], at following variables.
+# Should raise jinja[spacing] at line [14, 15, 16, 17, 18, 19, 22, 32, 38], at following variables.
 # ".bad_var_1", ".bad_var_2", ".bad_var_3", ".invalid_multiline_nested_json", ".invalid_nested_json"
 good_var_1: "{{ good_format }}"
 good_var_2: "Value: {{ good_format }}"
@@ -11,12 +11,12 @@ jinja_whitespace_control: |
   {{ good_format }}/
   {{- good_format }}
   {{- good_format -}}
-bad_var_1: "{{bad_format}}"
-bad_var_2: "Value: {{ bad_format}}"
-bad_var_3: "{{bad_format }}"
-bad_var_4: "{{ bad_format|filter }}"
-bad_var_5: "Value: {{ bad_format |filter }}"
-bad_var_6: "{{ bad_format| filter }}"
+bad_var_1: "{{bad_format}}" # <-- 1
+bad_var_2: "Value: {{ bad_format}}" # <-- 2
+bad_var_3: "{{bad_format }}" # <-- 3
+bad_var_4: "{{ bad_format|filter }}" # <-- 4
+bad_var_5: "Value: {{ bad_format |filter }}" # <-- 5
+bad_var_6: "{{ bad_format| filter }}" # <-- 6
 # spell-checker: disable-next-line
 non_jinja_var: "data = ${lookup{$local_part}lsearch{/etc/aliases}}" # noqa: jinja[spacing]
 json_inside_jinja: "{{ {'test': {'subtest': variable}} }}"
@@ -29,13 +29,13 @@ multiline_vars: # Assert that no false positive on multiline exists
       http://example.com/{{
       case2 }}
 valid_nested_json: "{{ {'dummy_2': {'nested_dummy_1': 'value_1', 'nested_dummy_2': value_2}} | combine(dummy_1) }}"
-invalid_nested_json: "{{ {'dummy_2': {'nested_dummy_1': 'value_1', 'nested_dummy_2': value_2}} | combine(dummy_1)}}"
+invalid_nested_json: "{{ {'dummy_2': {'nested_dummy_1': 'value_1', 'nested_dummy_2': value_2}} | combine(dummy_1)}}" # <-- 7
 
 valid_multiline_nested_json: >-
   {{ {'dummy_2': {'nested_dummy_1': value_1,
                   'nested_dummy_2': value_2}} |
   combine(dummy_1) }}
-invalid_multiline_nested_json: >-
+invalid_multiline_nested_json: >- # <-- 8
   {{ {'dummy_2': {'nested_dummy_1': value_1,
                   'nested_dummy_2': value_2}} |
   combine(dummy_1)}}

src/ansiblelint/rules/command_instead_of_shell.py

@@ -40,11 +40,11 @@
     changed_when: false
 
   - name: Shell with jinja filter
-    ansible.builtin.shell: echo {{ "hello"|upper }}
+    ansible.builtin.shell: echo {{ "hello" | upper }}
     changed_when: false
 
   - name: Sshell with jinja filter (fqcn)
-    ansible.builtin.shell: echo {{ "hello"|upper }}
+    ansible.builtin.shell: echo {{ "hello" | upper }}
     changed_when: false
 """
 
@@ -143,7 +143,10 @@ def matchtask(
 
     from ansiblelint.testing import RunFromText  # pylint: disable=ungrouped-imports
 
-    @pytest.mark.parametrize(("text", "expected"), ((SUCCESS_PLAY, 0), (FAIL_PLAY, 3)))
+    @pytest.mark.parametrize(
+        ("text", "expected"),
+        (pytest.param(SUCCESS_PLAY, 0, id="0"), pytest.param(FAIL_PLAY, 3, id="1")),
+    )
     def test_rule_command_instead_of_shell(
         default_text_runner: RunFromText, text: str, expected: int
     ) -> None:

src/ansiblelint/rules/jinja.md

@@ -1,20 +1,25 @@
 ## jinja
 
 This rule can report problems related to jinja2 string templates. The current
-version can report `jinja[spacing]` when there are no spaces between variables
-and operators, including filters, like `{{ var_name | filter }}`. This
-improves readability and makes it less likely to introduce typos.
+version can report:
+
+- `jinja[spacing]` when there are no spaces between variables
+  and operators, including filters, like `{{ var_name | filter }}`. This
+  improves readability and makes it less likely to introduce typos.
+- `jinja[invalid]` when the jinja2 template is invalid
 
 ### Problematic code
 
 ```yaml
 ---
-foo: "{{some|dict2items}}"
+foo: "{{some|dict2items}}" # <-- jinja[spacing]
+bar: "{{ & }}" # <-- jinja[invalid]
 ```
 
 ### Correct code
 
 ```yaml
 ---
 foo: "{{ some | dict2items }}"
+bar: "{{ '&' }}"
 ```