Ensure Jinja2 template header overrides are used (#75306)

Fixes #75275

changelogs/fragments/75275-ensure-jinja2-header-overrides-used.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - template - ensure Jinja2 overrides from template header are used (https://github.com/ansible/ansible/issues/75275)

lib/ansible/template/__init__.py

@@ -1079,15 +1079,21 @@ def do_template(self, data, preserve_trailing_newlines=True, escape_backslashes=
         if fail_on_undefined is None:
             fail_on_undefined = self._fail_on_undefined_errors
 
+        has_template_overrides = data.startswith(JINJA2_OVERRIDE)
+
         try:
-            # allows template header overrides to change jinja2 options.
-            if overrides is None:
-                myenv = self.environment
-            else:
+            # NOTE Creating an overlay that lives only inside do_template means that overrides are not applied
+            # when templating nested variables in AnsibleJ2Vars where Templar.environment is used, not the overlay.
+            # This is historic behavior that is kept for backwards compatibility.
+            if overrides:
                 myenv = self.environment.overlay(overrides)
+            elif has_template_overrides:
+                myenv = self.environment.overlay()
+            else:
+                myenv = self.environment
 
             # Get jinja env overrides from template
-            if hasattr(data, 'startswith') and data.startswith(JINJA2_OVERRIDE):
+            if has_template_overrides:
                 eol = data.find('\n')
                 line = data[len(JINJA2_OVERRIDE):eol]
                 data = data[eol + 1:]

test/integration/targets/template/in_template_overrides.j2

@@ -0,0 +1,5 @@
+#jinja2:variable_start_string:'<<' , variable_end_string:'>>'
+var_a: << var_a >>
+var_b: << var_b >>
+var_c: << var_c >>
+var_d: << var_d >>

test/integration/targets/template/in_template_overrides.yml

@@ -0,0 +1,28 @@
+- hosts: localhost
+  gather_facts: false
+  vars:
+    var_a: "value"
+    var_b: "{{ var_a }}"
+    var_c: "<< var_a >>"
+  tasks:
+    - set_fact:
+        var_d: "{{ var_a }}"
+
+    - block:
+        - template:
+            src: in_template_overrides.j2
+            dest: out.txt
+
+        - command: cat out.txt
+          register: out
+
+        - assert:
+            that:
+              - "'var_a: value' in out.stdout"
+              - "'var_b: value' in out.stdout"
+              - "'var_c: << var_a >>' in out.stdout"
+              - "'var_d: value' in out.stdout"
+      always:
+        - file:
+            path: out.txt
+            state: absent

test/integration/targets/template/runme.sh

@@ -38,3 +38,5 @@ ansible-playbook 72262.yml -v "$@"
 # ensure unsafe is preserved, even with extra newlines
 ansible-playbook unsafe.yml -v "$@"
 
+# ensure Jinja2 overrides from a template are used
+ansible-playbook in_template_overrides.yml -v "$@"