rework some of ansible's inner templating

lib/ansible/parsing/yaml/dumper.py

@@ -26,6 +26,7 @@
 from ansible.parsing.yaml.objects import AnsibleUnicode, AnsibleSequence, AnsibleMapping, AnsibleVaultEncryptedUnicode
 from ansible.utils.unsafe_proxy import AnsibleUnsafeText, AnsibleUnsafeBytes, NativeJinjaUnsafeText, NativeJinjaText
 from ansible.template import AnsibleUndefined
+from ansible.template.vars import AutoVars
 from ansible.vars.hostvars import HostVars, HostVarsVars
 from ansible.vars.manager import VarsWithSources
 
@@ -82,7 +83,7 @@ def represent_undefined(self, data):
 )
 
 AnsibleDumper.add_representer(
-    HostVarsVars,
+    AutoVars,
     represent_hostvars,
 )
 

lib/ansible/template/__init__.py

@@ -51,7 +51,8 @@
 from ansible.plugins.loader import filter_loader, lookup_loader, test_loader
 from ansible.template.native_helpers import ansible_native_concat, ansible_eval_concat, ansible_concat
 from ansible.template.template import AnsibleJ2Template
-from ansible.template.vars import AnsibleJ2Vars
+from ansible.template.vars import AnsibleJ2Vars, AutoVars, is_unsafe
+from ansible.utils.collection_loader import AnsibleCollectionRef
 from ansible.utils.display import Display
 from ansible.utils.listify import listify_lookup_plugin_terms
 from ansible.utils.native_jinja import NativeJinjaText
@@ -369,27 +370,8 @@ def __init__(self, *args, **kwargs):
         super(AnsibleContext, self).__init__(*args, **kwargs)
         self.unsafe = False
 
-    def _is_unsafe(self, val):
-        '''
-        Our helper function, which will also recursively check dict and
-        list entries due to the fact that they may be repr'd and contain
-        a key or value which contains jinja2 syntax and would otherwise
-        lose the AnsibleUnsafe value.
-        '''
-        if isinstance(val, dict):
-            for key in val.keys():
-                if self._is_unsafe(val[key]):
-                    return True
-        elif isinstance(val, list):
-            for item in val:
-                if self._is_unsafe(item):
-                    return True
-        elif getattr(val, '__UNSAFE__', False) is True:
-            return True
-        return False
-
     def _update_unsafe(self, val):
-        if val is not None and not self.unsafe and self._is_unsafe(val):
+        if val is not None and not self.unsafe and is_unsafe(val):
             self.unsafe = True
 
     def resolve_or_missing(self, key):
@@ -597,6 +579,7 @@ def __init__(self, loader, variables=None):
         self.environment.globals['query'] = self.environment.globals['q'] = self._query_lookup
         self.environment.globals['now'] = self._now_datetime
         self.environment.globals['undef'] = self._make_undefined
+        self.environment.globals['vars'] = AutoVars(self)
 
         # the current rendering context under which the templar class is working
         self.cur_context = None
@@ -970,7 +953,7 @@ def do_template(self, data, preserve_trailing_newlines=True, escape_backslashes=
             if disable_lookups:
                 t.globals['query'] = t.globals['q'] = t.globals['lookup'] = self._fail_lookup
 
-            jvars = AnsibleJ2Vars(self, t.globals)
+            jvars = AnsibleJ2Vars(self, t.globals, {'vars': AutoVars(self)})
 
             # In case this is a recursive call to do_template we need to
             # save/restore cur_context to prevent overriding __UNSAFE__.

lib/ansible/template/vars.py

@@ -7,9 +7,51 @@
 
 from ansible.errors import AnsibleError, AnsibleUndefinedVariable
 from ansible.module_utils.common.text.converters import to_native
-
-
-__all__ = ['AnsibleJ2Vars']
+from ansible.module_utils.common._collections_compat import Mapping, Sequence
+
+STATIC_VARS = [
+    'ansible_version',
+    'ansible_play_hosts',
+    'ansible_dependent_role_names',
+    'ansible_play_role_names',
+    'ansible_role_names',
+    'inventory_hostname',
+    'inventory_hostname_short',
+    'inventory_file',
+    'inventory_dir',
+    'groups',
+    'group_names',
+    'omit',
+    'playbook_dir',
+    'play_hosts',
+    'role_names',
+    'ungrouped',
+]
+
+__all__ = ['AnsibleJ2Vars', 'AutoVars', 'is_unsafe']
+
+
+def is_unsafe(val):
+    '''
+    Our helper function, which will also recursively check dict and
+    list entries due to the fact that they may be repr'd and contain
+    a key or value which contains jinja2 syntax and would otherwise
+    lose the AnsibleUnsafe value.
+    '''
+
+    if isinstance(val, Mapping):
+        for key in val.keys():
+            if is_unsafe(val[key]):
+                return True
+    elif isinstance(val, Sequence):
+        for item in val:
+            if is_unsafe(item):
+                return True
+    elif getattr(val, '__UNSAFE__', False) is True:
+        # TODO: should we change to 'unsafe' class check?
+        return True
+
+    return False
 
 
 def _process_locals(_l):
@@ -36,8 +78,10 @@ def __init__(self, templar, globals, locals=None):
     def __getitem__(self, varname):
         variable = super().__getitem__(varname)
 
+        # HostVars and AutoVars are special self templting returns.
+        # this is how 'vars' and 'hostvars' magic variables are implemented.
         from ansible.vars.hostvars import HostVars
-        if (varname == "vars" and isinstance(variable, dict)) or isinstance(variable, HostVars) or hasattr(variable, '__UNSAFE__'):
+        if (varname == "vars" and isinstance(variable, dict)) or isinstance(variable, (AutoVars, HostVars)) or hasattr(variable, '__UNSAFE__'):
             return variable
 
         try:
@@ -74,3 +118,50 @@ def add_locals(self, locals):
         new_locals = current_locals | locals
 
         return AnsibleJ2Vars(self._templar, current_globals, locals=new_locals)
+
+
+class AutoVars(Mapping):
+    ''' A special view of template vars on demand. '''
+
+    def __init__(self, templar, myvars=None):
+
+        self._t = templar
+
+        # this allows for vars that are part of this object to be
+        # resolved even if they depend on vars not contained within.
+        if myvars is None:
+            self._vars = self._t._available_variables
+        else:
+            self._vars = myvars
+
+    def __getitem__(self, var):
+        from ansible.vars.hostvars import HostVars
+        if is_unsafe(self._vars[var]) or isinstance(self._vars[var], (HostVars, AnsibleJ2Vars, AutoVars)):
+            res = self._vars[var]
+        else:
+            res = self._t.template(self._vars[var], fail_on_undefined=False, static_vars=STATIC_VARS)
+        return res
+
+    def __contains__(self, var):
+        return (var in self._vars)
+
+    def __iter__(self):
+        for var in self._vars.keys():
+            yield self.__getitem__(var)
+
+    def __len__(self):
+        return len(self._vars.keys())
+
+    def __repr__(self):
+        return repr(self.__iter__())
+
+    def __readonly__(self, *args, **kwargs):
+        raise RuntimeError("Cannot modify this variable, it is read only.")
+
+    __setitem__ = __readonly__
+    __delitem__ = __readonly__
+    pop = __readonly__
+    popitem = __readonly__
+    clear = __readonly__
+    update = __readonly__
+    setdefault = __readonly__

lib/ansible/vars/hostvars.py

@@ -21,28 +21,11 @@
 
 from collections.abc import Mapping
 
+from ansible.utils.vars import combine_vars
 from ansible.template import Templar, AnsibleUndefined
+from ansible.template.vars import AutoVars
 
-STATIC_VARS = [
-    'ansible_version',
-    'ansible_play_hosts',
-    'ansible_dependent_role_names',
-    'ansible_play_role_names',
-    'ansible_role_names',
-    'inventory_hostname',
-    'inventory_hostname_short',
-    'inventory_file',
-    'inventory_dir',
-    'groups',
-    'group_names',
-    'omit',
-    'playbook_dir',
-    'play_hosts',
-    'role_names',
-    'ungrouped',
-]
-
-__all__ = ['HostVars', 'HostVarsVars']
+__all__ = ['HostVars']
 
 
 # Note -- this is a Mapping, not a MutableMapping
@@ -54,6 +37,8 @@ def __init__(self, inventory, variable_manager, loader):
         self._loader = loader
         self._variable_manager = variable_manager
         variable_manager._hostvars = self
+        self._templating_vars = {}
+        self._templar = None
 
     def set_variable_manager(self, variable_manager):
         self._variable_manager = variable_manager
@@ -62,6 +47,9 @@ def set_variable_manager(self, variable_manager):
     def set_inventory(self, inventory):
         self._inventory = inventory
 
+    def set_available_vars(self, myvars):
+        self._templating_vars = myvars
+
     def _find_host(self, host_name):
         # does not use inventory.hosts so it can create localhost on demand
         return self._inventory.get_host(host_name)
@@ -91,10 +79,29 @@ def __setstate__(self, state):
             self._variable_manager._hostvars = self
 
     def __getitem__(self, host_name):
-        data = self.raw_get(host_name)
-        if isinstance(data, AnsibleUndefined):
-            return data
-        return HostVarsVars(data, loader=self._loader)
+        '''
+        wrap all the magic and either return undeifned or
+        a self templating on demand object pretending to be a dict
+        '''
+        data = {}
+
+        # don't keep cached cause vars sources can change over life of this object.
+        raw_data = self.raw_get(host_name)
+
+        # if its not undefined, wrap in autotemplating object
+        if isinstance(raw_data, AnsibleUndefined):
+            data = raw_data
+        else:
+            tvars = combine_vars(self._templating_vars, raw_data)
+            if self._templar is None:
+                # initialize templar if we had not before
+                self._templar = Templar(variables=tvars, loader=self._loader)
+            else:
+                self._templar.available_variables = tvars
+
+            data = AutoVars(self._templar, raw_data)
+
+        return data
 
     def set_host_variable(self, host, varname, value):
         self._variable_manager.set_host_variable(host, varname, value)

lib/ansible/vars/manager.py

@@ -438,6 +438,9 @@ def plugins_by_groups():
         if task and host and task.delegate_to is not None and include_delegate_to:
             all_vars['ansible_delegated_vars'], all_vars['_ansible_loop_cache'] = self._get_delegated_vars(play, task, all_vars)
 
+        if 'hostvars' in all_vars:
+            all_vars['hostvars'].set_available_vars(all_vars)
+
         display.debug("done with get_vars()")
         if C.DEFAULT_DEBUG:
             # Use VarsWithSources wrapper class to display var sources

test/integration/targets/loops/tasks/main.yml

@@ -388,8 +388,8 @@
 
 - assert:
     that:
-      - foo[0] != 'foo1.0'
-      - foo[0] == unsafe_value
+      - foo[0] != unsafe_value
+      - foo[0] == 'foo1.0'
   vars:
     unsafe_value: !unsafe 'foo{{ version_64169 }}'