Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support vars plugins in collections #61078

Open
wants to merge 17 commits into
base: devel
from
@@ -18,6 +18,7 @@
from ansible.plugins.loader import vars_loader
from ansible.utils.vars import combine_vars
from ansible.utils.display import Display
from ansible.vars.plugins import get_vars_from_inventory_sources

display = Display()

@@ -184,41 +185,12 @@ def dump(stuff):

return results

# FIXME: refactor to use same for VM
def get_plugin_vars(self, path, entity):

data = {}

def _get_plugin_vars(plugin, path, entities):
data = {}
try:
data = plugin.get_vars(self.loader, path, entity)
except AttributeError:
try:
if isinstance(entity, Host):
data = combine_vars(data, plugin.get_host_vars(entity.name))
else:
data = combine_vars(data, plugin.get_group_vars(entity.name))
except AttributeError:
if hasattr(plugin, 'run'):
raise AnsibleError("Cannot use v1 type vars plugin %s from %s" % (plugin._load_name, plugin._original_path))
else:
raise AnsibleError("Invalid vars plugin %s from %s" % (plugin._load_name, plugin._original_path))
return data

for plugin in vars_loader.all():
data = combine_vars(data, _get_plugin_vars(plugin, path, entity))

return data

def _get_group_variables(self, group):

# get info from inventory source
res = group.get_vars()

# FIXME: add switch to skip vars plugins, add vars plugin info
for inventory_dir in self.inventory._sources:
res = combine_vars(res, self.get_plugin_vars(inventory_dir, group))
res = combine_vars(res, get_vars_from_inventory_sources(self.loader, self.inventory._sources, [group], 'inventory'))

if group.priority != 1:
res['ansible_group_priority'] = group.priority
@@ -231,12 +203,10 @@ def _get_host_variables(self, host):
# only get vars defined directly host
hostvars = host.get_vars()

# FIXME: add switch to skip vars plugins, add vars plugin info
for inventory_dir in self.inventory._sources:
hostvars = combine_vars(hostvars, self.get_plugin_vars(inventory_dir, host))
hostvars = combine_vars(hostvars, get_vars_from_inventory_sources(self.loader, self.inventory._sources, [host], 'inventory'))
else:
# get all vars flattened by host, but skip magic hostvars
hostvars = self.vm.get_vars(host=host, include_hostvars=False)
hostvars = self.vm.get_vars(host=host, include_hostvars=False, stage='inventory')

return self._remove_internal(hostvars)

@@ -1691,6 +1691,18 @@ RETRY_FILES_SAVE_PATH:
ini:
- {key: retry_files_save_path, section: defaults}
type: path
RUN_VARS_PLUGINS:
name: When should vars plugins run relative to inventory
default: demand
description:
- This setting can be used to optimize vars_plugin usage depending on user's inventory size and play selection.
- Setting to 'demand' will run vars_plugins relative to inventory sources anytime vars are 'demanded' by tasks.
This conversation was marked as resolved by s-hertel

This comment has been minimized.

Copy link
@samccann

samccann Sep 30, 2019

Contributor
Suggested change
- Setting to 'demand' will run vars_plugins relative to inventory sources anytime vars are 'demanded' by tasks.
- Setting to C(demand) will run vars_plugins relative to inventory sources anytime vars are 'demanded' by tasks.

This comment has been minimized.

Copy link
@s-hertel

s-hertel Sep 30, 2019

Author Contributor

Going to let my ci_complete tests finish and then I'll update these. Thank you for re-reviewing!

- Setting to 'start' will run vars_plugins relative to inventory sources after importing that inventory source.
This conversation was marked as resolved by s-hertel

This comment has been minimized.

Copy link
@samccann

samccann Sep 30, 2019

Contributor
Suggested change
- Setting to 'start' will run vars_plugins relative to inventory sources after importing that inventory source.
- Setting to C(start) will run vars_plugins relative to inventory sources after importing that inventory source.
env: [{name: ANSIBLE_RUN_VARS_PLUGINS}]
ini:
- {key: run_vars_plugins, section: defaults}
type: str
choices: ['demand', 'start']
SHOW_CUSTOM_STATS:
name: Display custom stats
default: False
@@ -1745,6 +1757,15 @@ USE_PERSISTENT_CONNECTIONS:
ini:
- {key: use_persistent_connections, section: defaults}
type: boolean
VARIABLE_PLUGINS_ENABLED:
name: Vars plugin whitelist
default: ['host_group_vars']
description: Whitelist for variable plugins that require it.
env: [{name: ANSIBLE_VARS_ENABLED}]
ini:
- {key: vars_plugins_enabled, section: defaults}
type: list
version_added: "2.9"
This conversation was marked as resolved by s-hertel

This comment has been minimized.

Copy link
@samccann

samccann Sep 26, 2019

Contributor

was this really added in 2.9 or did you mean 2.10 here?

This comment has been minimized.

Copy link
@s-hertel

s-hertel Sep 26, 2019

Author Contributor

It was 2.9 at the time of opening the PR. I need to update these.

VARIABLE_PRECEDENCE:
name: Group variable precedence
default: ['all_inventory', 'groups_inventory', 'all_plugins_inventory', 'all_plugins_play', 'groups_plugins_inventory', 'groups_plugins_play']
@@ -102,9 +102,9 @@ def __getitem__(self, y):
DEFAULT_REMOTE_PASS = None
DEFAULT_SUBSET = None
# FIXME: expand to other plugins, but never doc fragments
CONFIGURABLE_PLUGINS = ('become', 'cache', 'callback', 'cliconf', 'connection', 'httpapi', 'inventory', 'lookup', 'shell')
CONFIGURABLE_PLUGINS = ('become', 'cache', 'callback', 'cliconf', 'connection', 'httpapi', 'inventory', 'lookup', 'shell', 'vars')
# NOTE: always update the docs/docsite/Makefile to match
DOCUMENTABLE_PLUGINS = CONFIGURABLE_PLUGINS + ('module', 'strategy', 'vars')
DOCUMENTABLE_PLUGINS = CONFIGURABLE_PLUGINS + ('module', 'strategy')
IGNORE_FILES = ("COPYING", "CONTRIBUTING", "LICENSE", "README", "VERSION", "GUIDELINES") # ignore during module search
INTERNAL_RESULT_KEYS = ('add_host', 'add_group')
LOCALHOST = ('127.0.0.1', 'localhost', '::1')
@@ -39,6 +39,8 @@
from ansible.utils.helpers import deduplicate_list
from ansible.utils.path import unfrackpath
from ansible.utils.display import Display
from ansible.utils.vars import combine_vars
from ansible.vars.plugins import get_vars_from_inventory_sources

display = Display()

@@ -230,6 +232,12 @@ def parse_sources(self, cache=False):
else:
display.warning("No inventory was parsed, only implicit localhost is available")

if C.RUN_VARS_PLUGINS == 'start':
for group in self.groups.values():
group.vars = combine_vars(group.vars, get_vars_from_inventory_sources(self._loader, self._sources, [group], 'inventory'))
for host in self.hosts.values():
host.vars = combine_vars(host.vars, get_vars_from_inventory_sources(self._loader, self._sources, [host], 'inventory'))

def parse_source(self, source, cache=False):
''' Generate or update inventory for the source provided '''

@@ -18,22 +18,29 @@
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins import AnsiblePlugin
from ansible.utils.path import basedir
from ansible.utils.display import Display

display = Display()


class BaseVarsPlugin(object):
class BaseVarsPlugin(AnsiblePlugin):

"""
Loads variables for groups and/or hosts
"""

def __init__(self):
""" constructor """
super(BaseVarsPlugin, self).__init__()
self._display = display

def has_option(self, option):
This conversation was marked as resolved by s-hertel

This comment has been minimized.

Copy link
@bcoca

bcoca Aug 22, 2019

Member

move this to AnsiblePlugin?

if not self._options:
self.set_options()
return option in self._options

def get_vars(self, loader, path, entities):
""" Gets variables. """
self._basedir = basedir(path)
@@ -28,6 +28,17 @@
- Hidden (starting with '.') and backup (ending with '~') files and directories are ignored.
- Only applies to inventory sources that are existing paths.
options:
stage:
This conversation was marked as resolved by s-hertel

This comment has been minimized.

Copy link
@bcoca

bcoca Aug 22, 2019

Member

not a blocker, but is it worth making a doc fragment?

description: Control when this vars plugin may be executed.
default: all
choices: ['all', 'inventory', 'task']
type: str
ini:
- key: stage
section: host_group_vars
env:
- name: ANSIBLE_VARS_PLUGIN_STAGE
version_added: "2.9"
_valid_extensions:
default: [".yml", ".yaml", ".json"]
description:
@@ -55,6 +66,8 @@

class VarsModule(BaseVarsPlugin):

REQUIRES_WHITELIST = True

def get_vars(self, loader, path, entities, cache=True):
''' parses the inventory file '''

@@ -35,17 +35,18 @@
from ansible.errors import AnsibleError, AnsibleParserError, AnsibleUndefinedVariable, AnsibleFileNotFound, AnsibleAssertionError, AnsibleTemplateError
from ansible.inventory.host import Host
from ansible.inventory.helpers import sort_groups, get_group_vars
from ansible.module_utils._text import to_bytes, to_text
from ansible.module_utils._text import to_text
from ansible.module_utils.common._collections_compat import Mapping, MutableMapping, Sequence
from ansible.module_utils.six import iteritems, text_type, string_types
from ansible.plugins.loader import lookup_loader, vars_loader
from ansible.plugins.loader import lookup_loader
from ansible.vars.fact_cache import FactCache
from ansible.template import Templar
from ansible.utils.display import Display
from ansible.utils.listify import listify_lookup_plugin_terms
from ansible.utils.vars import combine_vars, load_extra_vars, load_options_vars
from ansible.utils.unsafe_proxy import wrap_var
from ansible.vars.clean import namespace_facts, clean_facts
from ansible.vars.plugins import get_vars_from_inventory_sources, get_vars_from_path

display = Display()

@@ -141,7 +142,7 @@ def set_inventory(self, inventory):
self._inventory = inventory

def get_vars(self, play=None, host=None, task=None, include_hostvars=True, include_delegate_to=True, use_cache=True,
_hosts=None, _hosts_all=None):
_hosts=None, _hosts_all=None, stage='task'):
'''
Returns the variables, with optional "context" given via the parameters
for the play, host, and task (which could possibly result in different
@@ -231,25 +232,13 @@ def _get_plugin_vars(plugin, path, entities):
# internal fuctions that actually do the work
def _plugins_inventory(entities):
''' merges all entities by inventory source '''
data = {}
for inventory_dir in self._inventory._sources:
if ',' in inventory_dir and not os.path.exists(inventory_dir): # skip host lists
continue
elif not os.path.isdir(to_bytes(inventory_dir)): # always pass 'inventory directory'
inventory_dir = os.path.dirname(inventory_dir)

for plugin in vars_loader.all():

data = combine_vars(data, _get_plugin_vars(plugin, inventory_dir, entities))
return data
return get_vars_from_inventory_sources(self._loader, self._inventory._sources, entities, stage)

def _plugins_play(entities):
''' merges all entities adjacent to play '''
data = {}
for plugin in vars_loader.all():

for path in basedirs:
data = combine_vars(data, _get_plugin_vars(plugin, path, entities))
for path in basedirs:
data = combine_vars(data, get_vars_from_path(self._loader, path, entities, stage))
return data

# configurable functions that are sortable via config, rememer to add to _ALLOWED if expanding this list
@@ -281,7 +270,8 @@ def plugins_by_groups():
'''
data = {}
for group in host_groups:
data[group] = combine_vars(data[group], _plugins_inventory(group))
if C.RUN_VARS_PLUGINS == 'demand':
data[group] = combine_vars(data[group], _plugins_inventory(group))
data[group] = combine_vars(data[group], _plugins_play(group))
return data

@@ -296,7 +286,8 @@ def plugins_by_groups():

# host vars, from inventory, inventory adjacent and play adjacent via plugins
all_vars = combine_vars(all_vars, host.get_vars())
all_vars = combine_vars(all_vars, _plugins_inventory([host]))
if C.RUN_VARS_PLUGINS == 'demand':
all_vars = combine_vars(all_vars, _plugins_inventory([host]))
all_vars = combine_vars(all_vars, _plugins_play([host]))

# finally, the facts caches for this host, if it exists
@@ -0,0 +1,81 @@
# Copyright (c) 2018 Ansible Project
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os

from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.inventory.host import Host
from ansible.module_utils._text import to_bytes
from ansible.plugins.loader import vars_loader
from ansible.utils.collection_loader import AnsibleCollectionRef
from ansible.utils.display import Display
from ansible.utils.vars import combine_vars

display = Display()


def get_plugin_vars(loader, plugin, path, entities):

data = {}
try:
data = plugin.get_vars(loader, path, entities)
except AttributeError:
try:
for entity in entities:
if isinstance(entity, Host):
data.update(plugin.get_host_vars(entity.name))
else:
data.update(plugin.get_group_vars(entity.name))
except AttributeError:
if hasattr(plugin, 'run'):
raise AnsibleError("Cannot use v1 type vars plugin %s from %s" % (plugin._load_name, plugin._original_path))
else:
raise AnsibleError("Invalid vars plugin %s from %s" % (plugin._load_name, plugin._original_path))
return data


def get_vars_from_path(loader, path, entities, stage):

data = {}

vars_plugin_list = list(vars_loader.all())
for plugin_name in C.VARIABLE_PLUGINS_ENABLED:
if AnsibleCollectionRef.is_valid_fqcr(plugin_name):
vars_plugin = vars_loader.get(plugin_name)
if vars_plugin is None:
# Error if there's no play directory or the name is wrong?
continue
if vars_plugin not in vars_plugin_list:
vars_plugin_list.append(vars_plugin)

for plugin in vars_plugin_list:
if plugin._load_name not in C.VARIABLE_PLUGINS_ENABLED and getattr(plugin, 'REQUIRES_WHITELIST', False):
# 2.x plugins shipped with ansible should require whitelisting, older or non shipped should load automatically
continue
if hasattr(plugin, 'get_option') and plugin.has_option('stage') and plugin.get_option('stage') not in ('all', stage):
continue

data = combine_vars(data, get_plugin_vars(loader, plugin, path, entities))

return data


def get_vars_from_inventory_sources(loader, sources, entities, stage):

data = {}
for path in sources:

if ',' in path and not os.path.exists(path): # skip host lists
continue
elif not os.path.isdir(to_bytes(path)):
# always pass the directory of the inventory source file
path = os.path.dirname(path)

data = combine_vars(data, get_vars_from_path(loader, path, entities, stage))

return data
@@ -0,0 +1,45 @@
# Copyright 2019 RedHat, inc
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
#############################################
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

DOCUMENTATION = '''
vars: custom_vars
version_added: "2.9"
This conversation was marked as resolved by s-hertel

This comment has been minimized.

Copy link
@samccann

samccann Sep 26, 2019

Contributor

2.9 or 2.10

short_description: load host and group vars
description: test loading host and group vars from a collection
options:
stage:
default: all
choices: ['all', 'inventory', 'task']
type: str
ini:
- key: stage
section: custom_vars
env:
- name: ANSIBLE_VARS_PLUGIN_STAGE
'''

from ansible.plugins.vars import BaseVarsPlugin


class VarsModule(BaseVarsPlugin):

def get_vars(self, loader, path, entities, cache=True):
super(VarsModule, self).get_vars(loader, path, entities)
return {'collection': 'collection_root_user'}
ProTip! Use n and p to navigate between commits in a pull request.
You can’t perform that action at this time.