From 7f4328ad1261295d43f58f3ebc42444ac03cc5ea Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Thu, 5 Sep 2019 18:50:22 -0700 Subject: [PATCH] allow external collections under the ansible NS (#61908) * fixes #59988 --- changelogs/fragments/allow_ansible_ns.yml | 2 ++ lib/ansible/utils/collection_loader.py | 31 ++++++++++++++----- .../ansible/builtin/plugins/modules/ping.py | 13 ++++++++ .../bullcoll/plugins/modules/bullmodule.py | 13 ++++++++ .../integration/targets/collections/posix.yml | 13 ++++++++ 5 files changed, 65 insertions(+), 7 deletions(-) create mode 100644 changelogs/fragments/allow_ansible_ns.yml create mode 100644 test/integration/targets/collections/collection_root_user/ansible_collections/ansible/builtin/plugins/modules/ping.py create mode 100644 test/integration/targets/collections/collection_root_user/ansible_collections/ansible/bullcoll/plugins/modules/bullmodule.py diff --git a/changelogs/fragments/allow_ansible_ns.yml b/changelogs/fragments/allow_ansible_ns.yml new file mode 100644 index 00000000000000..e98ecc59ae780e --- /dev/null +++ b/changelogs/fragments/allow_ansible_ns.yml @@ -0,0 +1,2 @@ +bugfixes: +- allow external collections to be created in the 'ansible' collection namespace (https://github.com/ansible/ansible/issues/59988) diff --git a/lib/ansible/utils/collection_loader.py b/lib/ansible/utils/collection_loader.py index 76334ba27505d8..cc7e4df54c0000 100644 --- a/lib/ansible/utils/collection_loader.py +++ b/lib/ansible/utils/collection_loader.py @@ -23,7 +23,10 @@ import_module = __import__ _SYNTHETIC_PACKAGES = { - 'ansible_collections.ansible': dict(type='pkg_only'), + # these provide fallback package definitions when there are no on-disk paths + 'ansible_collections': dict(type='pkg_only', allow_external_subpackages=True), + 'ansible_collections.ansible': dict(type='pkg_only', allow_external_subpackages=True), + # these implement the ansible.builtin synthetic collection mapped to the packages inside the ansible distribution 'ansible_collections.ansible.builtin': dict(type='pkg_only'), 'ansible_collections.ansible.builtin.plugins': dict(type='map', map='ansible.plugins'), 'ansible_collections.ansible.builtin.plugins.module_utils': dict(type='map', map='ansible.module_utils', graft=True), @@ -101,7 +104,7 @@ def default_collection(self): def find_module(self, fullname, path=None): # this loader is only concerned with items under the Ansible Collections namespace hierarchy, ignore others - if fullname.startswith('ansible_collections.') or fullname == 'ansible_collections': + if fullname and fullname.split('.', 1)[0] == 'ansible_collections': return self return None @@ -110,6 +113,8 @@ def load_module(self, fullname): if sys.modules.get(fullname): return sys.modules[fullname] + newmod = None + # this loader implements key functionality for Ansible collections # * implicit distributed namespace packages for the root Ansible namespace (no pkgutil.extend_path hackery reqd) # * implicit package support for Python 2.7 (no need for __init__.py in collections, except to use standard Py2.7 tooling) @@ -132,10 +137,13 @@ def load_module(self, fullname): synpkg_remainder = '' if not synpkg_def: - synpkg_def = _SYNTHETIC_PACKAGES.get(parent_pkg_name) - synpkg_remainder = '.' + fullname.rpartition('.')[2] + # if the parent is a grafted package, we have some special work to do, otherwise just look for stuff on disk + parent_synpkg_def = _SYNTHETIC_PACKAGES.get(parent_pkg_name) + if parent_synpkg_def and parent_synpkg_def.get('graft'): + synpkg_def = parent_synpkg_def + synpkg_remainder = '.' + fullname.rpartition('.')[2] - # FIXME: collapse as much of this back to on-demand as possible (maybe stub packages that get replaced when actually loaded?) + # FUTURE: collapse as much of this back to on-demand as possible (maybe stub packages that get replaced when actually loaded?) if synpkg_def: pkg_type = synpkg_def.get('type') if not pkg_type: @@ -159,9 +167,13 @@ def load_module(self, fullname): newmod.__loader__ = self newmod.__path__ = [] - sys.modules[fullname] = newmod + if not synpkg_def.get('allow_external_subpackages'): + # if external subpackages are NOT allowed, we're done + sys.modules[fullname] = newmod + return newmod - return newmod + # if external subpackages ARE allowed, check for on-disk implementations and return a normal + # package if we find one, otherwise return the one we created here if not parent_pkg: # top-level package, look for NS subpackages on all collection paths package_paths = [self._extend_path_with_ns(p, fullname) for p in self.n_collection_paths] @@ -217,6 +229,11 @@ def load_module(self, fullname): return newmod + # even if we didn't find one on disk, fall back to a synthetic package if we have one... + if newmod: + sys.modules[fullname] = newmod + return newmod + # FIXME: need to handle the "no dirs present" case for at least the root and synthetic internal collections like ansible.builtin raise ImportError('module {0} not found'.format(fullname)) diff --git a/test/integration/targets/collections/collection_root_user/ansible_collections/ansible/builtin/plugins/modules/ping.py b/test/integration/targets/collections/collection_root_user/ansible_collections/ansible/builtin/plugins/modules/ping.py new file mode 100644 index 00000000000000..0747670929983d --- /dev/null +++ b/test/integration/targets/collections/collection_root_user/ansible_collections/ansible/builtin/plugins/modules/ping.py @@ -0,0 +1,13 @@ +#!/usr/bin/python +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import json + + +def main(): + print(json.dumps(dict(changed=False, source='overridden ansible.builtin (should not be possible)'))) + + +if __name__ == '__main__': + main() diff --git a/test/integration/targets/collections/collection_root_user/ansible_collections/ansible/bullcoll/plugins/modules/bullmodule.py b/test/integration/targets/collections/collection_root_user/ansible_collections/ansible/bullcoll/plugins/modules/bullmodule.py new file mode 100644 index 00000000000000..5ea354e7d0482c --- /dev/null +++ b/test/integration/targets/collections/collection_root_user/ansible_collections/ansible/bullcoll/plugins/modules/bullmodule.py @@ -0,0 +1,13 @@ +#!/usr/bin/python +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import json + + +def main(): + print(json.dumps(dict(changed=False, source='user_ansible_bullcoll'))) + + +if __name__ == '__main__': + main() diff --git a/test/integration/targets/collections/posix.yml b/test/integration/targets/collections/posix.yml index b94cee777b8837..63937dc5c31163 100644 --- a/test/integration/targets/collections/posix.yml +++ b/test/integration/targets/collections/posix.yml @@ -22,6 +22,16 @@ name: testns.testcoll.maskedmodule register: maskedmodule_out + # ensure the ansible ns can have real collections added to it + - name: call an external module in the ansible namespace + ansible.bullcoll.bullmodule: + register: bullmodule_out + + # ensure the ansible ns cannot override ansible.builtin externally + - name: call an external module in the ansible.builtin collection (should use the built in module) + ansible.builtin.ping: + register: builtin_ping_out + # action in a collection subdir - name: test subdir action FQ testns.testcoll.action_subdir.subdir_ping_action: @@ -59,6 +69,9 @@ - systestmodule_out.source == 'sys' - contentadjmodule_out.source == 'content_adj' - not maskedmodule_out.plugin_path + - bullmodule_out.source == 'user_ansible_bullcoll' + - builtin_ping_out.source is not defined + - builtin_ping_out.ping == 'pong' - subdir_ping_action_out is not changed - subdir_ping_module_out is not changed - granular_out.mu_result == 'thingtocall in leaf'