Initial ansible-test support for collections.

test/runner/lib/ansible_util.py

@@ -14,7 +14,7 @@
     display,
     find_python,
     ApplicationError,
-    INSTALL_ROOT,
+    ANSIBLE_ROOT,
 )
 
 from lib.util_common import (
@@ -26,6 +26,9 @@
     EnvironmentConfig,
 )
 
+from lib.data import (
+    data_context,
+)
 
 CHECK_YAML_VERSIONS = {}
 
@@ -40,17 +43,17 @@ def ansible_environment(args, color=True, ansible_config=None):
     env = common_environment()
     path = env['PATH']
 
-    ansible_path = os.path.join(INSTALL_ROOT, 'bin')
+    ansible_path = os.path.join(ANSIBLE_ROOT, 'bin')
 
     if not path.startswith(ansible_path + os.path.pathsep):
         path = ansible_path + os.path.pathsep + path
 
     if ansible_config:
         pass
     elif isinstance(args, IntegrationConfig):
-        ansible_config = os.path.join(INSTALL_ROOT, 'test/integration/%s.cfg' % args.command)
+        ansible_config = os.path.join(ANSIBLE_ROOT, 'test/integration/%s.cfg' % args.command)
     else:
-        ansible_config = os.path.join(INSTALL_ROOT, 'test/%s/ansible.cfg' % args.command)
+        ansible_config = os.path.join(ANSIBLE_ROOT, 'test/%s/ansible.cfg' % args.command)
 
     if not args.explain and not os.path.exists(ansible_config):
         raise ApplicationError('Configuration not found: %s' % ansible_config)
@@ -63,7 +66,7 @@ def ansible_environment(args, color=True, ansible_config=None):
         ANSIBLE_RETRY_FILES_ENABLED='false',
         ANSIBLE_CONFIG=os.path.abspath(ansible_config),
         ANSIBLE_LIBRARY='/dev/null',
-        PYTHONPATH=os.path.join(INSTALL_ROOT, 'lib'),
+        PYTHONPATH=os.path.join(ANSIBLE_ROOT, 'lib'),
         PAGER='/bin/cat',
         PATH=path,
     )
@@ -76,6 +79,11 @@ def ansible_environment(args, color=True, ansible_config=None):
             ANSIBLE_LOG_PATH=os.path.abspath('test/results/logs/debug.log'),
         ))
 
+    if data_context().content.collection:
+        env.update(dict(
+            ANSIBLE_COLLECTIONS_PATHS=data_context().content.collection.root,
+        ))
+
     return env
 
 
@@ -88,7 +96,7 @@ def check_pyyaml(args, version):
         return
 
     python = find_python(version)
-    stdout, _dummy = run_command(args, [python, os.path.join(INSTALL_ROOT, 'test/runner/yamlcheck.py')], capture=True)
+    stdout, _dummy = run_command(args, [python, os.path.join(ANSIBLE_ROOT, 'test/runner/yamlcheck.py')], capture=True)
 
     if args.explain:
         return

test/runner/lib/classification.py

@@ -19,18 +19,22 @@
 
 from lib.util import (
     display,
+    is_subdir,
 )
 
 from lib.import_analysis import (
     get_python_module_utils_imports,
+    get_python_module_utils_name,
 )
 
 from lib.csharp_import_analysis import (
     get_csharp_module_utils_imports,
+    get_csharp_module_utils_name,
 )
 
 from lib.powershell_import_analysis import (
     get_powershell_module_utils_imports,
+    get_powershell_module_utils_name,
 )
 
 from lib.config import (
@@ -42,6 +46,10 @@
     ChangeDescription,
 )
 
+from lib.data import (
+    data_context,
+)
+
 FOCUSED_TARGET = '__focused__'
 
 
@@ -184,7 +192,7 @@ def __init__(self, args):
         self.compile_targets = list(walk_compile_targets())
         self.units_targets = list(walk_units_targets())
         self.sanity_targets = list(walk_sanity_targets())
-        self.powershell_targets = [t for t in self.sanity_targets if os.path.splitext(t.path)[1] == '.ps1']
+        self.powershell_targets = [t for t in self.sanity_targets if os.path.splitext(t.path)[1] in ('.ps1', '.psm1')]
         self.csharp_targets = [t for t in self.sanity_targets if os.path.splitext(t.path)[1] == '.cs']
 
         self.units_modules = set(t.module for t in self.units_targets if t.module)
@@ -258,7 +266,7 @@ def get_dependent_paths_internal(self, path):
         """
         ext = os.path.splitext(os.path.split(path)[1])[1]
 
-        if path.startswith('lib/ansible/module_utils/'):
+        if is_subdir(path, data_context().content.module_utils_path):
             if ext == '.py':
                 return self.get_python_module_utils_usage(path)
 
@@ -288,10 +296,7 @@ def get_python_module_utils_usage(self, path):
             after = time.time()
             display.info('Processed %d python module_utils in %d second(s).' % (len(self.python_module_utils_imports), after - before))
 
-        name = os.path.splitext(path)[0].replace('/', '.')[4:]
-
-        if name.endswith('.__init__'):
-            name = name[:-9]
+        name = get_python_module_utils_name(path)
 
         return sorted(self.python_module_utils_imports[name])
 
@@ -307,7 +312,7 @@ def get_powershell_module_utils_usage(self, path):
             after = time.time()
             display.info('Processed %d powershell module_utils in %d second(s).' % (len(self.powershell_module_utils_imports), after - before))
 
-        name = os.path.splitext(os.path.basename(path))[0]
+        name = get_powershell_module_utils_name(path)
 
         return sorted(self.powershell_module_utils_imports[name])
 
@@ -323,7 +328,7 @@ def get_csharp_module_utils_usage(self, path):
             after = time.time()
             display.info('Processed %d C# module_utils in %d second(s).' % (len(self.csharp_module_utils_imports), after - before))
 
-        name = os.path.splitext(os.path.basename(path))[0]
+        name = get_csharp_module_utils_name(path)
 
         return sorted(self.csharp_module_utils_imports[name])
 

test/runner/lib/cli.py

@@ -21,7 +21,6 @@
     generate_pip_command,
     read_lines_without_comments,
     MAXFD,
-    INSTALL_ROOT,
 )
 
 from lib.delegation import (
@@ -81,17 +80,25 @@
     initialize_cloud_plugins,
 )
 
+from lib.data import (
+    data_context,
+)
+
+from lib.util_common import (
+    CommonConfig,
+)
+
 import lib.cover
 
 
 def main():
     """Main program function."""
     try:
-        os.chdir(INSTALL_ROOT)
+        os.chdir(data_context().content.root)
         initialize_cloud_plugins()
         sanity_init()
         args = parse_args()
-        config = args.config(args)
+        config = args.config(args)  # type: CommonConfig
         display.verbosity = config.verbosity
         display.truncate = config.truncate
         display.redact = config.redact
@@ -106,8 +113,13 @@ def main():
 
         try:
             args.func(config)
+            delegate_args = None
         except Delegate as ex:
-            delegate(config, ex.exclude, ex.require, ex.integration_targets)
+            # save delegation args for use once we exit the exception handler
+            delegate_args = (ex.exclude, ex.require, ex.integration_targets)
+
+        if delegate_args:
+            delegate(config, *delegate_args)
 
         display.review_warnings()
     except ApplicationWarning as ex:
@@ -614,24 +626,30 @@ def add_environments(parser, tox_version=False, tox_only=False):
                               action='store_true',
                               help='run from the local environment')
 
-    if tox_version:
-        environments.add_argument('--tox',
-                                  metavar='VERSION',
-                                  nargs='?',
-                                  default=None,
-                                  const='.'.join(str(i) for i in sys.version_info[:2]),
-                                  choices=SUPPORTED_PYTHON_VERSIONS,
-                                  help='run from a tox virtualenv: %s' % ', '.join(SUPPORTED_PYTHON_VERSIONS))
+    if data_context().content.is_ansible:
+        if tox_version:
+            environments.add_argument('--tox',
+                                      metavar='VERSION',
+                                      nargs='?',
+                                      default=None,
+                                      const='.'.join(str(i) for i in sys.version_info[:2]),
+                                      choices=SUPPORTED_PYTHON_VERSIONS,
+                                      help='run from a tox virtualenv: %s' % ', '.join(SUPPORTED_PYTHON_VERSIONS))
+        else:
+            environments.add_argument('--tox',
+                                      action='store_true',
+                                      help='run from a tox virtualenv')
+
+        tox = parser.add_argument_group(title='tox arguments')
+
+        tox.add_argument('--tox-sitepackages',
+                         action='store_true',
+                         help='allow access to globally installed packages')
     else:
-        environments.add_argument('--tox',
-                                  action='store_true',
-                                  help='run from a tox virtualenv')
-
-    tox = parser.add_argument_group(title='tox arguments')
-
-    tox.add_argument('--tox-sitepackages',
-                     action='store_true',
-                     help='allow access to globally installed packages')
+        environments.set_defaults(
+            tox=None,
+            tox_sitepackages=False,
+        )
 
     if tox_only:
         environments.set_defaults(
@@ -739,9 +757,14 @@ def add_extra_docker_options(parser, integration=True):
                         dest='docker_pull',
                         help='do not explicitly pull the latest docker images')
 
-    docker.add_argument('--docker-keep-git',
-                        action='store_true',
-                        help='transfer git related files into the docker container')
+    if data_context().content.is_ansible:
+        docker.add_argument('--docker-keep-git',
+                            action='store_true',
+                            help='transfer git related files into the docker container')
+    else:
+        docker.set_defaults(
+            docker_keep_git=False,
+        )
 
     docker.add_argument('--docker-seccomp',
                         metavar='SC',
@@ -848,10 +871,10 @@ def complete_network_testcase(prefix, parsed_args, **_):
         return []
 
     test_dir = 'test/integration/targets/%s/tests' % parsed_args.include[0]
-    connection_dirs = [path for path in [os.path.join(test_dir, name) for name in os.listdir(test_dir)] if os.path.isdir(path)]
+    connection_dirs = data_context().content.get_dirs(test_dir)
 
     for connection_dir in connection_dirs:
-        for testcase in os.listdir(connection_dir):
+        for testcase in [os.path.basename(path) for path in data_context().content.get_files(connection_dir)]:
             if testcase.startswith(prefix):
                 testcases.append(testcase.split('.')[0])
 

test/runner/lib/cloud/__init__.py

@@ -13,13 +13,16 @@
 import re
 import tempfile
 
+import lib.types as t
+
 from lib.util import (
     ApplicationError,
     display,
     is_shippable,
     import_plugins,
     load_plugins,
     ABC,
+    to_bytes,
 )
 
 from lib.target import (
@@ -30,6 +33,10 @@
     IntegrationConfig,
 )
 
+from lib.data import (
+    data_context,
+)
+
 PROVIDERS = {}
 ENVIRONMENTS = {}
 
@@ -55,7 +62,7 @@ def get_cloud_platforms(args, targets=None):
     if targets is None:
         cloud_platforms = set(args.metadata.cloud_config or [])
     else:
-        cloud_platforms = set(get_cloud_platform(t) for t in targets)
+        cloud_platforms = set(get_cloud_platform(target) for target in targets)
 
     cloud_platforms.discard(None)
 
@@ -145,7 +152,7 @@ def cloud_init(args, targets):
         results[provider.platform] = dict(
             platform=provider.platform,
             setup_seconds=int(end_time - start_time),
-            targets=[t.name for t in targets],
+            targets=[target.name for target in targets],
         )
 
     if not args.explain and results:
@@ -175,6 +182,17 @@ def __init__(self, args):
         self.args = args
         self.platform = self.__module__.split('.')[2]
 
+        def config_callback(files):  # type: (t.List[t.Tuple[str, str]]) -> None
+            """Add the config file to the payload file list."""
+            if self._get_cloud_config(self._CONFIG_PATH, ''):
+                pair = (self.config_path, os.path.relpath(self.config_path, data_context().content.root))
+
+                if pair not in files:
+                    display.info('Including %s config: %s -> %s' % (self.platform, pair[0], pair[1]), verbosity=3)
+                    files.append(pair)
+
+        data_context().register_payload_callback(config_callback)
+
     @property
     def setup_executed(self):
         """
@@ -194,7 +212,7 @@ def config_path(self):
         """
         :rtype: str
         """
-        return os.path.join(os.getcwd(), self._get_cloud_config(self._CONFIG_PATH))
+        return os.path.join(data_context().content.root, self._get_cloud_config(self._CONFIG_PATH))
 
     @config_path.setter
     def config_path(self, value):
@@ -334,7 +352,7 @@ def _write_config(self, content):
 
             display.info('>>> Config: %s\n%s' % (filename, content.strip()), verbosity=3)
 
-            config_fd.write(content.encode('utf-8'))
+            config_fd.write(to_bytes(content))
             config_fd.flush()
 
     def _read_config_template(self):