Improve handling of integration test aliases.

test/integration/targets/azure_rm_containerinstance/aliases

@@ -1,2 +1,4 @@
 cloud/azure
 destructive
+posix/ci/cloud/group2/azure
+unstable

test/integration/targets/azure_rm_keyvaultkey/aliases

@@ -1,3 +1,4 @@
 cloud/azure
-posix/ci/cloud/azure
+posix/ci/cloud/group2/azure
 destructive
+disabled

test/integration/targets/azure_rm_keyvaultsecret/aliases

@@ -1,3 +1,4 @@
 cloud/azure
-posix/ci/cloud/azure
+posix/ci/cloud/group2/azure
 destructive
+disabled

test/integration/targets/azure_rm_loadbalancer/aliases

@@ -1,2 +1,4 @@
 cloud/azure
+posix/ci/cloud/group3/azure
+unstable
 destructive

test/integration/targets/azure_rm_postgresqldatabase/aliases

@@ -1,2 +1,4 @@
 cloud/azure
 destructive
+posix/ci/cloud/group2/azure
+unstable

test/integration/targets/azure_rm_postgresqlserver/aliases

@@ -1,2 +1,4 @@
 cloud/azure
 destructive
+posix/ci/cloud/group2/azure
+unstable

test/integration/targets/azure_rm_securitygroup/aliases

@@ -1,3 +1,5 @@
 cloud/azure
+posix/ci/cloud/group2/azure
+unstable
 destructive
 azure_rm_securitygroup_facts

test/integration/targets/azure_rm_sqldatabase/aliases

@@ -1,2 +1,4 @@
 cloud/azure
 destructive
+posix/ci/cloud/group2/azure
+unstable

test/integration/targets/azure_rm_sqlserver_facts/aliases

@@ -1,2 +1,4 @@
 cloud/azure
 destructive
+posix/ci/cloud/group2/azure
+unstable

test/integration/targets/azure_rm_virtualmachine/aliases

@@ -1,2 +1,4 @@
 cloud/azure
+posix/ci/cloud/group3/azure
+unstable
 destructive

test/integration/targets/ec2_ami/aliases

@@ -1,2 +1,4 @@
 cloud/aws
+posix/ci/cloud/group4/aws
+disabled
 ec2_ami_facts

test/integration/targets/ec2_group/aliases

@@ -1 +1,3 @@
 cloud/aws
+posix/ci/cloud/group4/aws
+disabled

test/integration/targets/ec2_vpc_route_table/aliases

@@ -1 +1,3 @@
 cloud/aws
+posix/ci/cloud/group4/aws
+disabled

test/integration/targets/github_issue/aliases

@@ -1 +1,3 @@
 destructive
+posix/ci/group1
+disabled

test/integration/targets/influxdb_user/aliases

@@ -1,4 +1,6 @@
 destructive
+posix/ci/group1
+disabled
 skip/osx
 skip/freebsd
 skip/rhel

test/integration/targets/nuage_vspk/aliases

@@ -1 +1,3 @@
+posix/ci/group1
 skip/python3
+disabled

test/integration/targets/vmware_datastore_maintenancemode/aliases

@@ -1,2 +1,4 @@
 cloud/vcenter
 destructive
+posix/ci/cloud/group4/vcenter
+disabled

test/integration/targets/win_msg/aliases

@@ -0,0 +1,2 @@
+windows/ci/group2
+unstable

test/integration/targets/win_psexec/aliases

@@ -0,0 +1,2 @@
+windows/ci/group3
+unstable

test/integration/targets/win_rabbitmq_plugin/aliases

@@ -0,0 +1,2 @@
+windows/ci/group3
+disabled

test/integration/targets/win_toast/aliases

@@ -1 +1,2 @@
-
+windows/ci/group1
+disabled

test/integration/targets/win_uri/aliases

@@ -0,0 +1,2 @@
+windows/ci/group3
+unstable

test/integration/targets/win_wait_for/aliases

@@ -0,0 +1,2 @@
+windows/ci/group1
+unstable

test/integration/targets/xml/aliases

@@ -1 +1,3 @@
 destructive
+posix/ci/group1
+disabled

test/runner/lib/classification.py

@@ -2,6 +2,7 @@
 
 from __future__ import absolute_import, print_function
 
+import collections
 import os
 import re
 import time
@@ -33,13 +34,19 @@
     IntegrationConfig,
 )
 
+from lib.metadata import (
+    ChangeDescription,
+)
+
+FOCUSED_TARGET = '__focused__'
+
 
 def categorize_changes(args, paths, verbose_command=None):
     """
     :type args: TestConfig
     :type paths: list[str]
     :type verbose_command: str
-    :rtype paths: dict[str, list[str]]
+    :rtype: ChangeDescription
     """
     mapper = PathMapper(args)
 
@@ -51,12 +58,20 @@ def categorize_changes(args, paths, verbose_command=None):
         'network-integration': set(),
     }
 
+    focused_commands = collections.defaultdict(set)
+
+    deleted_paths = set()
+    original_paths = set()
     additional_paths = set()
+    no_integration_paths = set()
 
     for path in paths:
         if not os.path.exists(path):
+            deleted_paths.add(path)
             continue
 
+        original_paths.add(path)
+
         dependent_paths = mapper.get_dependent_paths(path)
 
         if not dependent_paths:
@@ -80,18 +95,28 @@ def categorize_changes(args, paths, verbose_command=None):
         tests = mapper.classify(path)
 
         if tests is None:
+            focused_target = False
+
             display.info('%s -> all' % path, verbosity=1)
             tests = all_tests(args)  # not categorized, run all tests
             display.warning('Path not categorized: %s' % path)
         else:
+            focused_target = tests.pop(FOCUSED_TARGET, False) and path in original_paths
+
             tests = dict((key, value) for key, value in tests.items() if value)
 
+            if focused_target and not any('integration' in command for command in tests):
+                no_integration_paths.add(path)  # path triggers no integration tests
+
             if verbose_command:
                 result = '%s: %s' % (verbose_command, tests.get(verbose_command) or 'none')
 
                 # identify targeted integration tests (those which only target a single integration command)
                 if 'integration' in verbose_command and tests.get(verbose_command):
                     if not any('integration' in command for command in tests if command != verbose_command):
+                        if focused_target:
+                            result += ' (focused)'
+
                         result += ' (targeted)'
             else:
                 result = '%s' % tests
@@ -101,15 +126,31 @@ def categorize_changes(args, paths, verbose_command=None):
         for command, target in tests.items():
             commands[command].add(target)
 
+            if focused_target:
+                focused_commands[command].add(target)
+
     for command in commands:
         commands[command].discard('none')
 
         if any(t == 'all' for t in commands[command]):
             commands[command] = set(['all'])
 
     commands = dict((c, sorted(commands[c])) for c in commands if commands[c])
+    focused_commands = dict((c, sorted(focused_commands[c])) for c in focused_commands)
+
+    for command in commands:
+        if commands[command] == ['all']:
+            commands[command] = []  # changes require testing all targets, do not filter targets
+
+    changes = ChangeDescription()
+    changes.command = verbose_command
+    changes.changed_paths = sorted(original_paths)
+    changes.deleted_paths = sorted(deleted_paths)
+    changes.regular_command_targets = commands
+    changes.focused_command_targets = focused_commands
+    changes.no_integration_paths = sorted(no_integration_paths)
 
-    return commands
+    return changes
 
 
 class PathMapper(object):
@@ -278,6 +319,7 @@ def _classify(self, path):
                     'integration': self.posix_integration_by_module.get(module_name) if ext == '.py' else None,
                     'windows-integration': self.windows_integration_by_module.get(module_name) if ext == '.ps1' else None,
                     'network-integration': self.network_integration_by_module.get(module_name),
+                    FOCUSED_TARGET: True,
                 }
 
             return minimal
@@ -459,6 +501,7 @@ def _classify(self, path):
                 'integration': target.name if 'posix/' in target.aliases else None,
                 'windows-integration': target.name if 'windows/' in target.aliases else None,
                 'network-integration': target.name if 'network/' in target.aliases else None,
+                FOCUSED_TARGET: True,
             }
 
         if path.startswith('test/integration/'):

test/runner/lib/config.py

@@ -162,6 +162,10 @@ def __init__(self, args, command):
         self.start_at_task = args.start_at_task  # type: str
         self.allow_destructive = args.allow_destructive  # type: bool
         self.allow_root = args.allow_root  # type: bool
+        self.allow_disabled = args.allow_disabled  # type: bool
+        self.allow_unstable = args.allow_unstable  # type: bool
+        self.allow_unstable_changed = args.allow_unstable_changed  # type: bool
+        self.allow_unsupported = args.allow_unsupported  # type: bool
         self.retry_on_error = args.retry_on_error  # type: bool
         self.continue_on_error = args.continue_on_error  # type: bool
         self.debug_strategy = args.debug_strategy  # type: bool

test/runner/lib/executor.py

@@ -88,6 +88,10 @@
     WindowsIntegrationConfig,
 )
 
+from lib.metadata import (
+    ChangeDescription,
+)
+
 SUPPORTED_PYTHON_VERSIONS = (
     '2.6',
     '2.7',
@@ -1029,23 +1033,24 @@ def get_changes_filter(args):
     """
     paths = detect_changes(args)
 
+    if not args.metadata.change_description:
+        if paths:
+            changes = categorize_changes(args, paths, args.command)
+        else:
+            changes = ChangeDescription()
+
+        args.metadata.change_description = changes
+
     if paths is None:
         return []  # change detection not enabled, do not filter targets
 
     if not paths:
         raise NoChangesDetected()
 
-    commands = categorize_changes(args, paths, args.command)
-
-    targets = commands.get(args.command)
-
-    if targets is None:
+    if args.metadata.change_description.targets is None:
         raise NoTestsForChanges()
 
-    if targets == ['all']:
-        return []  # changes require testing all targets, do not filter targets
-
-    return targets
+    return args.metadata.change_description.targets
 
 
 def detect_changes(args):
@@ -1175,6 +1180,49 @@ def get_integration_filter(args, targets):
     return get_integration_local_filter(args, targets)
 
 
+def common_integration_filter(args, targets, exclude):
+    """
+    :type args: IntegrationConfig
+    :type targets: tuple[IntegrationTarget]
+    :type exclude: list[str]
+    """
+    override_disabled = set(target for target in args.include if target.startswith('disabled/'))
+
+    if not args.allow_disabled:
+        skip = 'disabled/'
+        override = [target.name for target in targets if override_disabled & set(target.aliases)]
+        skipped = [target.name for target in targets if skip in target.aliases and target.name not in override]
+        if skipped:
+            exclude.extend(skipped)
+            display.warning('Excluding tests marked "%s" which require --allow-disabled or prefixing with "disabled/": %s'
+                            % (skip.rstrip('/'), ', '.join(skipped)))
+
+    override_unsupported = set(target for target in args.include if target.startswith('unsupported/'))
+
+    if not args.allow_unsupported:
+        skip = 'unsupported/'
+        override = [target.name for target in targets if override_unsupported & set(target.aliases)]
+        skipped = [target.name for target in targets if skip in target.aliases and target.name not in override]
+        if skipped:
+            exclude.extend(skipped)
+            display.warning('Excluding tests marked "%s" which require --allow-unsupported or prefixing with "unsupported/": %s'
+                            % (skip.rstrip('/'), ', '.join(skipped)))
+
+    override_unstable = set(target for target in args.include if target.startswith('unstable/'))
+
+    if args.allow_unstable_changed:
+        override_unstable |= set(args.metadata.change_description.focused_targets or [])
+
+    if not args.allow_unstable:
+        skip = 'unstable/'
+        override = [target.name for target in targets if override_unstable & set(target.aliases)]
+        skipped = [target.name for target in targets if skip in target.aliases and target.name not in override]
+        if skipped:
+            exclude.extend(skipped)
+            display.warning('Excluding tests marked "%s" which require --allow-unstable or prefixing with "unstable/": %s'
+                            % (skip.rstrip('/'), ', '.join(skipped)))
+
+
 def get_integration_local_filter(args, targets):
     """
     :type args: IntegrationConfig
@@ -1183,6 +1231,8 @@ def get_integration_local_filter(args, targets):
     """
     exclude = []
 
+    common_integration_filter(args, targets, exclude)
+
     if not args.allow_root and os.getuid() != 0:
         skip = 'needs/root/'
         skipped = [target.name for target in targets if skip in target.aliases]
@@ -1225,6 +1275,8 @@ def get_integration_docker_filter(args, targets):
     """
     exclude = []
 
+    common_integration_filter(args, targets, exclude)
+
     if not args.docker_privileged:
         skip = 'needs/privileged/'
         skipped = [target.name for target in targets if skip in target.aliases]
@@ -1271,6 +1323,8 @@ def get_integration_remote_filter(args, targets):
 
     exclude = []
 
+    common_integration_filter(args, targets, exclude)
+
     skip = 'skip/%s/' % platform
     skipped = [target.name for target in targets if skip in target.aliases]
     if skipped: