Filter container images by deployed services

This change adds the --service-environment-file/-e argument to the
prepare command which will take heat environment files that specify
which services are containerised.

When specified, the resulting image entries will be filtered only by
the containerised services discovered in the environment.

-e is used as the short argument so that tools like quickstart can use
the same variable to specify these environments for both the prepare
command and the overcloud deploy command.

Once this change merges, any calls to prepare in CI can be modified to
reduce the number of images being transferred for 'upload' calls. For
example, specifying -e environment/docker.yaml will reduce image
transfers from 85 to 44.

Change-Id: I35866fb24915ef80b86cc402015afb08cb155dcf
Partial-Bug: #1710992
Depends-On: #Ie8801b11921c46923b0f7c9aaba6bf524c464e82

releasenotes/notes/prepare-service-4281d7358be7450a.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - The "openstack overcloud container image prepare" command can now filter
+    the image list by the containerized services being deployed. This is done by
+    specifying the heat environment files which enable containerized services.

tripleoclient/tests/v1/overcloud_image/test_container_image.py

@@ -97,13 +97,23 @@ def test_container_image_prepare_noargs(self, mock_builder):
         )
 
     @mock.patch('tripleo_common.image.kolla_builder.KollaImageBuilder')
-    def test_container_image_prepare(self, mock_builder):
+    @mock.patch('heatclient.common.template_utils.'
+                'process_multiple_environments_and_files')
+    def test_container_image_prepare(self, pmef, mock_builder):
 
         temp = tempfile.mkdtemp()
         self.addCleanup(shutil.rmtree, temp)
         images_file = os.path.join(temp, 'overcloud_containers.yaml')
         env_file = os.path.join(temp, 'containers_env.yaml')
         tmpl_file = os.path.join(temp, 'overcloud_containers.yaml.j2')
+        aodh_file = os.path.join(temp, 'docker', 'services',
+                                 'overcloud_containers.yaml.j2')
+
+        resource_registry = {'resource_registry': {
+            'OS::TripleO::Services::AodhEvaluator': aodh_file,
+            'OS::TripleO::Services::AodhApi': aodh_file
+        }}
+        pmef.return_value = None, resource_registry
 
         arglist = [
             '--template-file',
@@ -126,16 +136,25 @@ def test_container_image_prepare(self, mock_builder):
             'ceph_image=mydaemon',
             '--set',
             'ceph_tag=mytag',
+            '-e',
+            'environment/docker.yaml'
         ]
         self.cmd.app.command_options = arglist
         verifylist = []
         cift = mock.MagicMock()
         cift.return_value = [{
             'imagename': 'tripleo/os-aodh-apifoo:passed-ci',
             'params': ['DockerAodhApiImage', 'DockerAodhConfigImage'],
+            'services': [
+                'OS::TripleO::Services::AodhApi',
+                'OS::TripleO::Services::AodhEvaluator',
+            ],
         }, {
-            'imagename': 'tripleo/os-heat-apifoo:passed-ci',
-            'params': ['DockerHeatApiImage'],
+            'imagename': 'tripleo/os-aodh-evaluatorfoo:passed-ci',
+            'params': ['DockerAodhEvaluatorImage'],
+            'services': [
+                'OS::TripleO::Services::AodhEvaluator',
+            ],
         }]
 
         mock_builder.return_value.container_images_from_template = cift
@@ -145,6 +164,7 @@ def test_container_image_prepare(self, mock_builder):
         self.cmd.take_action(parsed_args)
 
         mock_builder.assert_called_once_with([tmpl_file])
+        pmef.assert_called_once_with(['environment/docker.yaml'])
         cift.assert_called_once_with(
             filter=mock.ANY,
             name_prefix='os-',
@@ -159,14 +179,15 @@ def test_container_image_prepare(self, mock_builder):
             'container_images': [{
                 'imagename': 'tripleo/os-aodh-apifoo:passed-ci',
             }, {
-                'imagename': 'tripleo/os-heat-apifoo:passed-ci',
+                'imagename': 'tripleo/os-aodh-evaluatorfoo:passed-ci',
             }]
         }
         env_data = {
             'parameter_defaults': {
                 'DockerAodhApiImage': 'tripleo/os-aodh-apifoo:passed-ci',
                 'DockerAodhConfigImage': 'tripleo/os-aodh-apifoo:passed-ci',
-                'DockerHeatApiImage': 'tripleo/os-heat-apifoo:passed-ci',
+                'DockerAodhEvaluatorImage':
+                    'tripleo/os-aodh-evaluatorfoo:passed-ci'
             }
         }
         with open(images_file) as f:

tripleoclient/v1/container_image.py

@@ -21,6 +21,7 @@
 import sys
 import tempfile
 
+from heatclient.common import template_utils
 from osc_lib.command import command
 from osc_lib import exceptions as oscexc
 from osc_lib.i18n import _
@@ -243,6 +244,14 @@ def get_parser(self, prog_name):
             help=_("File to write resulting image entries to, as well as "
                    "stdout. Any existing file will be overwritten."),
         )
+        parser.add_argument(
+            '--service-environment-file', '-e', metavar='<file path>',
+            action='append', dest='environment_files',
+            help=_('Environment files specifying which services are '
+                   'containerized. Entries will be filtered to only contain '
+                   'images used by containerized services. (Can be specified '
+                   'more than once.)')
+        )
         parser.add_argument(
             "--env-file",
             dest="env_file",
@@ -277,6 +286,21 @@ def write_env_file(self, params, env_file):
             yaml.safe_dump({'parameter_defaults': params}, f,
                            default_flow_style=False)
 
+    def build_service_filter(self, environment_files):
+        if not environment_files:
+            return None
+
+        service_filter = set()
+        env_files, env = (
+            template_utils.process_multiple_environments_and_files(
+                environment_files))
+        for service, env_path in env.get('resource_registry', {}).items():
+            # Use the template path to determine if it represents a
+            # containerized service
+            if '/docker/services/' in env_path:
+                service_filter.add(service)
+        return service_filter
+
     def take_action(self, parsed_args):
         self.log.debug("take_action(%s)" % parsed_args)
         subs = {
@@ -287,11 +311,19 @@ def take_action(self, parsed_args):
         }
         self.parse_set_values(subs, parsed_args.set)
 
+        service_filter = self.build_service_filter(
+            parsed_args.environment_files)
+
         def ffunc(entry):
             imagename = entry.get('imagename', '')
             for p in parsed_args.excludes:
                 if re.search(p, imagename):
                     return None
+            if service_filter is not None:
+                # check the entry is for a service being deployed
+                image_services = set(entry.get('services', []))
+                if not service_filter.intersection(image_services):
+                    return None
             if parsed_args.pull_source:
                 entry['pull_source'] = parsed_args.pull_source
             if parsed_args.push_destination:
@@ -307,6 +339,8 @@ def ffunc(entry):
             if 'params' in entry:
                 for p in entry.pop('params'):
                     params[p] = imagename
+            if 'services' in entry:
+                del(entry['services'])
 
         if parsed_args.env_file:
             self.write_env_file(params, parsed_args.env_file)