openstack-* containers in dlrnapi_promoter

openstack-* is the prefix for master/victoria onwards. For older
branches, distro-binary-* is the prefix. This change updates the
default settings in legacy config and all affected tests as it is
the current workflow in production (new config engine is WIP).

Change-Id: Ie808f3abf5b5b3d4a764fb1ba6a1f67b3756a4d1

ci-scripts/dlrnapi_promoter/config/CentOS-7/master.ini

@@ -4,6 +4,8 @@ distro_version: 7
 release: master
 api_url: https://trunk.rdoproject.org/api-centos-master-uc
 username: ciuser
+build_method: tripleo
+container_preffix: openstack-
 # the password should be set as DLRNAPI_PASSWORD in the environment
 dry_run: no
 log_file: ~/web/promoter_logs/centos7_master.log

ci-scripts/dlrnapi_promoter/config/CentOS-7/queens.ini

@@ -5,6 +5,8 @@ release: queens
 api_url: https://trunk.rdoproject.org/api-centos-queens
 base_url: https://trunk.rdoproject.org/centos7-queens/
 username: ciuser
+build_method: kolla
+container_preffix: centos-binary-
 # the password should be set as DLRNAPI_PASSWORD in the environment
 dry_run: no
 log_file: ~/web/promoter_logs/centos7_queens.log

ci-scripts/dlrnapi_promoter/config/CentOS-7/rocky.ini

@@ -5,6 +5,8 @@ release: rocky
 api_url: https://trunk.rdoproject.org/api-centos-rocky
 base_url: https://trunk.rdoproject.org/centos7-rocky/
 username: ciuser
+build_method: kolla
+container_preffix: centos-binary-
 # the password should be set as DLRNAPI_PASSWORD in the environment
 dry_run: no
 log_file: ~/web/promoter_logs/centos7_rocky.log

ci-scripts/dlrnapi_promoter/config/CentOS-7/stein.ini

@@ -5,6 +5,8 @@ release: stein
 api_url: https://trunk.rdoproject.org/api-centos-stein
 base_url: https://trunk.rdoproject.org/centos7-stein/
 username: ciuser
+build_method: kolla
+container_preffix: centos-binary-
 # the password should be set as DLRNAPI_PASSWORD in the environment
 dry_run: no
 log_file: ~/web/promoter_logs/centos7_stein.log

ci-scripts/dlrnapi_promoter/config/CentOS-7/train.ini

@@ -5,6 +5,8 @@ release: train
 api_url: https://trunk.rdoproject.org/api-centos-train
 base_url: https://trunk.rdoproject.org/centos7-train/
 username: ciuser
+build_method: kolla
+container_preffix: centos-binary-
 # the password should be set as DLRNAPI_PASSWORD in the environment
 dry_run: no
 log_file: ~/web/promoter_logs/centos7_train.log

ci-scripts/dlrnapi_promoter/config/CentOS-8/master.ini

@@ -5,11 +5,12 @@ release: master
 api_url: https://trunk.rdoproject.org/api-centos8-master-uc
 base_url: https://trunk.rdoproject.org/centos8-master/
 username: ciuser
+build_method: tripleo
+container_preffix: openstack-
 # the password should be set as DLRNAPI_PASSWORD in the environment
 dry_run: no
 log_file: ~/web/promoter_logs/centos8_master.log
 containers_list_path: container-images/tripleo_containers.yaml
-build_method: tripleo
 latest_hashes_count: 10
 manifest_push: false
 

ci-scripts/dlrnapi_promoter/config/CentOS-8/train.ini

@@ -5,6 +5,8 @@ release: train
 api_url: https://trunk.rdoproject.org/api-centos8-train
 base_url: https://trunk.rdoproject.org/centos8-train/
 username: ciuser
+build_method: kolla
+container_preffix: centos-binary-
 # the password should be set as DLRNAPI_PASSWORD in the environment
 dry_run: no
 log_file: ~/web/promoter_logs/centos8_train.log

ci-scripts/dlrnapi_promoter/config/CentOS-8/ussuri.ini

@@ -9,6 +9,7 @@ username: ciuser
 dry_run: no
 containers_list_path: container-images/tripleo_containers.yaml
 build_method: tripleo
+container_preffix: centos-binary-
 log_file: ~/web/promoter_logs/centos8_ussuri.log
 latest_hashes_count: 10
 manifest_push: false

ci-scripts/dlrnapi_promoter/config/staging/CentOS-8/master.ini

@@ -9,6 +9,7 @@ containers_list_path: container-images/tripleo_containers.yaml
 containers_list_exclude_config: file:///tmp/promoter-staging/containers/build-containers-main.yaml
 username: ciuser
 build_method: tripleo
+container_preffix: openstack-
 # the password should be set as DLRNAPI_PASSWORD in the environment
 dry_run: no
 log_file: ~/web/promoter_logs/centos8_master.log

ci-scripts/dlrnapi_promoter/config_legacy.py

@@ -44,14 +44,15 @@ class PromoterLegacyConfigBase(object):
         'allowed_clients': 'registries_client,qcow_client,dlrn_client',
         'log_level': "INFO",
         "dlrn_api_host": "trunk.rdoproject.org",
-        "build_method": "kolla",
+        "build_method": "tripleo",
+        "container_preffix": "openstack-",
         "containers_list_base_url": ("https://opendev.org/openstack/"
                                      "tripleo-common/raw/commit/"),
         # For old container kolla based workflow, the source of truth of
         # containers is overcloud_containers.yaml.
         # For new container (non-kolla) based workflow, the source of truth
         # is tripleo_containers.yaml.
-        "containers_list_path": "container-images/overcloud_containers.yaml",
+        "containers_list_path": "container-images/tripleo_containers.yaml",
         "containers_list_exclude_config": (
             "https://opendev.org/openstack/"
             "tripleo-ci/raw/branch/master/roles/build-containers/vars/main.yaml"
@@ -294,6 +295,10 @@ def expand_config(self, config):
         config['build_method'] = \
             config.get('build_method', self.defaults['build_method']).lower()
 
+        config['container_preffix'] = \
+            config.get('container_preffix',
+                       self.defaults['container_preffix']).lower()
+
         config['distro'] = "{}{}".format(config['distro_name'],
                                          config['distro_version'])
         config['latest_hashes_count'] = \

ci-scripts/dlrnapi_promoter/repo_client.py

@@ -31,6 +31,7 @@ def __init__(self, config):
             config.containers_list_exclude_config
         self.release = config.release
         self.build_method = config.build_method
+        self.container_preffix = config.container_preffix
 
     def get_versions_csv(self, dlrn_hash, candidate_label):
         """
@@ -119,13 +120,11 @@ def get_containers_list(self, tripleo_common_commit, load_excludes=True):
         # Load the yaml content for further parsing
         container_list = yaml.safe_load(containers_content)
 
-        # construct container_name_preffix
+        # container_preffix is loaded from config
         # In container-images/tripleo_containers.yaml, the containers
-        # are named openstack-*.
+        # are named openstack-* for master/victoria onwards.
         # In container-images/overcloud_containers.yaml, the containers
-        # are named distro-binary-*.
-        container_preffix = {'kolla': 'binary-',
-                             'tripleo': 'openstack-'}
+        # are named distro-binary-* for older branches.
 
         if container_list:
             if 'container_images' in container_list:
@@ -156,20 +155,20 @@ def get_containers_list(self, tripleo_common_commit, load_excludes=True):
                     ]
 
                 # filter imagename based on image_source
-                # for imagename and image_source: kolla:
+                # for imagename and release up to ussuri
                 # docker.io/tripleomaster/centos-binary-aodh-api:current-tripleo
                 # We need to get aodh-api as a container name by striping with
                 # '/' and spliting -binary from the imagename
                 #
-                # for imagename and image_source: tripleo
+                # for imagename and release: master/victoria onwards
                 # quay.io/tripleomaster/openstack-base:current-tripleo
                 # we need to get tempest as a container name by striping with
                 # '/' and splitting -openstack from the imagename
 
                 full_list = [
-                    i.split(container_preffix[self.build_method])[-1]
+                    i.split(self.container_preffix)[-1]
                     for i in full_list
-                    if container_preffix[self.build_method] in i
+                    if self.container_preffix in i
                 ]
         else:
             full_list = []

ci-scripts/dlrnapi_promoter/test_config_legacy_unit.py

@@ -380,12 +380,13 @@ def test_expand_config_all_defaults(self, get_api_url_mock):
             'containers_list_base_url':
                 'https://opendev.org/openstack/tripleo-common/raw/commit/',
             'containers_list_path':
-                'container-images/overcloud_containers.yaml',
+                'container-images/tripleo_containers.yaml',
             "containers_list_exclude_config": (
                 "https://opendev.org/openstack/"
                 "tripleo-ci/raw/branch/master/roles/build-containers/vars"
                 "/main.yaml"),
-            'build_method': 'kolla',
+            'build_method': 'tripleo',
+            'container_preffix': 'openstack-',
             'distro': 'centos7',
             'distro_name': 'centos',
             'distro_version': '7',

ci-scripts/dlrnapi_promoter/test_repo_client_unit.py

@@ -48,7 +48,8 @@ def setUp(self):
         config = type("Config", (), {
             'repo_url': repo_url,
             'release': 'master',
-            'build_method': 'kolla',
+            'build_method': 'tripleo',
+            'container_preffix': config_defaults['container_preffix'],
             'containers_list_base_url': containers_list_base_url,
             'containers_list_path': config_defaults['containers_list_path'],
             'containers_list_exclude_config': "file://{}".format(
@@ -106,47 +107,63 @@ def setUp(self):
         containers_dir = os.path.join(self.temp_dir,
                                       self.versions_csv_rows[1]['Source Sha'],
                                       containers_file_dirname)
-        # containers names coming from overcloud_containers.yaml file
-        containers_list = """
+
+        # containers names coming from tripleo yaml file
+        tripleo_containers_list = """
 container_images:
-- image_source: kolla
-  imagename: quay.io/tripleomaster/centos-binary-nova-api:current-tripleo
-- image_source: kolla
-  imagename: quay.io/tripleomaster/centos-binary-neutron-server:current-tripleo
-- image_source: kolla
-  imagename: quay.io/tripleomaster/centos-binary-excluded:current-tripleo
-- image_source: kolla
-  imagename: quay.io/tripleomaster/centos-binary-ovn-controller:current-tripleo
+- image_source: tripleo
+  imagename: quay.io/tripleomaster/openstack-base:current-tripleo
+- image_source: tripleo
+  imagename: quay.io/tripleomaster/openstack-os:current-tripleo
+- image_source: tripleo
+  imagename: quay.io/tripleomaster/openstack-aodh-base:current-tripleo
 """
         os.makedirs(containers_dir)
         containers_file_path = \
             os.path.join(containers_dir,
                          os.path.basename(config_defaults[
                                               'containers_list_path']))
         with open(containers_file_path, "w") as containers_file:
-            containers_file.write(containers_list)
+            containers_file.write(tripleo_containers_list)
 
-        # containers names coming from tripleo yaml file
+        # containers names coming from overcloud_containers.yaml file
+        overcloud_containers_list = """
+container_images:
+- image_source: kolla
+  imagename: quay.io/tripleotrain/centos-binary-nova-api:current-tripleo
+- image_source: kolla
+  imagename: quay.io/tripleotrain/centos-binary-neutron-server:current-tripleo
+- image_source: kolla
+  imagename: quay.io/tripleotrain/centos-binary-excluded:current-tripleo
+- image_source: kolla
+  imagename: quay.io/tripleotrain/centos-binary-ovn-controller:current-tripleo
+"""
+        overcloud_containers_file_path = \
+            os.path.join(containers_dir, 'overcloud_containers.yaml')
+        with open(overcloud_containers_file_path, "w") as containers_file:
+            containers_file.write(overcloud_containers_list)
+
+        # containers names coming from yaml file for ussuri release
         tripleo_containers_list = """
 container_images:
 - image_source: tripleo
-  imagename: quay.io/tripleomaster/openstack-base:current-tripleo
+  imagename: quay.io/tripleou/centos-binary-base:current-tripleo
 - image_source: tripleo
-  imagename: quay.io/tripleomaster/openstack-os:current-tripleo
+  imagename: quay.io/tripleou/centos-binary-os:current-tripleo
 - image_source: tripleo
-  imagename: quay.io/tripleomaster/openstack-aodh-base:current-tripleo
+  imagename: quay.io/tripleou/centos-binary-aodh-base:current-tripleo
 """
         tripleo_containers_file_path = \
-            os.path.join(containers_dir, 'tripleo_containers.yaml')
+            os.path.join(containers_dir, 'ussuri_containers.yaml')
         with open(tripleo_containers_file_path, "w") as containers_file:
             containers_file.write(tripleo_containers_list)
 
         # containers names coming from yaml file for queens release
         tripleo_containers_list = """
 container_images:
-- imagename: quay.io/tripleomaster/centos-binary-base:current-tripleo
-- imagename: quay.io/tripleomaster/centos-binary-os:current-tripleo
-- imagename: quay.io/tripleomaster/centos-binary-aodh-base:current-tripleo
+- imagename: quay.io/tripleoqueens/centos-binary-base:current-tripleo
+- imagename: quay.io/tripleoqueens/centos-binary-os:current-tripleo
+- imagename: quay.io/tripleoqueens/centos-binary-aodh-base:current-tripleo
 """
         overcloud_containers_file_path = \
             os.path.join(containers_dir, 'queens_containers.yaml')
@@ -159,6 +176,7 @@ def setUp(self):
         exclude_config = {
             'exclude_containers': {
                 'master': excluded_containers,
+                'train': excluded_containers,
             },
         }
         with open(containers_list_exclude_config_path, "w") as exclude_file:
@@ -297,7 +315,11 @@ def test_get_commit_sha_failure(self,
     def test_get_containers_list_overcloud(self,
                                            mock_log_debug,
                                            mock_log_error):
-        self.client.build_method = "kolla"
+        self.client.release = "train"
+        self.client.container_preffix = "centos-binary-"
+        self.client.containers_list_path = (
+                'container-images/overcloud_containers.yaml'
+        )
         containers_list = self.client.get_containers_list(
             self.versions_csv_rows[1]['Source Sha'])
         self.assertEqual(containers_list, ['nova-api', 'neutron-server',
@@ -315,9 +337,6 @@ def test_get_containers_list_tripleo(self,
                                          mock_log_error):
         self.client.build_method = "tripleo"
         self.client.release = "foobar"
-        self.client.containers_list_path = (
-                'container-images/tripleo_containers.yaml'
-        )
         containers_list = self.client.get_containers_list(
             self.versions_csv_rows[1]['Source Sha'])
         self.assertEqual(containers_list, ['base', 'os', 'aodh-base'])
@@ -333,6 +352,8 @@ def test_get_containers_list_queens(self,
                                         mock_log_debug,
                                         mock_log_error):
         self.client.release = "queens"
+        self.client.build_method = "kolla"
+        self.client.container_preffix = "centos-binary-"
         self.client.containers_list_path = (
                 'container-images/queens_containers.yaml'
         )
@@ -350,6 +371,11 @@ def test_get_containers_list_queens(self,
     def test_get_containers_list_ok_no_excludes(self,
                                                 mock_log_debug,
                                                 mock_log_error):
+        self.client.release = "stein"
+        self.client.container_preffix = "centos-binary-"
+        self.client.containers_list_path = (
+                'container-images/overcloud_containers.yaml'
+        )
         containers_list = self.client.get_containers_list(
             self.versions_csv_rows[1]['Source Sha'], load_excludes=False)
         self.assertEqual(containers_list, ['nova-api', 'neutron-server',
@@ -468,3 +494,22 @@ def test_load_excludes_distro_not_found(self,
         self.assertFalse(mock_log_info.called)
         self.assertFalse(mock_log_debug.called)
         self.assertFalse(mock_log_exception.called)
+
+    @patch('logging.Logger.error')
+    @patch('logging.Logger.debug')
+    def test_get_containers_list_ussuri(self,
+                                        mock_log_debug,
+                                        mock_log_error):
+        self.client.release = "ussuri"
+        self.client.container_preffix = "centos-binary-"
+        self.client.containers_list_path = (
+                'container-images/ussuri_containers.yaml'
+        )
+        containers_list = self.client.get_containers_list(
+            self.versions_csv_rows[1]['Source Sha'])
+        mock_log_debug.assert_has_calls([
+            mock.call("Attempting Download of containers template at %s",
+                      mock.ANY)
+        ])
+        self.assertEqual(containers_list, ['base', 'os', 'aodh-base'])
+        mock_log_error.assert_not_called()