From c31fd90b57a630f75cd668c8bc5d7774c0b253aa Mon Sep 17 00:00:00 2001 From: Rafael Folco Date: Wed, 26 Aug 2020 18:24:40 -0300 Subject: [PATCH] openstack-* containers in dlrnapi_promoter openstack-* is the prefix for master/victoria onwards. For older branches, distro-binary-* is the prefix. This change updates the default settings in legacy config and all affected tests as it is the current workflow in production (new config engine is WIP). Change-Id: Ie808f3abf5b5b3d4a764fb1ba6a1f67b3756a4d1 --- .../config/CentOS-7/master.ini | 2 + .../config/CentOS-7/queens.ini | 2 + .../config/CentOS-7/rocky.ini | 2 + .../config/CentOS-7/stein.ini | 2 + .../config/CentOS-7/train.ini | 2 + .../config/CentOS-8/master.ini | 3 +- .../config/CentOS-8/train.ini | 2 + .../config/CentOS-8/ussuri.ini | 1 + .../config/staging/CentOS-8/master.ini | 1 + ci-scripts/dlrnapi_promoter/config_legacy.py | 9 +- ci-scripts/dlrnapi_promoter/repo_client.py | 17 ++-- .../test_config_legacy_unit.py | 5 +- .../dlrnapi_promoter/test_repo_client_unit.py | 93 ++++++++++++++----- 13 files changed, 103 insertions(+), 38 deletions(-) diff --git a/ci-scripts/dlrnapi_promoter/config/CentOS-7/master.ini b/ci-scripts/dlrnapi_promoter/config/CentOS-7/master.ini index 05ec1be94..96e7dea09 100644 --- a/ci-scripts/dlrnapi_promoter/config/CentOS-7/master.ini +++ b/ci-scripts/dlrnapi_promoter/config/CentOS-7/master.ini @@ -4,6 +4,8 @@ distro_version: 7 release: master api_url: https://trunk.rdoproject.org/api-centos-master-uc username: ciuser +build_method: tripleo +container_preffix: openstack- # the password should be set as DLRNAPI_PASSWORD in the environment dry_run: no log_file: ~/web/promoter_logs/centos7_master.log diff --git a/ci-scripts/dlrnapi_promoter/config/CentOS-7/queens.ini b/ci-scripts/dlrnapi_promoter/config/CentOS-7/queens.ini index eb92a8490..8fb0ab933 100644 --- a/ci-scripts/dlrnapi_promoter/config/CentOS-7/queens.ini +++ b/ci-scripts/dlrnapi_promoter/config/CentOS-7/queens.ini @@ -5,6 +5,8 @@ release: queens api_url: https://trunk.rdoproject.org/api-centos-queens base_url: https://trunk.rdoproject.org/centos7-queens/ username: ciuser +build_method: kolla +container_preffix: centos-binary- # the password should be set as DLRNAPI_PASSWORD in the environment dry_run: no log_file: ~/web/promoter_logs/centos7_queens.log diff --git a/ci-scripts/dlrnapi_promoter/config/CentOS-7/rocky.ini b/ci-scripts/dlrnapi_promoter/config/CentOS-7/rocky.ini index c6ef80964..669b011f8 100644 --- a/ci-scripts/dlrnapi_promoter/config/CentOS-7/rocky.ini +++ b/ci-scripts/dlrnapi_promoter/config/CentOS-7/rocky.ini @@ -5,6 +5,8 @@ release: rocky api_url: https://trunk.rdoproject.org/api-centos-rocky base_url: https://trunk.rdoproject.org/centos7-rocky/ username: ciuser +build_method: kolla +container_preffix: centos-binary- # the password should be set as DLRNAPI_PASSWORD in the environment dry_run: no log_file: ~/web/promoter_logs/centos7_rocky.log diff --git a/ci-scripts/dlrnapi_promoter/config/CentOS-7/stein.ini b/ci-scripts/dlrnapi_promoter/config/CentOS-7/stein.ini index c83f09c02..0f3659216 100644 --- a/ci-scripts/dlrnapi_promoter/config/CentOS-7/stein.ini +++ b/ci-scripts/dlrnapi_promoter/config/CentOS-7/stein.ini @@ -5,6 +5,8 @@ release: stein api_url: https://trunk.rdoproject.org/api-centos-stein base_url: https://trunk.rdoproject.org/centos7-stein/ username: ciuser +build_method: kolla +container_preffix: centos-binary- # the password should be set as DLRNAPI_PASSWORD in the environment dry_run: no log_file: ~/web/promoter_logs/centos7_stein.log diff --git a/ci-scripts/dlrnapi_promoter/config/CentOS-7/train.ini b/ci-scripts/dlrnapi_promoter/config/CentOS-7/train.ini index 2a6dcbbf2..91ca7d6c2 100644 --- a/ci-scripts/dlrnapi_promoter/config/CentOS-7/train.ini +++ b/ci-scripts/dlrnapi_promoter/config/CentOS-7/train.ini @@ -5,6 +5,8 @@ release: train api_url: https://trunk.rdoproject.org/api-centos-train base_url: https://trunk.rdoproject.org/centos7-train/ username: ciuser +build_method: kolla +container_preffix: centos-binary- # the password should be set as DLRNAPI_PASSWORD in the environment dry_run: no log_file: ~/web/promoter_logs/centos7_train.log diff --git a/ci-scripts/dlrnapi_promoter/config/CentOS-8/master.ini b/ci-scripts/dlrnapi_promoter/config/CentOS-8/master.ini index 9e4fab774..4e2c9160e 100644 --- a/ci-scripts/dlrnapi_promoter/config/CentOS-8/master.ini +++ b/ci-scripts/dlrnapi_promoter/config/CentOS-8/master.ini @@ -5,11 +5,12 @@ release: master api_url: https://trunk.rdoproject.org/api-centos8-master-uc base_url: https://trunk.rdoproject.org/centos8-master/ username: ciuser +build_method: tripleo +container_preffix: openstack- # the password should be set as DLRNAPI_PASSWORD in the environment dry_run: no log_file: ~/web/promoter_logs/centos8_master.log containers_list_path: container-images/tripleo_containers.yaml -build_method: tripleo latest_hashes_count: 10 manifest_push: false diff --git a/ci-scripts/dlrnapi_promoter/config/CentOS-8/train.ini b/ci-scripts/dlrnapi_promoter/config/CentOS-8/train.ini index fd479ef74..a50d9e8a1 100644 --- a/ci-scripts/dlrnapi_promoter/config/CentOS-8/train.ini +++ b/ci-scripts/dlrnapi_promoter/config/CentOS-8/train.ini @@ -5,6 +5,8 @@ release: train api_url: https://trunk.rdoproject.org/api-centos8-train base_url: https://trunk.rdoproject.org/centos8-train/ username: ciuser +build_method: kolla +container_preffix: centos-binary- # the password should be set as DLRNAPI_PASSWORD in the environment dry_run: no log_file: ~/web/promoter_logs/centos8_train.log diff --git a/ci-scripts/dlrnapi_promoter/config/CentOS-8/ussuri.ini b/ci-scripts/dlrnapi_promoter/config/CentOS-8/ussuri.ini index 76f7f4cf5..6b3728c28 100644 --- a/ci-scripts/dlrnapi_promoter/config/CentOS-8/ussuri.ini +++ b/ci-scripts/dlrnapi_promoter/config/CentOS-8/ussuri.ini @@ -9,6 +9,7 @@ username: ciuser dry_run: no containers_list_path: container-images/tripleo_containers.yaml build_method: tripleo +container_preffix: centos-binary- log_file: ~/web/promoter_logs/centos8_ussuri.log latest_hashes_count: 10 manifest_push: false diff --git a/ci-scripts/dlrnapi_promoter/config/staging/CentOS-8/master.ini b/ci-scripts/dlrnapi_promoter/config/staging/CentOS-8/master.ini index 2cd462ab7..1fe44a0e8 100644 --- a/ci-scripts/dlrnapi_promoter/config/staging/CentOS-8/master.ini +++ b/ci-scripts/dlrnapi_promoter/config/staging/CentOS-8/master.ini @@ -9,6 +9,7 @@ containers_list_path: container-images/tripleo_containers.yaml containers_list_exclude_config: file:///tmp/promoter-staging/containers/build-containers-main.yaml username: ciuser build_method: tripleo +container_preffix: openstack- # the password should be set as DLRNAPI_PASSWORD in the environment dry_run: no log_file: ~/web/promoter_logs/centos8_master.log diff --git a/ci-scripts/dlrnapi_promoter/config_legacy.py b/ci-scripts/dlrnapi_promoter/config_legacy.py index 5c54a09a2..589b3027e 100644 --- a/ci-scripts/dlrnapi_promoter/config_legacy.py +++ b/ci-scripts/dlrnapi_promoter/config_legacy.py @@ -44,14 +44,15 @@ class PromoterLegacyConfigBase(object): 'allowed_clients': 'registries_client,qcow_client,dlrn_client', 'log_level': "INFO", "dlrn_api_host": "trunk.rdoproject.org", - "build_method": "kolla", + "build_method": "tripleo", + "container_preffix": "openstack-", "containers_list_base_url": ("https://opendev.org/openstack/" "tripleo-common/raw/commit/"), # For old container kolla based workflow, the source of truth of # containers is overcloud_containers.yaml. # For new container (non-kolla) based workflow, the source of truth # is tripleo_containers.yaml. - "containers_list_path": "container-images/overcloud_containers.yaml", + "containers_list_path": "container-images/tripleo_containers.yaml", "containers_list_exclude_config": ( "https://opendev.org/openstack/" "tripleo-ci/raw/branch/master/roles/build-containers/vars/main.yaml" @@ -294,6 +295,10 @@ def expand_config(self, config): config['build_method'] = \ config.get('build_method', self.defaults['build_method']).lower() + config['container_preffix'] = \ + config.get('container_preffix', + self.defaults['container_preffix']).lower() + config['distro'] = "{}{}".format(config['distro_name'], config['distro_version']) config['latest_hashes_count'] = \ diff --git a/ci-scripts/dlrnapi_promoter/repo_client.py b/ci-scripts/dlrnapi_promoter/repo_client.py index 8925ca480..cccf20fc4 100644 --- a/ci-scripts/dlrnapi_promoter/repo_client.py +++ b/ci-scripts/dlrnapi_promoter/repo_client.py @@ -31,6 +31,7 @@ def __init__(self, config): config.containers_list_exclude_config self.release = config.release self.build_method = config.build_method + self.container_preffix = config.container_preffix def get_versions_csv(self, dlrn_hash, candidate_label): """ @@ -119,13 +120,11 @@ def get_containers_list(self, tripleo_common_commit, load_excludes=True): # Load the yaml content for further parsing container_list = yaml.safe_load(containers_content) - # construct container_name_preffix + # container_preffix is loaded from config # In container-images/tripleo_containers.yaml, the containers - # are named openstack-*. + # are named openstack-* for master/victoria onwards. # In container-images/overcloud_containers.yaml, the containers - # are named distro-binary-*. - container_preffix = {'kolla': 'binary-', - 'tripleo': 'openstack-'} + # are named distro-binary-* for older branches. if container_list: if 'container_images' in container_list: @@ -156,20 +155,20 @@ def get_containers_list(self, tripleo_common_commit, load_excludes=True): ] # filter imagename based on image_source - # for imagename and image_source: kolla: + # for imagename and release up to ussuri # docker.io/tripleomaster/centos-binary-aodh-api:current-tripleo # We need to get aodh-api as a container name by striping with # '/' and spliting -binary from the imagename # - # for imagename and image_source: tripleo + # for imagename and release: master/victoria onwards # quay.io/tripleomaster/openstack-base:current-tripleo # we need to get tempest as a container name by striping with # '/' and splitting -openstack from the imagename full_list = [ - i.split(container_preffix[self.build_method])[-1] + i.split(self.container_preffix)[-1] for i in full_list - if container_preffix[self.build_method] in i + if self.container_preffix in i ] else: full_list = [] diff --git a/ci-scripts/dlrnapi_promoter/test_config_legacy_unit.py b/ci-scripts/dlrnapi_promoter/test_config_legacy_unit.py index 5b1d0b8e4..d09828844 100644 --- a/ci-scripts/dlrnapi_promoter/test_config_legacy_unit.py +++ b/ci-scripts/dlrnapi_promoter/test_config_legacy_unit.py @@ -380,12 +380,13 @@ def test_expand_config_all_defaults(self, get_api_url_mock): 'containers_list_base_url': 'https://opendev.org/openstack/tripleo-common/raw/commit/', 'containers_list_path': - 'container-images/overcloud_containers.yaml', + 'container-images/tripleo_containers.yaml', "containers_list_exclude_config": ( "https://opendev.org/openstack/" "tripleo-ci/raw/branch/master/roles/build-containers/vars" "/main.yaml"), - 'build_method': 'kolla', + 'build_method': 'tripleo', + 'container_preffix': 'openstack-', 'distro': 'centos7', 'distro_name': 'centos', 'distro_version': '7', diff --git a/ci-scripts/dlrnapi_promoter/test_repo_client_unit.py b/ci-scripts/dlrnapi_promoter/test_repo_client_unit.py index 0ae104422..4650fb49a 100644 --- a/ci-scripts/dlrnapi_promoter/test_repo_client_unit.py +++ b/ci-scripts/dlrnapi_promoter/test_repo_client_unit.py @@ -48,7 +48,8 @@ def setUp(self): config = type("Config", (), { 'repo_url': repo_url, 'release': 'master', - 'build_method': 'kolla', + 'build_method': 'tripleo', + 'container_preffix': config_defaults['container_preffix'], 'containers_list_base_url': containers_list_base_url, 'containers_list_path': config_defaults['containers_list_path'], 'containers_list_exclude_config': "file://{}".format( @@ -106,17 +107,16 @@ def setUp(self): containers_dir = os.path.join(self.temp_dir, self.versions_csv_rows[1]['Source Sha'], containers_file_dirname) - # containers names coming from overcloud_containers.yaml file - containers_list = """ + + # containers names coming from tripleo yaml file + tripleo_containers_list = """ container_images: -- image_source: kolla - imagename: quay.io/tripleomaster/centos-binary-nova-api:current-tripleo -- image_source: kolla - imagename: quay.io/tripleomaster/centos-binary-neutron-server:current-tripleo -- image_source: kolla - imagename: quay.io/tripleomaster/centos-binary-excluded:current-tripleo -- image_source: kolla - imagename: quay.io/tripleomaster/centos-binary-ovn-controller:current-tripleo +- image_source: tripleo + imagename: quay.io/tripleomaster/openstack-base:current-tripleo +- image_source: tripleo + imagename: quay.io/tripleomaster/openstack-os:current-tripleo +- image_source: tripleo + imagename: quay.io/tripleomaster/openstack-aodh-base:current-tripleo """ os.makedirs(containers_dir) containers_file_path = \ @@ -124,29 +124,46 @@ def setUp(self): os.path.basename(config_defaults[ 'containers_list_path'])) with open(containers_file_path, "w") as containers_file: - containers_file.write(containers_list) + containers_file.write(tripleo_containers_list) - # containers names coming from tripleo yaml file + # containers names coming from overcloud_containers.yaml file + overcloud_containers_list = """ +container_images: +- image_source: kolla + imagename: quay.io/tripleotrain/centos-binary-nova-api:current-tripleo +- image_source: kolla + imagename: quay.io/tripleotrain/centos-binary-neutron-server:current-tripleo +- image_source: kolla + imagename: quay.io/tripleotrain/centos-binary-excluded:current-tripleo +- image_source: kolla + imagename: quay.io/tripleotrain/centos-binary-ovn-controller:current-tripleo +""" + overcloud_containers_file_path = \ + os.path.join(containers_dir, 'overcloud_containers.yaml') + with open(overcloud_containers_file_path, "w") as containers_file: + containers_file.write(overcloud_containers_list) + + # containers names coming from yaml file for ussuri release tripleo_containers_list = """ container_images: - image_source: tripleo - imagename: quay.io/tripleomaster/openstack-base:current-tripleo + imagename: quay.io/tripleou/centos-binary-base:current-tripleo - image_source: tripleo - imagename: quay.io/tripleomaster/openstack-os:current-tripleo + imagename: quay.io/tripleou/centos-binary-os:current-tripleo - image_source: tripleo - imagename: quay.io/tripleomaster/openstack-aodh-base:current-tripleo + imagename: quay.io/tripleou/centos-binary-aodh-base:current-tripleo """ tripleo_containers_file_path = \ - os.path.join(containers_dir, 'tripleo_containers.yaml') + os.path.join(containers_dir, 'ussuri_containers.yaml') with open(tripleo_containers_file_path, "w") as containers_file: containers_file.write(tripleo_containers_list) # containers names coming from yaml file for queens release tripleo_containers_list = """ container_images: -- imagename: quay.io/tripleomaster/centos-binary-base:current-tripleo -- imagename: quay.io/tripleomaster/centos-binary-os:current-tripleo -- imagename: quay.io/tripleomaster/centos-binary-aodh-base:current-tripleo +- imagename: quay.io/tripleoqueens/centos-binary-base:current-tripleo +- imagename: quay.io/tripleoqueens/centos-binary-os:current-tripleo +- imagename: quay.io/tripleoqueens/centos-binary-aodh-base:current-tripleo """ overcloud_containers_file_path = \ os.path.join(containers_dir, 'queens_containers.yaml') @@ -159,6 +176,7 @@ def setUp(self): exclude_config = { 'exclude_containers': { 'master': excluded_containers, + 'train': excluded_containers, }, } with open(containers_list_exclude_config_path, "w") as exclude_file: @@ -297,7 +315,11 @@ def test_get_commit_sha_failure(self, def test_get_containers_list_overcloud(self, mock_log_debug, mock_log_error): - self.client.build_method = "kolla" + self.client.release = "train" + self.client.container_preffix = "centos-binary-" + self.client.containers_list_path = ( + 'container-images/overcloud_containers.yaml' + ) containers_list = self.client.get_containers_list( self.versions_csv_rows[1]['Source Sha']) self.assertEqual(containers_list, ['nova-api', 'neutron-server', @@ -315,9 +337,6 @@ def test_get_containers_list_tripleo(self, mock_log_error): self.client.build_method = "tripleo" self.client.release = "foobar" - self.client.containers_list_path = ( - 'container-images/tripleo_containers.yaml' - ) containers_list = self.client.get_containers_list( self.versions_csv_rows[1]['Source Sha']) self.assertEqual(containers_list, ['base', 'os', 'aodh-base']) @@ -333,6 +352,8 @@ def test_get_containers_list_queens(self, mock_log_debug, mock_log_error): self.client.release = "queens" + self.client.build_method = "kolla" + self.client.container_preffix = "centos-binary-" self.client.containers_list_path = ( 'container-images/queens_containers.yaml' ) @@ -350,6 +371,11 @@ def test_get_containers_list_queens(self, def test_get_containers_list_ok_no_excludes(self, mock_log_debug, mock_log_error): + self.client.release = "stein" + self.client.container_preffix = "centos-binary-" + self.client.containers_list_path = ( + 'container-images/overcloud_containers.yaml' + ) containers_list = self.client.get_containers_list( self.versions_csv_rows[1]['Source Sha'], load_excludes=False) self.assertEqual(containers_list, ['nova-api', 'neutron-server', @@ -468,3 +494,22 @@ def test_load_excludes_distro_not_found(self, self.assertFalse(mock_log_info.called) self.assertFalse(mock_log_debug.called) self.assertFalse(mock_log_exception.called) + + @patch('logging.Logger.error') + @patch('logging.Logger.debug') + def test_get_containers_list_ussuri(self, + mock_log_debug, + mock_log_error): + self.client.release = "ussuri" + self.client.container_preffix = "centos-binary-" + self.client.containers_list_path = ( + 'container-images/ussuri_containers.yaml' + ) + containers_list = self.client.get_containers_list( + self.versions_csv_rows[1]['Source Sha']) + mock_log_debug.assert_has_calls([ + mock.call("Attempting Download of containers template at %s", + mock.ANY) + ]) + self.assertEqual(containers_list, ['base', 'os', 'aodh-base']) + mock_log_error.assert_not_called()