From d6f46d544fd757fdc811d738422f85e64a6dc09b Mon Sep 17 00:00:00 2001 From: Marcin Juszkiewicz Date: Mon, 14 Oct 2019 04:36:48 -0700 Subject: [PATCH 1/7] handle push error properly If there was some error during pushing then Kolla was greeting with "all went fine" message anyway: INFO:kolla.common.utils.aodh-api:Trying to push the image ERROR:kolla.common.utils.aodh-api:Get http://10.101.16.1:5000/v2/: dial tcp 10.101.16.1:5000: connect: no route to host INFO:kolla.common.utils.aodh-api:Pushed successfully This patch changes that. Now if there is an error during push then proper exception is raised to PushTask and image is marked with PUSH_ERROR status. This way at the end of build it is easy to spot which images did not got pushed to registry: INFO:kolla.common.utils:=========================== INFO:kolla.common.utils:Images that failed to build INFO:kolla.common.utils:=========================== ERROR:kolla.common.utils:base Failed with status: push_error ERROR:kolla.common.utils:nova-api Failed with status: push_error ERROR:kolla.common.utils:nova-base Failed with status: push_error ERROR:kolla.common.utils:nova-compute Failed with status: push_error ERROR:kolla.common.utils:nova-compute-ironic Failed with status: push_error ERROR:kolla.common.utils:nova-conductor Failed with status: push_error Closes-Bug: #1848019 Change-Id: Id2ab97bf4c0dc3423268a0ea435b56f4a65f7196 (cherry picked from commit 52cac09d3d0b8dcdbc8d68a0bcc8092008220309) --- kolla/image/build.py | 11 +++++++++-- kolla/tests/test_build.py | 40 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/kolla/image/build.py b/kolla/image/build.py index 2e88061049..5fd2cde0ce 100755 --- a/kolla/image/build.py +++ b/kolla/image/build.py @@ -317,6 +317,11 @@ def run(self): self.success = True +class PushError(Exception): + """Raised when there is a problem with pushing image to repository.""" + pass + + class PushTask(DockerTask): """Task that pushes an image to a docker repository.""" @@ -340,6 +345,9 @@ def run(self): ' have the correct privileges to run Docker' ' (root)') image.status = STATUS_CONNECTION_ERROR + except PushError as exception: + self.logger.error(exception) + image.status = STATUS_PUSH_ERROR except Exception: self.logger.exception('Unknown error when pushing') image.status = STATUS_PUSH_ERROR @@ -364,8 +372,7 @@ def push_image(self, image): if 'stream' in response: self.logger.info(response['stream']) elif 'errorDetail' in response: - image.status = STATUS_ERROR - self.logger.error(response['errorDetail']['message']) + raise PushError(response['errorDetail']['message']) # Reset any previous errors. image.status = STATUS_BUILT diff --git a/kolla/tests/test_build.py b/kolla/tests/test_build.py index b0ce009a9a..f9b2deab0b 100644 --- a/kolla/tests/test_build.py +++ b/kolla/tests/test_build.py @@ -83,6 +83,7 @@ def test_push_image(self, mock_client): @mock.patch.dict(os.environ, clear=True) @mock.patch('docker.APIClient') def test_push_image_failure(self, mock_client): + """failure on connecting Docker API""" self.dc = mock_client mock_client().push.side_effect = Exception pusher = build.PushTask(self.conf, self.image) @@ -96,6 +97,7 @@ def test_push_image_failure(self, mock_client): @mock.patch.dict(os.environ, clear=True) @mock.patch('docker.APIClient') def test_push_image_failure_retry(self, mock_client): + """failure on connecting Docker API, success on retry""" self.dc = mock_client mock_client().push.side_effect = [Exception, []] pusher = build.PushTask(self.conf, self.image) @@ -112,6 +114,44 @@ def test_push_image_failure_retry(self, mock_client): self.assertTrue(pusher.success) self.assertEqual(build.STATUS_BUILT, self.image.status) + @mock.patch('docker.version', '3.0.0') + @mock.patch.dict(os.environ, clear=True) + @mock.patch('docker.APIClient') + def test_push_image_failure_error(self, mock_client): + """Docker connected, failure to push""" + self.dc = mock_client + mock_client().push.return_value = [{'errorDetail': {'message': + 'mock push fail'}}] + pusher = build.PushTask(self.conf, self.image) + pusher.run() + mock_client().push.assert_called_once_with( + self.image.canonical_name, decode=True, stream=True) + self.assertFalse(pusher.success) + self.assertEqual(build.STATUS_PUSH_ERROR, self.image.status) + + @mock.patch('docker.version', '3.0.0') + @mock.patch.dict(os.environ, clear=True) + @mock.patch('docker.APIClient') + def test_push_image_failure_error_retry(self, mock_client): + """Docker connected, failure to push, success on retry""" + self.dc = mock_client + mock_client().push.return_value = [{'errorDetail': {'message': + 'mock push fail'}}] + pusher = build.PushTask(self.conf, self.image) + pusher.run() + mock_client().push.assert_called_once_with( + self.image.canonical_name, decode=True, stream=True) + self.assertFalse(pusher.success) + self.assertEqual(build.STATUS_PUSH_ERROR, self.image.status) + + # Try again, this time without exception. + mock_client().push.return_value = [{'stream': 'mock push passes'}] + pusher.reset() + pusher.run() + self.assertEqual(2, mock_client().push.call_count) + self.assertTrue(pusher.success) + self.assertEqual(build.STATUS_BUILT, self.image.status) + @mock.patch.dict(os.environ, clear=True) @mock.patch('docker.APIClient') def test_build_image(self, mock_client): From 0f6980d851a92a6b740a590d0c1229384cc763b0 Mon Sep 17 00:00:00 2001 From: Keith Plant Date: Fri, 18 Oct 2019 08:19:39 -0400 Subject: [PATCH 2/7] Bump Openstack Exporter version to 0.6.0 There appears to be a bug in the prometheus openstack exporter versions <= 0.2.1 causing cinder data to stop being collected after about 24 hours. Bumping to latest stable version 0.6.0. Change-Id: Iccaf4c0bb79e7e33072e37aa83711b37938d6719 Signed-off-by: Keith Plant (cherry picked from commit cd7f78e8f87b194633b60bcce89852f701a77c90) --- docker/prometheus/prometheus-openstack-exporter/Dockerfile.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/prometheus/prometheus-openstack-exporter/Dockerfile.j2 b/docker/prometheus/prometheus-openstack-exporter/Dockerfile.j2 index ff5d55e130..9a5e17c94c 100644 --- a/docker/prometheus/prometheus-openstack-exporter/Dockerfile.j2 +++ b/docker/prometheus/prometheus-openstack-exporter/Dockerfile.j2 @@ -4,7 +4,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% block prometheus_openstack_exporter_header %}{% endblock %} {% block prometheus_openstack_exporter_repository_version %} -ENV prometheus_openstack_exporter_version=0.2.1 +ENV prometheus_openstack_exporter_version=0.6.0 {% endblock %} {% block prometheus_openstack_exporter_install %} From d6d5790a59e8e5147278054940f362ba0f731bf0 Mon Sep 17 00:00:00 2001 From: Dincer Celik Date: Mon, 21 Oct 2019 17:26:54 +0300 Subject: [PATCH 3/7] Fix python3 compatibility for barbican Barbican refers to python3 but uwsgi refers to python2. This change fixes this issue. Change-Id: I508bb082a319955b3e008235e158eaa07a1e8a74 Closes-Bug: #1849128 (cherry picked from commit 823e45a6c909437a6530c9f34f9e4f5cce8ab79c) --- docker/barbican/barbican-api/Dockerfile.j2 | 2 +- docker/barbican/barbican-base/Dockerfile.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/barbican/barbican-api/Dockerfile.j2 b/docker/barbican/barbican-api/Dockerfile.j2 index e25e6fb058..a2bc60e46d 100644 --- a/docker/barbican/barbican-api/Dockerfile.j2 +++ b/docker/barbican/barbican-api/Dockerfile.j2 @@ -35,7 +35,7 @@ RUN sed -i -r 's,^(Listen 80),#\1,' /etc/httpd/conf/httpd.conf \ 'apache2', 'barbican-api', 'libapache2-mod-wsgi-py3', - 'uwsgi-plugin-python' + 'uwsgi-plugin-python3' ] %} {{ macros.install_packages(barbican_api_packages | customizable("packages")) }} diff --git a/docker/barbican/barbican-base/Dockerfile.j2 b/docker/barbican/barbican-base/Dockerfile.j2 index 4d6843d654..0cf9821c90 100644 --- a/docker/barbican/barbican-base/Dockerfile.j2 +++ b/docker/barbican/barbican-base/Dockerfile.j2 @@ -25,7 +25,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% endif %} {% elif base_package_type == 'deb' %} - {% set barbican_base_packages = ['uwsgi-plugin-python'] %} + {% set barbican_base_packages = ['uwsgi-plugin-python3'] %} {% endif %} {{ macros.install_packages(barbican_base_packages | customizable("packages")) }} From ff815baf44b0ed4348b4c9c63eef704e05e10c37 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 21 Oct 2019 15:48:01 +0100 Subject: [PATCH 4/7] CI: Increase job run attempts to 5 Attempts affect pre failures. This means we can increase stability of jobs by rejecting nodes that fail pre without failing runs at the same time (unless we are really unlucky and hit b0rken nodes 5 times in a row). This change is adapted from I17b7f878c742fa8db66f738526855a02ab9f1905 in kolla-ansible. Change-Id: Ied5d2cb8ffa47b90833ee3cf241797601906f9b2 (cherry picked from commit 98fb7dc8ff2136d658c9a79ed40f6e42e7893662) --- .zuul.d/base.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.zuul.d/base.yaml b/.zuul.d/base.yaml index 34a0a28479..8847eb55c9 100644 --- a/.zuul.d/base.yaml +++ b/.zuul.d/base.yaml @@ -45,7 +45,7 @@ pre-run: tests/playbooks/pre.yml run: tests/playbooks/run.yml post-run: tests/playbooks/post.yml - attempts: 1 + attempts: 5 irrelevant-files: - ^.*\.rst$ - ^doc/.* From d65eab1eaf7a86774ac0b2f3c98646b85fd0e641 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 23 Oct 2019 10:56:55 +0100 Subject: [PATCH 5/7] Revert to uwsgi-plugin-python for barbican source images Stein only I508bb082a319955b3e008235e158eaa07a1e8a74 was backported to stein to fix an issue with barbican images which contained the python 2 version of uwsgi-plugin-python on Ubuntu/Debian, but should have used uwsgi-plugin-python3. However, in Stein only Ubuntu/Debian binary images used python 3 - not source images. This change reverts to uwsgi-plugin-python for Ubuntu/Debian source. Change-Id: I62c1a42c29ce7273028f99edbb77f4362a2452ce Related-Bug: #1849128 --- docker/barbican/barbican-base/Dockerfile.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/barbican/barbican-base/Dockerfile.j2 b/docker/barbican/barbican-base/Dockerfile.j2 index 0cf9821c90..4d6843d654 100644 --- a/docker/barbican/barbican-base/Dockerfile.j2 +++ b/docker/barbican/barbican-base/Dockerfile.j2 @@ -25,7 +25,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% endif %} {% elif base_package_type == 'deb' %} - {% set barbican_base_packages = ['uwsgi-plugin-python3'] %} + {% set barbican_base_packages = ['uwsgi-plugin-python'] %} {% endif %} {{ macros.install_packages(barbican_base_packages | customizable("packages")) }} From 7115cde259817a325e1585b6e87ebd79c7cd8f7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Piliszek?= Date: Mon, 28 Oct 2019 09:39:30 +0100 Subject: [PATCH 6/7] Fix centos qdrouterd - use RDO qpid packages only Disables other sources with qpid packages. Change-Id: I508b8258e0d0aacfa0e4ff56706b5f273d2fbf0a Closes-bug: #1850044 (cherry picked from commit e41e86c805167b9ab2f5c6217f8f8ad5cf773f52) --- docker/qdrouterd/Dockerfile.j2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/qdrouterd/Dockerfile.j2 b/docker/qdrouterd/Dockerfile.j2 index d9071cdefe..19ce0755b7 100644 --- a/docker/qdrouterd/Dockerfile.j2 +++ b/docker/qdrouterd/Dockerfile.j2 @@ -16,6 +16,9 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build 'qpid-dispatch-tools' ] %} +# make sure qpid is pulled from centos-openstack-RELEASE (RDO) repo +RUN yum-config-manager --disable epel centos-release-opstools extras + {{ macros.install_packages(qdrouterd_packages | customizable("packages")) }} {% elif base_package_type == 'deb' %} From 5a08221cc4270dec3af5ff613731cc7be7576124 Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Mon, 28 Oct 2019 10:07:20 +0000 Subject: [PATCH 7/7] Change ODL repo to nexus.opendaylight.org Change-Id: I242945edba84b891ccbf82d5fcce8ef5fe37ad3d (cherry picked from commit 939a56035b8f15b74dff2d68154963735b0f73b4) --- docker/base/opendaylight.repo | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/base/opendaylight.repo b/docker/base/opendaylight.repo index 44c1eb26d5..c1e8798c9e 100644 --- a/docker/base/opendaylight.repo +++ b/docker/base/opendaylight.repo @@ -1,6 +1,5 @@ [opendaylight] -name=CentOS CBS OpenDaylight Release Repository -# opendaylight package is not signed, so download from HTTPS source at least -baseurl=https://cbs.centos.org/repos/nfv7-opendaylight-6-release/x86_64/os/ +name=OpenDaylight Carbon +baseurl=https://nexus.opendaylight.org/content/repositories/opendaylight-carbon-epel-7-x86_64-devel/ enabled=1 gpgcheck=0