From bc2544b8e22f00d075b77b03ca42056e4998efce Mon Sep 17 00:00:00 2001 From: Marcin Juszkiewicz Date: Fri, 4 Feb 2022 12:34:32 +0100 Subject: [PATCH 01/23] pin out some package from Debian OpenStack Team repos Debian OpenStack Team backports many packages to get their stuff done. Sometimes something goes wrong: INFO:kolla.common.utils.kolla-toolbox:The following packages have unmet dependencies: INFO:kolla.common.utils.kolla-toolbox: git : Depends: git-man (< 1:2.30.2-.) but 1:2.34.1-1~bpo11+1 is to be installed INFO:kolla.common.utils.kolla-toolbox:E: Unable to correct problems, you have held broken packages. So let pin out problematic packages as we are fine with bullseye version of git. Change-Id: I8c740655c7f7803d9ab36dac79d30e35e65dd586 (cherry picked from commit 2553c1845c7c4cf04d720b820e4bb13519eca4c9) --- docker/base/apt_preferences.debian | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/base/apt_preferences.debian b/docker/base/apt_preferences.debian index 0ccc2e0167..62b5805054 100644 --- a/docker/base/apt_preferences.debian +++ b/docker/base/apt_preferences.debian @@ -22,3 +22,8 @@ Pin-Priority: -1000 Package: openvswitch* python3-openvswitch ovn-* Pin: release n=bullseye-backports Pin-Priority: 1024 + +# NOTE(hrw): Debian OpenStack Team repo has some backports we do not want +Package: git* +Pin: release n=bullseye +Pin-Priority: 1024 From 2b605d3b92a82e5c4054e3c92713db26b7a19d4f Mon Sep 17 00:00:00 2001 From: Doug Szumski Date: Fri, 8 Mar 2019 16:14:54 +0000 Subject: [PATCH 02/23] Add Prometheus libvirt exporter image This adds a libvirt exporter image for use with Prometheus. Partially-Implements: blueprint libvirt-exporter Change-Id: Ice2af99a323496d3821762851b663899d15f569a (cherry picked from commit afce8598ac098fdb2e11ebac587ef7b13a6d4ec8) --- .../prometheus-libvirt-exporter/Dockerfile.j2 | 45 +++++++++++++++++++ ...eus-libvirt-exporter-8d505dc8b74f8625.yaml | 4 ++ 2 files changed, 49 insertions(+) create mode 100644 docker/prometheus/prometheus-libvirt-exporter/Dockerfile.j2 create mode 100644 releasenotes/notes/add-prometheus-libvirt-exporter-8d505dc8b74f8625.yaml diff --git a/docker/prometheus/prometheus-libvirt-exporter/Dockerfile.j2 b/docker/prometheus/prometheus-libvirt-exporter/Dockerfile.j2 new file mode 100644 index 0000000000..bcdcc7a56c --- /dev/null +++ b/docker/prometheus/prometheus-libvirt-exporter/Dockerfile.j2 @@ -0,0 +1,45 @@ +FROM {{ namespace }}/{{ infra_image_prefix }}prometheus-base:{{ tag }} +{% block labels %} +LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build_date }}" +{% endblock %} + +{% import "macros.j2" as macros with context %} + +{% block prometheus_libvirt_exporter_header %}{% endblock %} + +{% if base_package_type == 'rpm' %} + {% set prometheus_libvirt_exporter_packages = [ + 'git', + 'go', + 'libvirt-devel', + ] %} +{% elif base_package_type == 'deb' %} + {% set prometheus_libvirt_exporter_packages = [ + 'build-essential', + 'git', + 'golang-go', + 'libvirt-dev', + 'pkg-config', + ] %} +{% endif %} + +{{ macros.install_packages(prometheus_libvirt_exporter_packages | customizable("packages")) }} + +{% block prometheus_libvirt_exporter_version %} +ARG prometheus_libvirt_exporter_version=2.3.0 +ARG prometheus_libvirt_exporter_path=github.com/AlexZzz/libvirt-exporter +{% endblock %} + +{% block prometheus_libvirt_exporter_install %} +ENV GOPATH=/build +RUN go mod init libvirt-exporter \ + && go get -v ${prometheus_libvirt_exporter_path}@${prometheus_libvirt_exporter_version} \ + && mv /build/bin/libvirt-exporter /opt \ + && rm -rf /build + +{% endblock %} + +{% block prometheus_libvirt_exporter_footer %}{% endblock %} +{% block footer %}{% endblock %} + +USER root diff --git a/releasenotes/notes/add-prometheus-libvirt-exporter-8d505dc8b74f8625.yaml b/releasenotes/notes/add-prometheus-libvirt-exporter-8d505dc8b74f8625.yaml new file mode 100644 index 0000000000..2f85a5b94d --- /dev/null +++ b/releasenotes/notes/add-prometheus-libvirt-exporter-8d505dc8b74f8625.yaml @@ -0,0 +1,4 @@ +--- +features: + - Added a container image for Prometheus libvirt exporter, to be used + for monitoring deployments which provide VMs with libvirt. From aa6286efecb508c31e2b4b11d92f9a9338dcccc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Piliszek?= Date: Tue, 21 Dec 2021 12:54:31 +0000 Subject: [PATCH 03/23] [CI] Test Ironic on Debian Depends-On: https://review.opendev.org/c/openstack/kolla-ansible/+/797190 Change-Id: I8ada844a2b034547c045248b324c1955ad4cf6ae (cherry picked from commit 02e8e76ae762663c898be4b866daea03abd19494) --- .zuul.d/debian.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.zuul.d/debian.yaml b/.zuul.d/debian.yaml index 146884ecd9..39e89be604 100644 --- a/.zuul.d/debian.yaml +++ b/.zuul.d/debian.yaml @@ -20,6 +20,8 @@ experimental: jobs: - kolla-build-no-infra-wheels-debian-source + - kolla-ansible-debian-source-ironic: + files: ^docker\/(base|dnsmasq|ironic|ironic-inspector|iscsid|openstack-base)\/.* - job: name: kolla-build-debian-source From c80522274302c9a5eca0f5b70345cd7ea17049a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Piliszek?= Date: Tue, 8 Mar 2022 16:23:26 +0100 Subject: [PATCH 04/23] Install glusterfs-client in Debuntu Closes-Bug: #1964140 Change-Id: I4658a06b077d096ab9f2cb5070e39c98b5a27254 (cherry picked from commit da33e18ac83ff4696dcc599f46c12f007ba54856) --- docker/manila/manila-share/Dockerfile.j2 | 2 ++ releasenotes/notes/bug-1964140-57b433329bab067e.yaml | 6 ++++++ 2 files changed, 8 insertions(+) create mode 100644 releasenotes/notes/bug-1964140-57b433329bab067e.yaml diff --git a/docker/manila/manila-share/Dockerfile.j2 b/docker/manila/manila-share/Dockerfile.j2 index 04ddbf4ec5..68e8534557 100644 --- a/docker/manila/manila-share/Dockerfile.j2 +++ b/docker/manila/manila-share/Dockerfile.j2 @@ -22,6 +22,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% set manila_share_packages = [ 'manila-share', 'ceph-common', + 'glusterfs-client', 'python3-cephfs', 'python3-rados', 'python3-rbd', @@ -42,6 +43,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% elif base_package_type == 'deb' %} {% set manila_share_packages = [ 'ceph-common', + 'glusterfs-client', 'python3-cephfs', 'python3-rados', 'python3-rbd', diff --git a/releasenotes/notes/bug-1964140-57b433329bab067e.yaml b/releasenotes/notes/bug-1964140-57b433329bab067e.yaml new file mode 100644 index 0000000000..8869ab6420 --- /dev/null +++ b/releasenotes/notes/bug-1964140-57b433329bab067e.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Installs ``glusterfs-client`` in Debian and Ubuntu ``manila-share`` images + to support GlusterFS across supported distributions. + `LP#1964140 `__ From 292e78312a75dc7cb7c060d138eec7d34918d295 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 9 Mar 2022 11:48:20 +0000 Subject: [PATCH 05/23] libvirt: add Cyrus SASL packages for DIGEST-MD5 Adds Cyrus SASL packages necessary for the DIGEST-MD5 and SCRAM-SHA-256 mechanisms. These can be used for libvirt SASL authentication. Change-Id: I13e19ca29eeab40cd08fa3afe2cdf7531867f81b Partial-Bug: #1964013 (cherry picked from commit e1efa9d257c45e5f99364974ea66a6d6ba5a5605) --- docker/nova/nova-compute/Dockerfile.j2 | 8 ++++++++ docker/nova/nova-libvirt/Dockerfile.j2 | 3 +++ releasenotes/notes/libvirt-sasl-07a8a1a25d2450c6.yaml | 6 ++++++ 3 files changed, 17 insertions(+) create mode 100644 releasenotes/notes/libvirt-sasl-07a8a1a25d2450c6.yaml diff --git a/docker/nova/nova-compute/Dockerfile.j2 b/docker/nova/nova-compute/Dockerfile.j2 index 8c0df7fd4d..6a10e66272 100644 --- a/docker/nova/nova-compute/Dockerfile.j2 +++ b/docker/nova/nova-compute/Dockerfile.j2 @@ -14,6 +14,8 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% set nova_compute_packages = [ 'ceph-common', + 'cyrus-sasl-md5', + 'cyrus-sasl-scram', 'device-mapper-multipath', 'e2fsprogs', 'genisoimage', @@ -50,6 +52,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build 'e2fsprogs', 'genisoimage', 'ironic-common', + 'libsasl2-modules-gssapi-mit', 'multipath-tools', 'nfs-common', 'nova-compute', @@ -65,6 +68,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build 'python3-rados', 'python3-rbd', 'python3-rtslib-fb', + 'sasl2-bin', 'sysfsutils', 'targetcli-fb', 'xfsprogs' @@ -88,6 +92,8 @@ RUN rm -f /etc/nova/nova-compute.conf {% set nova_compute_packages = [ 'ceph-common', + 'cyrus-sasl-md5', + 'cyrus-sasl-scram', 'device-mapper-multipath', 'dosfstools', 'e2fsprogs', @@ -127,6 +133,7 @@ RUN rm -f /etc/nova/nova-compute.conf 'e2fsprogs', 'genisoimage', 'libosinfo-bin', + 'libsasl2-modules-gssapi-mit', 'multipath-tools', 'nfs-common', 'nvme-cli', @@ -140,6 +147,7 @@ RUN rm -f /etc/nova/nova-compute.conf 'python3-rbd', 'python3-rtslib-fb', 'qemu-utils', + 'sasl2-bin', 'sysfsutils', 'targetcli-fb', 'xfsprogs' diff --git a/docker/nova/nova-libvirt/Dockerfile.j2 b/docker/nova/nova-libvirt/Dockerfile.j2 index e6da29880c..db3a43885f 100644 --- a/docker/nova/nova-libvirt/Dockerfile.j2 +++ b/docker/nova/nova-libvirt/Dockerfile.j2 @@ -15,6 +15,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% set nova_libvirt_packages = [ 'ceph-common', + 'cyrus-sasl-md5', 'cyrus-sasl-scram', 'libguestfs', 'libvirt-client', @@ -41,6 +42,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% set nova_libvirt_packages = [ 'ceph-common', 'ebtables', + 'libsasl2-modules-gssapi-mit', 'libvirt-clients', 'libvirt-daemon-system', 'openvswitch-switch', @@ -51,6 +53,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build 'qemu-block-extra', 'qemu-system', 'qemu-utils', + 'sasl2-bin', 'trousers' ] %} diff --git a/releasenotes/notes/libvirt-sasl-07a8a1a25d2450c6.yaml b/releasenotes/notes/libvirt-sasl-07a8a1a25d2450c6.yaml new file mode 100644 index 0000000000..e88bc0d922 --- /dev/null +++ b/releasenotes/notes/libvirt-sasl-07a8a1a25d2450c6.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Adds Cyrus SASL packages necessary for the DIGEST-MD5 and SCRAM-SHA-256 + mechanisms. These can be used for libvirt SASL authentication. + `LP#1964013 `__ From 3a6a17970e289b890c03381bf0e8376fdd31ed64 Mon Sep 17 00:00:00 2001 From: Marcin Juszkiewicz Date: Mon, 28 Mar 2022 13:42:51 +0200 Subject: [PATCH 06/23] Use jinja2.pass_context instead of contextfilter The contextfilter decorator was deprecated in jinja2 3.0.0, and has been dropped in 3.1.0. This results in the following warning, and failed attempts to use filters: [WARNING]: Skipping plugin (filters.py) as it seems to be invalid: module 'jinja2' has no attribute 'contextfilter' This change switches to use the pass_context decorator. The minimum version of Jinja2 is raised to 3 to ensure pass_context is present. This change has been updated to also support Jinja2 2.x releases, since the Wallaby upper constraints specify 2.11.3. In practice, most users will not use UC to install kolla. CoAuthored-by: Mark Goddard Change-Id: I5efab66e487e06abd1a56af97d7e7caa1ebc880d --- kolla/template/filters.py | 9 +++++++-- kolla/template/methods.py | 8 ++++++-- .../notes/jinja2-pass-context-3f3febcd944e3a51.yaml | 4 ++++ 3 files changed, 17 insertions(+), 4 deletions(-) create mode 100644 releasenotes/notes/jinja2-pass-context-3f3febcd944e3a51.yaml diff --git a/kolla/template/filters.py b/kolla/template/filters.py index 49708ee3fc..f2fbc6e145 100644 --- a/kolla/template/filters.py +++ b/kolla/template/filters.py @@ -12,11 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -from jinja2 import contextfilter +# NOTE: jinja2 3.1.0 dropped contextfilter in favour of pass_context. +try: + from jinja2 import pass_context +except ImportError: + from jinja2 import contextfilter as pass_context + from jinja2 import Undefined -@contextfilter +@pass_context def customizable(context, val_list, call_type): # NOTE(mgoddard): Don't try to customise undefined values. There are cases # where this might happen, for example using a generic template overrides diff --git a/kolla/template/methods.py b/kolla/template/methods.py index 420a70306b..ddd7e0b7e4 100644 --- a/kolla/template/methods.py +++ b/kolla/template/methods.py @@ -15,7 +15,11 @@ import os import yaml -from jinja2 import contextfunction +# NOTE: jinja2 3.1.0 dropped contextfilter in favour of pass_context. +try: + from jinja2 import pass_context +except ImportError: + from jinja2 import contextfilter as pass_context def debian_package_install(packages, clean_package_cache=True): @@ -71,7 +75,7 @@ def debian_package_install(packages, clean_package_cache=True): return ' && '.join(cmds) -@contextfunction +@pass_context def handle_repos(context, reponames, mode): """NOTE(hrw): we need to handle CentOS, Debian and Ubuntu with one macro. diff --git a/releasenotes/notes/jinja2-pass-context-3f3febcd944e3a51.yaml b/releasenotes/notes/jinja2-pass-context-3f3febcd944e3a51.yaml new file mode 100644 index 0000000000..3a7ecc729c --- /dev/null +++ b/releasenotes/notes/jinja2-pass-context-3f3febcd944e3a51.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Fixes an issue seen when using Jinja2 3.1.0. From 8c29f15cabdef2419458891a3c8b9780a780326e Mon Sep 17 00:00:00 2001 From: Marcin Juszkiewicz Date: Wed, 30 Mar 2022 09:52:58 +0200 Subject: [PATCH 07/23] macros/pip: revert to old setuptools way AArch64 builds fail with "AttributeError: install_layout". This is due to setuptools issue in Debian: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1003252 Switching setuptools to use distutils from stdlib makes error go away. Backports: Xena, Wallaby (this is where it was first reported). Change-Id: I97102e18e4ec47bca94d17d964936f039580b06b (cherry picked from commit 497b40fe5a4984b959c63f0c5320311f5810192e) --- docker/macros.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/macros.j2 b/docker/macros.j2 index 61e3edce16..f39286e283 100644 --- a/docker/macros.j2 +++ b/docker/macros.j2 @@ -34,7 +34,7 @@ {% macro install_pip(packages, constraints = true) %} {%- if packages is sequence and packages|length > 0 -%} - python3 -m pip --no-cache-dir install --upgrade{{ ' ' }} + SETUPTOOLS_USE_DISTUTILS=stdlib python3 -m pip --no-cache-dir install --upgrade{{ ' ' }} {%- if constraints %}-c /requirements/upper-constraints.txt {% endif -%} {{ packages | join(' ') }} {%- else -%} From 5197793d9f8b807e0fd1082ead1f852b7b643961 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Thu, 31 Mar 2022 12:24:04 +0200 Subject: [PATCH 08/23] Fix disabling of curlrc in healthcheck_curl The -q (aka --disable) option only works if it is the first parameter on the command line. Change-Id: Ia9ab0bdc95e658c17e5be5abefcf96f1c05ee84f Closes-Bug: #1967272 (cherry picked from commit 196b742ba271fd7e3521448f83e8a2914f5c7dc6) --- docker/base/healthcheck_curl | 2 +- .../healthcheck-curl-disable-curlrc-0f85aad47379e2a5.yaml | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/healthcheck-curl-disable-curlrc-0f85aad47379e2a5.yaml diff --git a/docker/base/healthcheck_curl b/docker/base/healthcheck_curl index 2ef8538bac..6df232fe89 100755 --- a/docker/base/healthcheck_curl +++ b/docker/base/healthcheck_curl @@ -5,7 +5,7 @@ : ${HEALTHCHECK_CURL_OUTPUT:='/dev/null'} export NSS_SDB_USE_CACHE=no -curl -g -k -q -s -S --fail -o "${HEALTHCHECK_CURL_OUTPUT}" \ +curl -q -g -k -s -S --fail -o "${HEALTHCHECK_CURL_OUTPUT}" \ --max-time "${HEALTHCHECK_CURL_MAX_TIME}" \ --user-agent "${HEALTHCHECK_CURL_USER_AGENT}" \ --write-out "${HEALTHCHECK_CURL_WRITE_OUT}" \ diff --git a/releasenotes/notes/healthcheck-curl-disable-curlrc-0f85aad47379e2a5.yaml b/releasenotes/notes/healthcheck-curl-disable-curlrc-0f85aad47379e2a5.yaml new file mode 100644 index 0000000000..c1c00d6ce5 --- /dev/null +++ b/releasenotes/notes/healthcheck-curl-disable-curlrc-0f85aad47379e2a5.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes disabling the use of the ``curlrc`` configuration file in + ``healthcheck_curl``. `LP#1967272 `__ From 58f83d7ea4bd7be6a7d3529e84110d3df3bcef39 Mon Sep 17 00:00:00 2001 From: Marcin Juszkiewicz Date: Fri, 1 Apr 2022 12:56:56 +0200 Subject: [PATCH 09/23] cloudkitty-api: make sure that we install packages Debuntu binary images did not got packages installed. Change-Id: Ie28e8580406f3f805500ed4020895e5e9ebf7c31 (cherry picked from commit a8fcb02f40c94c2101edff9ec4a3101d1d14fe97) --- docker/cloudkitty/cloudkitty-api/Dockerfile.j2 | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docker/cloudkitty/cloudkitty-api/Dockerfile.j2 b/docker/cloudkitty/cloudkitty-api/Dockerfile.j2 index 4932c8a30b..f49f43ecbb 100644 --- a/docker/cloudkitty/cloudkitty-api/Dockerfile.j2 +++ b/docker/cloudkitty/cloudkitty-api/Dockerfile.j2 @@ -13,8 +13,6 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% set cloudkitty_api_packages = [ 'openstack-cloudkitty-api' ] %} -{{ macros.install_packages(cloudkitty_api_packages | customizable("packages")) }} - {% elif base_package_type == 'deb' %} {% set cloudkitty_api_packages = [ @@ -23,12 +21,12 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% endif %} {% elif install_type == 'source' %} - {% set cloudkitty_api_packages = [ ] %} -{{ macros.install_packages(cloudkitty_api_packages | customizable("packages")) }} {% endif %} +{{ macros.install_packages(cloudkitty_api_packages | customizable("packages")) }} + COPY extend_start.sh /usr/local/bin/kolla_cloudkitty_extend_start RUN chmod 755 /usr/local/bin/kolla_cloudkitty_extend_start From 531dd4444e33c80916cfd0e08de5580c92dc5217 Mon Sep 17 00:00:00 2001 From: Marcin Juszkiewicz Date: Mon, 4 Apr 2022 13:35:01 +0200 Subject: [PATCH 10/23] elasticsearch: install Java first on CentOS too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit elasticsearch-oss package fails to install if we install it with Java: INFO:kolla.common.utils.elasticsearch:could not find java; set JAVA_HOME INFO:kolla.common.utils.elasticsearch:error: %prein(elasticsearch-oss-0:6.8.23-1.noarch) scriptlet failed, exit status 1 INFO:kolla.common.utils.elasticsearch:Error in PREIN scriptlet in rpm package elasticsearch-oss Backport down to ussuri needed. Change-Id: I72d7920acd8d15941c8c57a4186186212b273a38 (cherry picked from commit 976465c448c4de2bd323021b946770b26ab1e912) --- docker/elasticsearch/elasticsearch/Dockerfile.j2 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docker/elasticsearch/elasticsearch/Dockerfile.j2 b/docker/elasticsearch/elasticsearch/Dockerfile.j2 index d09a0d8175..5633ce6d32 100644 --- a/docker/elasticsearch/elasticsearch/Dockerfile.j2 +++ b/docker/elasticsearch/elasticsearch/Dockerfile.j2 @@ -12,8 +12,12 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {{ macros.enable_extra_repos(['elasticsearch']) }} {% if base_package_type == 'rpm' %} + +# NOTE(hrw): post-install script of elasticsearch fails when trying to +# install elasticsearch and java together. +{{ macros.install_packages(['java-11-openjdk-headless']) }} + {% set elasticsearch_packages = [ - 'java-11-openjdk-headless', 'elasticsearch-oss', ] %} From cd58db65eaf669c31507d66d872d92913adafe6d Mon Sep 17 00:00:00 2001 From: Yamato Tanaka Date: Tue, 29 Mar 2022 11:35:12 +0900 Subject: [PATCH 11/23] Emit log when copying file/directory permissions Currently, kolla sets file permissions at _set_properties_from_file() method, but no logs are emitted here. This can make it difficult to investigate the cause of issues which are caused by setting permissions. This patch makes _set_properties_from_file() output a log. Closes-Bug: #1966825 Change-Id: I91116931aa30fd2c5e51ed1ac04663fcfa4a5f36 (cherry picked from commit 4c071fcc3a5efa01240602d6566035029fce2d5b) --- docker/base/set_configs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/base/set_configs.py b/docker/base/set_configs.py index e624f29ba8..1d1fc48aec 100644 --- a/docker/base/set_configs.py +++ b/docker/base/set_configs.py @@ -120,6 +120,7 @@ def _set_properties(self, source, dest): self._set_properties_from_conf(dest) def _set_properties_from_file(self, source, dest): + LOG.info('Copying permissions from %s onto %s', source, dest) shutil.copystat(source, dest) stat = os.stat(source) os.chown(dest, stat.st_uid, stat.st_gid) From 6b88dc0f09cab3b3a439b2ba6c5c38bf1f8b720d Mon Sep 17 00:00:00 2001 From: Marcin Juszkiewicz Date: Wed, 13 Apr 2022 20:19:38 +0200 Subject: [PATCH 12/23] Fix image builds with sources using a type=git A recent change to git [1] introduced a new behaviour to work around a CVE [2] that disallows any git operations in directories not owned by the current user. This may seem unrelated to installation, but it plays havoc with PBR, which calls out to git to get to get revision history. So if you are "pip install"-ing from a source tree you don't own, the PBR git calls in that tree now fail and the install blows up. When using type=source, kolla clones the repository, then creates a tarball from it, which is ADDed to the image. The ownership of the files in the tarball is preserved, which in this case will be the user running kolla-build. Since the Docker build runs as root, we hit the PBR issue. Our solution is to make sure that any tarball we generate from git sources have all files owned by root:root so that the root user is able to use git commands when building container images. [1] https://github.com/git/git/commit/8959555cee7ec045958f9b6dd62e541affb7e7d9 [2] https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-24765. Closes-Bug: #1969096 Related-Bug: #1968877 Co-Authored-By: Mark Goddard Change-Id: I2cbf1f539880d512aa223c3ef3a4b19ee18854ac (cherry picked from commit c4fda7baa3ffc36b555c32a34a00042b6035b917) --- kolla/image/build.py | 12 +++++++++++- .../notes/git-security-fix-fix-ea56c0071585237d.yaml | 9 +++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/git-security-fix-fix-ea56c0071585237d.yaml diff --git a/kolla/image/build.py b/kolla/image/build.py index 9135625f8c..08d990539c 100755 --- a/kolla/image/build.py +++ b/kolla/image/build.py @@ -409,8 +409,18 @@ def process_source(self, image, source): image.status = Status.ERROR return + # NOTE(mgoddard): Change ownership of files to root:root. This + # avoids an issue introduced by the fix for git CVE-2022-24765, + # which breaks PBR when the source checkout is not owned by the + # user installing it. LP#1969096 + def reset_userinfo(tarinfo): + tarinfo.uid = tarinfo.gid = 0 + tarinfo.uname = tarinfo.gname = "root" + return tarinfo + with tarfile.open(dest_archive, 'w') as tar: - tar.add(clone_dir, arcname=os.path.basename(clone_dir)) + tar.add(clone_dir, arcname=os.path.basename(clone_dir), + filter=reset_userinfo) elif source.get('type') == 'local': self.logger.debug("Getting local archive from %s", diff --git a/releasenotes/notes/git-security-fix-fix-ea56c0071585237d.yaml b/releasenotes/notes/git-security-fix-fix-ea56c0071585237d.yaml new file mode 100644 index 0000000000..a168f5d619 --- /dev/null +++ b/releasenotes/notes/git-security-fix-fix-ea56c0071585237d.yaml @@ -0,0 +1,9 @@ +--- +fixes: + - | + Fixes an issue building images that use a source with a ``type`` of + ``git``, when using a git that includes the fix for `CVE-2022-24765 + `__ (2.35.2 + or later). By default, this includes the ``gnocchi-base`` image, but may + include other images with a non-default configuration. `LP#837710 + `__ From 164cda1c41078eb0f5babc985915732131ed6765 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Piliszek?= Date: Fri, 15 Apr 2022 09:58:14 +0000 Subject: [PATCH 13/23] Revert "CI: add templated Dockerfiles to build logs" This reverts commit 09c890b12463a4b2c8767231189e7016f337fbc3. Reason for revert: That change caused all Kolla's work items to be transferred. This includes tarballs, their extracted contents, cloned git repositories, etc. to also be transferred to swift which takes extra time, pollutes logs and decreases the stability of jobs slightly (the mechanism is not designed for this kind of load). The collection of templated Dockerfiles in CI has to be reimplemented if it is wanted. Change-Id: Ideaf66ce18c993802f4d20f490caa47cec0dde5e (cherry picked from commit f33ef03977536ecb29ea464b9e557a6caacf4345) --- .zuul.d/base.yaml | 1 - tests/playbooks/run.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.zuul.d/base.yaml b/.zuul.d/base.yaml index d630cf100e..bc88cba08b 100644 --- a/.zuul.d/base.yaml +++ b/.zuul.d/base.yaml @@ -113,7 +113,6 @@ extra-vars: kolla_logs_dir: "{{ zuul_output_dir }}/logs/kolla" kolla_build_logs_dir: "{{ kolla_logs_dir }}/build" - kolla_work_dir: "{{ kolla_logs_dir }}" virtualenv_path: "/tmp/kolla-virtualenv" - job: diff --git a/tests/playbooks/run.yml b/tests/playbooks/run.yml index 0142c08070..27aebe6a27 100644 --- a/tests/playbooks/run.yml +++ b/tests/playbooks/run.yml @@ -14,7 +14,6 @@ # NOTE(yoctozepto): to avoid issues with IPv6 not enabled in the docker daemon # and since we don't need isolated networks here, use host networking network_mode: host - work_dir: "{{ kolla_work_dir }}" tasks: - name: Ensure /etc/kolla exists file: From 6e3381b0e2ed1cfae1b7b4035c0fa339c2aad779 Mon Sep 17 00:00:00 2001 From: Marcin Juszkiewicz Date: Thu, 14 Apr 2022 15:32:47 +0200 Subject: [PATCH 14/23] enable logging to file for quiet mode We have 'quiet' mode where only minimal data is printed to the console. But there were no logs at all then. This change generates log files (if 'logs-dir' argument is used) during quiet build. Also enables 'quiet' mode for CI so Zuul will not have to parse 29MB JSON file each time. Change-Id: If7d5c2807f0947a8bbbc1ceb8531c9b9c9287c1f (cherry picked from commit 0cf5b1d4e6e871562c0ca0f48a21594f8d197471) --- kolla/common/utils.py | 34 +++++++++++++------ ...quiet-mode-with-logs-0abafc07923945ac.yaml | 6 ++++ tests/playbooks/run.yml | 1 + 3 files changed, 30 insertions(+), 11 deletions(-) create mode 100644 releasenotes/notes/quiet-mode-with-logs-0abafc07923945ac.yaml diff --git a/kolla/common/utils.py b/kolla/common/utils.py index 9cd8ecca9d..0729127284 100644 --- a/kolla/common/utils.py +++ b/kolla/common/utils.py @@ -21,21 +21,33 @@ def make_a_logger(conf=None, image_name=None): log = logging.getLogger(".".join([__name__, image_name])) else: log = logging.getLogger(__name__) + + if conf is not None and conf.debug: + loglevel = logging.DEBUG + else: + loglevel = logging.INFO + if not log.handlers: - if conf is None or not conf.logs_dir or not image_name: - handler = logging.StreamHandler(sys.stderr) - log.propagate = False + stream_handler = logging.StreamHandler(sys.stderr) + stream_handler.setFormatter(logging.Formatter(logging.BASIC_FORMAT)) + # NOTE(hrw): quiet mode matters only on console + if conf is not None and conf.quiet: + stream_handler.setLevel(logging.CRITICAL) else: + stream_handler.setLevel(loglevel) + log.addHandler(stream_handler) + log.propagate = False + + if conf is not None and conf.logs_dir and image_name: filename = os.path.join(conf.logs_dir, "%s.log" % image_name) handler = logging.FileHandler(filename, delay=True) - handler.setFormatter(logging.Formatter(logging.BASIC_FORMAT)) - log.addHandler(handler) - if conf is not None and conf.debug: - log.setLevel(logging.DEBUG) - elif conf is not None and conf.quiet and image_name: - log.setLevel(logging.CRITICAL) - else: - log.setLevel(logging.INFO) + # NOTE(hrw): logfile will be INFO or DEBUG + handler.setLevel(loglevel) + handler.setFormatter(logging.Formatter(logging.BASIC_FORMAT)) + log.addHandler(handler) + + # NOTE(hrw): needs to be high, handlers have own levels + log.setLevel(logging.DEBUG) return log diff --git a/releasenotes/notes/quiet-mode-with-logs-0abafc07923945ac.yaml b/releasenotes/notes/quiet-mode-with-logs-0abafc07923945ac.yaml new file mode 100644 index 0000000000..1b8e67855a --- /dev/null +++ b/releasenotes/notes/quiet-mode-with-logs-0abafc07923945ac.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Quiet mode (enabled with ``--quiet`` argument) can be combined with + ``--logs-dir`` option now. Console output will be quiet as expected while + building output will be stored in separate log files. diff --git a/tests/playbooks/run.yml b/tests/playbooks/run.yml index 0142c08070..4ae9635531 100644 --- a/tests/playbooks/run.yml +++ b/tests/playbooks/run.yml @@ -8,6 +8,7 @@ DEFAULT: debug: true logs_dir: "{{ kolla_build_logs_dir }}" + quiet: true base: "{{ base_distro }}" install_type: "{{ install_type }}" template_override: /etc/kolla/template_overrides.j2 From 52375aeea7a67ca1223aacb3672f3ab3db5d0fd1 Mon Sep 17 00:00:00 2001 From: Marcin Juszkiewicz Date: Fri, 6 May 2022 10:25:22 +0200 Subject: [PATCH 15/23] prometheus-libvirt-exporter: fix build with newer Go I have no idea how Go works. This change make build happen. Change-Id: I09ea136c7354143f45f856d5e8d8ed1192879622 (cherry picked from commit 9630032fd61c6f0623bd1ae7577756e7166427f7) --- docker/prometheus/prometheus-libvirt-exporter/Dockerfile.j2 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/prometheus/prometheus-libvirt-exporter/Dockerfile.j2 b/docker/prometheus/prometheus-libvirt-exporter/Dockerfile.j2 index bcdcc7a56c..80e4028fe7 100644 --- a/docker/prometheus/prometheus-libvirt-exporter/Dockerfile.j2 +++ b/docker/prometheus/prometheus-libvirt-exporter/Dockerfile.j2 @@ -33,8 +33,10 @@ ARG prometheus_libvirt_exporter_path=github.com/AlexZzz/libvirt-exporter {% block prometheus_libvirt_exporter_install %} ENV GOPATH=/build RUN go mod init libvirt-exporter \ + && cd ${GOPATH} \ && go get -v ${prometheus_libvirt_exporter_path}@${prometheus_libvirt_exporter_version} \ - && mv /build/bin/libvirt-exporter /opt \ + && go build ${prometheus_libvirt_exporter_path} \ + && mv /build/libvirt-exporter /opt \ && rm -rf /build {% endblock %} From cfd0fb0f2804090d9a220e31f6293b84febbcf21 Mon Sep 17 00:00:00 2001 From: Marcin Juszkiewicz Date: Fri, 6 May 2022 07:45:56 +0200 Subject: [PATCH 16/23] cloudkitty: disable building for ubuntu/binary There are no packages in UCA. http://lists.openstack.org/pipermail/openstack-discuss/2022-May/028419.html Change-Id: Iaf5ce2b7276b275b21d892d0044a126ab901507f (cherry picked from commit 3d837170ea7196a08692a5f9f715e8b41da189a8) --- kolla/image/build.py | 1 + 1 file changed, 1 insertion(+) diff --git a/kolla/image/build.py b/kolla/image/build.py index 08d990539c..e4a2fb873a 100755 --- a/kolla/image/build.py +++ b/kolla/image/build.py @@ -144,6 +144,7 @@ class Status(Enum): }, 'ubuntu+binary': { + "cloudkitty-base", # no binary packages in UCA "senlin-conductor", # no binary package "senlin-health-manager", # no binary package "tacker-base", From 0009ffb7e8b6dd08510b67b9f94b2f96595837ff Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 21 Mar 2022 11:44:38 +0000 Subject: [PATCH 17/23] masakari: add Cyrus SASL packages to monitors image Adds Cyrus SASL packages necessary for the DIGEST-MD5 and SCRAM-SHA-256 mechanisms. These can be used for libvirt SASL authentication. Follow up to I13e19ca29eeab40cd08fa3afe2cdf7531867f81b. Partial-Bug: #1964013 Change-Id: Ic4b6171789c3d360317599310492771bd78828ec (cherry picked from commit 5cc480a617c68ed20c80bffec16321a937f5a32b) --- docker/masakari/masakari-monitors/Dockerfile.j2 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker/masakari/masakari-monitors/Dockerfile.j2 b/docker/masakari/masakari-monitors/Dockerfile.j2 index 62da6fcfe0..0a9c2d1739 100644 --- a/docker/masakari/masakari-monitors/Dockerfile.j2 +++ b/docker/masakari/masakari-monitors/Dockerfile.j2 @@ -20,10 +20,12 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% elif base_package_type == 'deb' %} {% set masakari_base_packages = [ + 'libsasl2-modules-gssapi-mit', 'masakari-host-monitor', 'masakari-instance-monitor', 'masakari-introspective-instance-monitor', 'masakari-process-monitor', + 'sasl2-bin', ] %} {% endif %} @@ -35,6 +37,8 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% if base_package_type == 'rpm' %} {% set masakari_monitors_packages = [ + 'cyrus-sasl-md5', + 'cyrus-sasl-scram', 'python3-libvirt', 'pacemaker-cli', 'tcpdump', @@ -43,8 +47,10 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% elif base_package_type == 'deb' %} {% set masakari_monitors_packages = [ + 'libsasl2-modules-gssapi-mit', 'python3-libvirt', 'pacemaker-cli-utils', + 'sasl2-bin', 'tcpdump', ] %} From 0e54080e61ad194906c8d49ba1e33b2ff0388504 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Piliszek?= Date: Wed, 18 May 2022 16:20:12 +0200 Subject: [PATCH 18/23] Fix local sources of git repositories This is I2cbf1f539880d512aa223c3ef3a4b19ee18854ac extended to fix the case when a git repository is used with a git repo. This is probably a rarer use case but, still, we use it in CI for in-review changes testing. Change-Id: I77b0dcd2e9dfd8ea8390a471b80c8954b67ef91b (cherry picked from commit b888f68daf6cb3ed9782fe85045858b6a04d4911) --- kolla/image/build.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/kolla/image/build.py b/kolla/image/build.py index e4a2fb873a..62039a3ae9 100755 --- a/kolla/image/build.py +++ b/kolla/image/build.py @@ -366,6 +366,15 @@ def followups(self): def process_source(self, image, source): dest_archive = os.path.join(image.path, source['name'] + '-archive') + # NOTE(mgoddard): Change ownership of files to root:root. This + # avoids an issue introduced by the fix for git CVE-2022-24765, + # which breaks PBR when the source checkout is not owned by the + # user installing it. LP#1969096 + def reset_userinfo(tarinfo): + tarinfo.uid = tarinfo.gid = 0 + tarinfo.uname = tarinfo.gname = "root" + return tarinfo + if source.get('type') == 'url': self.logger.debug("Getting archive from %s", source['source']) try: @@ -410,15 +419,6 @@ def process_source(self, image, source): image.status = Status.ERROR return - # NOTE(mgoddard): Change ownership of files to root:root. This - # avoids an issue introduced by the fix for git CVE-2022-24765, - # which breaks PBR when the source checkout is not owned by the - # user installing it. LP#1969096 - def reset_userinfo(tarinfo): - tarinfo.uid = tarinfo.gid = 0 - tarinfo.uname = tarinfo.gname = "root" - return tarinfo - with tarfile.open(dest_archive, 'w') as tar: tar.add(clone_dir, arcname=os.path.basename(clone_dir), filter=reset_userinfo) @@ -429,7 +429,8 @@ def reset_userinfo(tarinfo): if os.path.isdir(source['source']): with tarfile.open(dest_archive, 'w') as tar: tar.add(source['source'], - arcname=os.path.basename(source['source'])) + arcname=os.path.basename(source['source']), + filter=reset_userinfo) else: shutil.copyfile(source['source'], dest_archive) From 6a6fce5bf82516fc020a01af11e6e62a375165d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Piliszek?= Date: Sat, 21 May 2022 17:45:37 +0200 Subject: [PATCH 19/23] [bifrost] Force Bifrost to use the correct u-c Following up on [1], Bifrost also needs to use correct u-c and, surprise-surprise, this is set via a third variable, this time environment one for the bash script. Oh well. Let's do it. This has the added benefit of not requiring additional download of u-c as it reuses the already existing one. [1] https://review.opendev.org/c/openstack/kolla/+/842276 Change-Id: I2cc79c476d089977b57006dd4ce2e9a8c6206aa1 (cherry picked from commit 633c89b423c6d83609120faa87d4b4d63e59ccfa) --- docker/bifrost/bifrost-base/Dockerfile.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/bifrost/bifrost-base/Dockerfile.j2 b/docker/bifrost/bifrost-base/Dockerfile.j2 index e05d80218a..d238396875 100644 --- a/docker/bifrost/bifrost-base/Dockerfile.j2 +++ b/docker/bifrost/bifrost-base/Dockerfile.j2 @@ -40,7 +40,7 @@ RUN apt-get update && \ {%- else %} RUN echo " " && \ {%- endif %} - bash -c './scripts/env-setup.sh && \ + bash -c 'TOX_CONSTRAINTS_FILE=/requirements/upper-constraints.txt ./scripts/env-setup.sh && \ ansible-playbook -vvvv -i /bifrost/playbooks/inventory/target /bifrost/playbooks/install.yaml \ -e @/tmp/build_arg.yml && \ {%- if base_package_type == 'deb' %} From 6298c0e07324757e97627a206f5467a317de6d97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Piliszek?= Date: Thu, 26 May 2022 10:27:53 +0200 Subject: [PATCH 20/23] Fix Ubuntu image builds Regarding rabbitmq and erlang. Change-Id: I6bc8b489a810849744c8bead6b9e350c5d3e36b7 (cherry picked from commit 240b3cf448354bf1a7271f8f81d730e51da856cb) --- docker/base/Dockerfile.j2 | 4 ++-- kolla/template/repos.yaml | 22 +++++++++---------- ...mq-erlang-cloudsmith-c837bf4a450dd802.yaml | 10 +++++++++ 3 files changed, 22 insertions(+), 14 deletions(-) create mode 100644 releasenotes/notes/rabbitmq-erlang-cloudsmith-c837bf4a450dd802.yaml diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2 index b172ac1cc9..11abeca6a9 100644 --- a/docker/base/Dockerfile.j2 +++ b/docker/base/Dockerfile.j2 @@ -322,8 +322,8 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom ] %} {% set base_apt_keys = [ - {'name': 'erlang', 'keyid': 'F77F1EDA57EBB1CC'}, - {'name': 'rabbitmq', 'keyid': 'F6609E60DC62814E'}, + {'name': 'erlang', 'keyid': 'E495BB49CC4BBE5B'}, + {'name': 'rabbitmq', 'keyid': '9F4587F226208342'}, {'name': 'haproxy', 'keyid': 'CFFB779AADC995E4F350A060505D97A41C61B9CD'}, ] %} diff --git a/kolla/template/repos.yaml b/kolla/template/repos.yaml index 1afb87f200..b75f13038d 100644 --- a/kolla/template/repos.yaml +++ b/kolla/template/repos.yaml @@ -49,9 +49,8 @@ debian: component: "main" gpg_key: "elasticsearch.asc" erlang: - # NOTE(mnasiadka): Erlang repo - Debian Buster/Bullseye needs to use bionic as per RabbitMQ docs - url: "http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu" - suite: "bionic" + url: "https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-erlang/deb/debian" + suite: "bullseye" component: "main" gpg_key: "erlang.gpg" grafana: @@ -80,7 +79,7 @@ debian: component: "main" gpg_key: "mariadb.gpg" rabbitmq: - url: "https://packagecloud.io/rabbitmq/rabbitmq-server/debian/" + url: "https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-server/deb/debian" suite: "bullseye" component: "main" gpg_key: "rabbitmq.gpg" @@ -102,9 +101,8 @@ debian-aarch64: component: "main" gpg_key: "elasticsearch.asc" erlang: - # NOTE(mnasiadka): Erlang repo - Debian Buster/Bullseye needs to use bionic as per RabbitMQ docs - url: "http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu" - suite: "bionic" + url: "https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-erlang/deb/debian" + suite: "bullseye" component: "main" gpg_key: "erlang.gpg" grafana: @@ -133,7 +131,7 @@ debian-aarch64: component: "main" gpg_key: "mariadb.gpg" rabbitmq: - url: "https://packagecloud.io/rabbitmq/rabbitmq-server/debian/" + url: "https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-server/deb/debian" suite: "bullseye" component: "main" gpg_key: "rabbitmq.gpg" @@ -155,7 +153,7 @@ ubuntu: component: "main" gpg_key: "elasticsearch.asc" erlang: - url: "http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu" + url: "https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-erlang/deb/ubuntu" suite: "focal" component: "main" gpg_key: "erlang.gpg" @@ -190,7 +188,7 @@ ubuntu: component: "main" gpg_key: "mariadb.gpg" rabbitmq: - url: "https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/" + url: "https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-server/deb/ubuntu" suite: "focal" component: "main" gpg_key: "rabbitmq.gpg" @@ -212,7 +210,7 @@ ubuntu-aarch64: component: "main" gpg_key: "elasticsearch.asc" erlang: - url: "http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu" + url: "https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-erlang/deb/ubuntu" suite: "focal" component: "main" gpg_key: "erlang.gpg" @@ -242,7 +240,7 @@ ubuntu-aarch64: component: "main" gpg_key: "mariadb.gpg" rabbitmq: - url: "https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/" + url: "https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-server/deb/ubuntu" suite: "focal" component: "main" gpg_key: "rabbitmq.gpg" diff --git a/releasenotes/notes/rabbitmq-erlang-cloudsmith-c837bf4a450dd802.yaml b/releasenotes/notes/rabbitmq-erlang-cloudsmith-c837bf4a450dd802.yaml new file mode 100644 index 0000000000..b87bf97632 --- /dev/null +++ b/releasenotes/notes/rabbitmq-erlang-cloudsmith-c837bf4a450dd802.yaml @@ -0,0 +1,10 @@ +--- +fixes: + - | + Fixes the Debian and Ubuntu images to use rabbitmq and erlang from + cloudsmith so that the images are still buildable and use proper versions. +upgrade: + - | + The Debian and Ubuntu images use rabbitmq and erlang from cloudsmith now. + Operators might want to mirror/proxy this new source as it provides the + correct set of packages unlike the previous combination. From e90786ab9affe99e3c6986f54a580844f1598e39 Mon Sep 17 00:00:00 2001 From: Marcin Juszkiewicz Date: Wed, 1 Jun 2022 17:16:22 +0200 Subject: [PATCH 21/23] Allow to provide own repos.yaml file Several people use Kolla behind firewall/proxy. Internal mirrors are often in use then. We do not provide a way to replace repos.yaml file in an easy way which may lead to complicated solutions or template overrides. This patch adds a way to provide own copy of repos.yaml file. Change-Id: I0b07da22fea27e0ff4e90aaad19e50d84ff9a121 (cherry picked from commit 213190ae030e9971225ea3f103b4be0a7f2532d4) --- kolla/common/config.py | 4 +++- kolla/image/build.py | 2 ++ kolla/template/methods.py | 6 +++++- .../notes/user-provided-repos.yaml-d6e8583e9d9d01ef.yaml | 6 ++++++ 4 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/user-provided-repos.yaml-d6e8583e9d9d01ef.yaml diff --git a/kolla/common/config.py b/kolla/common/config.py index f53d664bc2..cd449a93b3 100755 --- a/kolla/common/config.py +++ b/kolla/common/config.py @@ -249,7 +249,9 @@ cfg.BoolOpt('summary', default=True, help='Show summary at the end of build'), cfg.BoolOpt('infra-rename', default=False, - help='Rename infrastructure images to infra') + help='Rename infrastructure images to infra'), + cfg.StrOpt('repos-yaml', default='', + help='Path to alternative repos.yaml file'), ] _BASE_OPTS = [ diff --git a/kolla/image/build.py b/kolla/image/build.py index 62039a3ae9..45383535d6 100755 --- a/kolla/image/build.py +++ b/kolla/image/build.py @@ -654,6 +654,7 @@ def __init__(self, conf): self.base_tag = conf.base_tag self.install_type = conf.install_type self.tag = conf.tag + self.repos_yaml = conf.repos_yaml self.base_arch = conf.base_arch self.debian_arch = self.base_arch if self.base_arch == 'aarch64': @@ -880,6 +881,7 @@ def create_dockerfiles(self): 'base_image': self.conf.base_image, 'base_distro_tag': self.base_tag, 'base_arch': self.base_arch, + 'repos_yaml': self.repos_yaml, 'use_dumb_init': self.use_dumb_init, 'base_package_type': self.base_package_type, 'debian_arch': self.debian_arch, diff --git a/kolla/template/methods.py b/kolla/template/methods.py index ddd7e0b7e4..534f87fe7c 100644 --- a/kolla/template/methods.py +++ b/kolla/template/methods.py @@ -95,7 +95,11 @@ def handle_repos(context, reponames, mode): if not isinstance(reponames, list): raise TypeError("First argument should be a list of repositories") - repofile = os.path.dirname(os.path.realpath(__file__)) + '/repos.yaml' + if context.get('repos_yaml'): + repofile = context.get('repos_yaml') + else: + repofile = os.path.dirname(os.path.realpath(__file__)) + '/repos.yaml' + with open(repofile, 'r') as repos_file: repo_data = {} for name, params in yaml.safe_load(repos_file).items(): diff --git a/releasenotes/notes/user-provided-repos.yaml-d6e8583e9d9d01ef.yaml b/releasenotes/notes/user-provided-repos.yaml-d6e8583e9d9d01ef.yaml new file mode 100644 index 0000000000..98e41ec3f7 --- /dev/null +++ b/releasenotes/notes/user-provided-repos.yaml-d6e8583e9d9d01ef.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Added an `--repos-yaml` argument to allow user to provide own file with + definitions of external package repositories. Useful for those building + in offline environments with set of internal mirrors. From 5baeaf498e147e909524ac3ca094e7c987d18af6 Mon Sep 17 00:00:00 2001 From: Maksim Malchuk Date: Tue, 7 Jun 2022 11:54:43 +0300 Subject: [PATCH 22/23] Pin diskimage-builder version due to drop Python3.6 Since Ibdbcf459608711ac64e7fefb1707f6708d68e750 merged to master we need to pin diskimage-builder to latest stable tag with support of the Python3.6 until we move to CentOS9. Change-Id: I9f9e381716611b71584ecf8b371f8b7f45c694ee Signed-off-by: Maksim Malchuk (cherry picked from commit 1b259ecc5bbbe48e083a08d2e67f42887bc2eec3) --- docker/bifrost/bifrost-base/Dockerfile.j2 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/bifrost/bifrost-base/Dockerfile.j2 b/docker/bifrost/bifrost-base/Dockerfile.j2 index d238396875..93282fb03f 100644 --- a/docker/bifrost/bifrost-base/Dockerfile.j2 +++ b/docker/bifrost/bifrost-base/Dockerfile.j2 @@ -42,6 +42,11 @@ RUN echo " " && \ {%- endif %} bash -c 'TOX_CONSTRAINTS_FILE=/requirements/upper-constraints.txt ./scripts/env-setup.sh && \ ansible-playbook -vvvv -i /bifrost/playbooks/inventory/target /bifrost/playbooks/install.yaml \ +{%- if base_distro == 'centos' %} +{# NOTE(mmalchuk): since DIB drop Python3.6 support we need to build DIB from #} +{# the latest supported stable tag until we move to CentOS9 #} + -e dib_git_branch="3.21.1" \ +{%- endif %} -e @/tmp/build_arg.yml && \ {%- if base_package_type == 'deb' %} apt-get clean && rm -rf /var/lib/apt/lists/*' From 1ab4e13fe31be4d3559cdf6288215d0a81788543 Mon Sep 17 00:00:00 2001 From: Maksim Malchuk Date: Fri, 3 Jun 2022 17:01:25 +0300 Subject: [PATCH 23/23] Don't ignore errors of 'apt-get update' command This change fixes an issue with 'apt-get update' command and stop container building with erroneous source repositories. Change-Id: I5ce2eb66a647e1805f086aa37942bf5d13eb4106 Signed-off-by: Maksim Malchuk (cherry picked from commit bce06d2d5a27fd2e430de0078b7d585cfc98d8a4) --- kolla/template/methods.py | 2 +- .../notes/apt-get-update-fatal-ed2bcc5c1d93c082.yaml | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/apt-get-update-fatal-ed2bcc5c1d93c082.yaml diff --git a/kolla/template/methods.py b/kolla/template/methods.py index 534f87fe7c..10a28cd0ed 100644 --- a/kolla/template/methods.py +++ b/kolla/template/methods.py @@ -53,7 +53,7 @@ def debian_package_install(packages, clean_package_cache=True): # handle the apt-get install if reg_packages: - cmds.append('apt-get update') + cmds.append('apt-get --error-on=any update') cmds.append('apt-get -y install --no-install-recommends {}'.format( ' '.join(reg_packages) )) diff --git a/releasenotes/notes/apt-get-update-fatal-ed2bcc5c1d93c082.yaml b/releasenotes/notes/apt-get-update-fatal-ed2bcc5c1d93c082.yaml new file mode 100644 index 0000000000..395c8936bd --- /dev/null +++ b/releasenotes/notes/apt-get-update-fatal-ed2bcc5c1d93c082.yaml @@ -0,0 +1,10 @@ +--- +fixes: + - | + The ``apt-get update`` command by default didn't fail on erroneous source + repositories, it show the warning 'W: Some index files failed to download. + They have been ignored, or old ones used instead.' and continue to work. + This causes some containers (eg. rabbitmq, kolla-toolbox) successfully + built, but makes them inconsistent because the official Ubuntu repository + contains packages with the same names. Now we use ``apt-get -eany update`` + command to stop building with an error in such cases.