From cbf43757f63e3052d12b1e94f4f4aa5fe494ebfe Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Thu, 15 Apr 2021 11:14:11 +0000 Subject: [PATCH 01/16] rabbitmq: Move to packagecloud bintray is being shut down 1st of May 2021, rabbitmq docs direct users to packagecloud - so let's move. For Victoria and below, includes I6a3ba599af3deedd450b4ffff6b86a0c04628340 (base: drop Linaro OBS key for CentOS and Ubuntu) Change-Id: I499251d13ca77aea2215f8a879b7bb8dd51f1c9f (cherry picked from commit 641b936bc7fd33203a80bb3827fce7c44d010476) --- docker/base/Dockerfile.j2 | 20 +++++-------------- docker/base/rabbitmq_rabbitmq-erlang.repo | 3 +-- docker/base/rabbitmq_rabbitmq-server.repo | 3 +-- docker/base/sources.list.debian | 3 --- docker/base/sources.list.ubuntu | 7 ++----- kolla/template/repos.yaml | 20 +++++++++---------- ...abbitmq-packagecloud-c009ff6d0d2bd02d.yaml | 6 ++++++ 7 files changed, 25 insertions(+), 37 deletions(-) create mode 100644 releasenotes/notes/rabbitmq-packagecloud-c009ff6d0d2bd02d.yaml diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2 index 590a061d54..baec98c392 100644 --- a/docker/base/Dockerfile.j2 +++ b/docker/base/Dockerfile.j2 @@ -86,7 +86,7 @@ COPY dnf.conf /etc/dnf/dnf.conf 'https://artifacts.elastic.co/GPG-KEY-elasticsearch', 'https://packages.grafana.com/gpg.key', 'https://repos.influxdata.com/influxdb.key', - 'https://www.rabbitmq.com/rabbitmq-release-signing-key.asc', + 'https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc', 'https://packages.treasuredata.com/GPG-KEY-td-agent', ] %} {% elif base_arch == 'aarch64' %} @@ -99,8 +99,7 @@ COPY dnf.conf /etc/dnf/dnf.conf {% set base_yum_repo_keys = [ 'https://packages.grafana.com/gpg.key', - 'https://www.rabbitmq.com/rabbitmq-release-signing-key.asc', - 'https://obs.linaro.org/repos/home:/marcin.juszkiewicz/centos7/repodata/repomd.xml.key', + 'https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc', ] %} # FIXME(mgoddard): Not available for CentOS 8 yet. # @@ -342,6 +341,7 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom {# 901F9177AB97ACBE -- Treasure Data, Inc (Treasure Agent Official Signing key) #} {# A20F259AEB9C94BB -- Sensuapp (Freight) #} {# F1656F24C74CD1D8 -- MariaDB Signing Key #} + {# F77F1EDA57EBB1CC -- RabbitMQ Signing Key #} {% set base_apt_keys = [ '391A9AA2147192839E9DB0315EDB1B62EC4926EA', '46095ACC8548582C1A2699A9D27D666CD88E42B4', @@ -351,20 +351,16 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom '901F9177AB97ACBE', 'A20F259AEB9C94BB', 'F1656F24C74CD1D8', + 'F77F1EDA57EBB1CC', ] %} {% set remote_apt_keys = [ - 'https://www.rabbitmq.com/rabbitmq-release-signing-key.asc', 'https://packages.grafana.com/gpg.key', 'https://repos.influxdata.com/influxdb.key', ] %} - {% if base_arch == 'aarch64' %} - {% set remote_apt_keys = remote_apt_keys + [ - 'https://obs.linaro.org/repos/home:/marcin.juszkiewicz/ubuntu-bionic/Release.key', - ] %} - {% endif %} {% elif base_distro == 'debian' %} {% set base_apt_keys = [ '46095ACC8548582C1A2699A9D27D666CD88E42B4', + 'F77F1EDA57EBB1CC', ] %} {% set remote_apt_keys = [ 'https://download.docker.com/linux/debian/gpg', @@ -373,13 +369,7 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom 'https://packages.treasuredata.com/GPG-KEY-td-agent', 'https://repos.influxdata.com/influxdb.key', 'https://obs.linaro.org/repos/home:/marcin.juszkiewicz/debian-buster/Release.key', - 'https://www.rabbitmq.com/rabbitmq-release-signing-key.asc', ] %} - {% if base_arch == 'aarch64' %} - {% set remote_apt_keys = remote_apt_keys + [ - 'https://obs.linaro.org/repos/home:/marcin.juszkiewicz/debian-buster/Release.key', - ] %} - {% endif %} {% set base_apt_packages = base_apt_packages + ['sudo',] %} diff --git a/docker/base/rabbitmq_rabbitmq-erlang.repo b/docker/base/rabbitmq_rabbitmq-erlang.repo index c9d9aed428..bcb4f40f1f 100644 --- a/docker/base/rabbitmq_rabbitmq-erlang.repo +++ b/docker/base/rabbitmq_rabbitmq-erlang.repo @@ -1,6 +1,5 @@ [rabbitmq_rabbitmq-erlang] name=rabbitmq_rabbitmq-erlang -baseurl=https://dl.bintray.com/rabbitmq-erlang/rpm/erlang/22/el/$releasever/ +baseurl=https://packagecloud.io/rabbitmq/erlang/el/8/$basearch gpgcheck=1 enabled=0 -gpgkey=https://www.rabbitmq.com/rabbitmq-release-signing-key.asc diff --git a/docker/base/rabbitmq_rabbitmq-server.repo b/docker/base/rabbitmq_rabbitmq-server.repo index 50f307d86a..0bbf8a82fd 100644 --- a/docker/base/rabbitmq_rabbitmq-server.repo +++ b/docker/base/rabbitmq_rabbitmq-server.repo @@ -1,6 +1,5 @@ [rabbitmq_rabbitmq-server] name=rabbitmq_rabbitmq-server -baseurl=https://dl.bintray.com/rabbitmq/rpm/rabbitmq-server/v3.8.x/el/$releasever/ +baseurl=https://packagecloud.io/rabbitmq/rabbitmq-server/el/8/$basearch gpgcheck=1 enabled=0 -gpgkey=https://www.rabbitmq.com/rabbitmq-release-signing-key.asc diff --git a/docker/base/sources.list.debian b/docker/base/sources.list.debian index a8972c1dd9..d74f36c332 100644 --- a/docker/base/sources.list.debian +++ b/docker/base/sources.list.debian @@ -31,6 +31,3 @@ deb http://buster-victoria.debian.net/debian buster-victoria-backports-nochange # erlang repo #deb https://dl.bintray.com/rabbitmq-erlang/debian/ buster erlang - -# erlang repo - aarch64 only -#deb https://obs.linaro.org/repos/home:/marcin.juszkiewicz/debian-buster ./ diff --git a/docker/base/sources.list.ubuntu b/docker/base/sources.list.ubuntu index 288deb16ec..224a572ec1 100644 --- a/docker/base/sources.list.ubuntu +++ b/docker/base/sources.list.ubuntu @@ -34,13 +34,10 @@ deb http://ubuntu-cloud.archive.canonical.com/ubuntu focal-updates/victoria main #deb http://ppa.launchpad.net/qpid/released/ubuntu/ focal main # rabbitmq repo -#deb https://dl.bintray.com/rabbitmq/debian/ focal main +#deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ focal main # erlang repo -#deb https://dl.bintray.com/rabbitmq-erlang/debian/ focal erlang - -# erlang repo - aarch64 only -#deb https://obs.linaro.org/repos/home:/marcin.juszkiewicz/ubuntu-focal ./ +#deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu focal main # MariaDB repo #deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu focal main diff --git a/kolla/template/repos.yaml b/kolla/template/repos.yaml index f6f26a8046..cb11657b2c 100644 --- a/kolla/template/repos.yaml +++ b/kolla/template/repos.yaml @@ -47,50 +47,50 @@ rhel: opstools: "centos-opstools" rabbitmq: "centos-rabbitmq-38" +# NOTE(mnasiadka): Erlang repo - Debian Buster/Bullseye needs to use bionic as per RabbitMQ docs debian: elasticsearch: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-6.x/apt stable main" - erlang: "deb https://dl.bintray.com/rabbitmq-erlang/debian/ buster erlang" + erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu bionic main" grafana: "deb https://packages.grafana.com/oss/deb stable main" influxdb: "deb https://repos.influxdata.com/debian buster stable" logstash: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-6.x/apt stable main" kibana: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-6.x/apt stable main" - rabbitmq: "deb https://dl.bintray.com/rabbitmq/debian/ buster main" + rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/debian/ buster main" td-agent: "deb http://packages.treasuredata.com/4/debian/buster buster contrib" debian-aarch64: elasticsearch: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-6.x/apt stable main" - erlang: "deb https://obs.linaro.org/repos/home:/marcin.juszkiewicz/debian-buster ./" + erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu bionic main" grafana: "deb https://packages.grafana.com/oss/deb stable main" influxdb: "deb https://repos.influxdata.com/debian buster stable" logstash: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-6.x/apt stable main" kibana: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-6.x/apt stable main" libvirt: "deb https://obs.linaro.org/repos/home:/marcin.juszkiewicz/debian-buster ./" - rabbitmq: "deb https://dl.bintray.com/rabbitmq/debian/ buster main" + rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/debian/ buster main" td-agent: "deb http://packages.treasuredata.com/4/debian/buster buster contrib" ubuntu: elasticsearch: "deb https://artifacts.elastic.co/packages/oss-6.x/apt stable main" - erlang: "deb https://dl.bintray.com/rabbitmq-erlang/debian/ focal erlang" + erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu focal main" grafana: "deb https://packages.grafana.com/oss/deb stable main" influxdb: "deb https://repos.influxdata.com/ubuntu focal stable" logstash: "deb https://artifacts.elastic.co/packages/oss-6.x/apt stable main" kibana: "deb https://artifacts.elastic.co/packages/oss-6.x/apt stable main" mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu focal main" qdrouterd: "deb http://ppa.launchpad.net/qpid/released/ubuntu/ focal main" - # NOTE(hrw): upstream uses bionic repo for focal - rabbitmq: "deb https://dl.bintray.com/rabbitmq/debian/ bionic main" + rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ focal main" td-agent: "deb http://packages.treasuredata.com/4/ubuntu/focal/ focal contrib" ubuntu-aarch64: elasticsearch: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-6.x/apt stable main" - erlang: "deb https://obs.linaro.org/repos/home:/marcin.juszkiewicz/ubuntu-focal ./" + erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu focal main" grafana: "deb https://packages.grafana.com/oss/deb stable main" influxdb: "deb https://repos.influxdata.com/ubuntu focal stable" logstash: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-6.x/apt stable main" mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu bionic main" - rabbitmq: "deb https://dl.bintray.com/rabbitmq/debian/ bionic main" + rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ focal main" td-agent: "deb http://packages.treasuredata.com/4/ubuntu/focal/ focal contrib" ubuntu-ppc64le: mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu focal main" - rabbitmq: "deb https://dl.bintray.com/rabbitmq/debian/ bionic main" + rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ focal main" diff --git a/releasenotes/notes/rabbitmq-packagecloud-c009ff6d0d2bd02d.yaml b/releasenotes/notes/rabbitmq-packagecloud-c009ff6d0d2bd02d.yaml new file mode 100644 index 0000000000..4f966a92a0 --- /dev/null +++ b/releasenotes/notes/rabbitmq-packagecloud-c009ff6d0d2bd02d.yaml @@ -0,0 +1,6 @@ +--- +upgrade: + - | + RabbitMQ and Erlang packages are now installed from ``packagecloud.io`` + (and PPA for Debian/Ubuntu) since ``bintray.com`` is getting shut down + May 1st, 2021. From 4c366e9fbc3294fd5f9822c31538766deeeafbdc Mon Sep 17 00:00:00 2001 From: Marcin Juszkiewicz Date: Mon, 26 Apr 2021 13:15:33 +0200 Subject: [PATCH 02/16] Disable trove for Debian/binary Package installation fails: Setting up trove-common (1:14.0.0-2~bpo10+1) ... install: cannot stat '/usr/share/trove-common/api-paste.ini': No such file or directory dpkg: error processing package trove-common (--configure): installed trove-common package post-installation script subprocess returned error exit status 1 Change-Id: Ia4d326601d14c5d55e8f67d8996458fc796583ea (cherry picked from commit 51c3c90af9a0c4ed27b4af5e0650b4e3a9eaeb53) --- kolla/image/build.py | 1 + 1 file changed, 1 insertion(+) diff --git a/kolla/image/build.py b/kolla/image/build.py index 58b132b9a7..8fa331cf12 100755 --- a/kolla/image/build.py +++ b/kolla/image/build.py @@ -172,6 +172,7 @@ class Status(Enum): "senlin-conductor", # no binary package "senlin-health-manager", # no binary package "tacker-base", + "trove-base", # package is not installable "neutron-mlnx-agent", }, From 3ee3c7d7c9702bb2f76d0e01f8bb6d4638e5b78e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Piliszek?= Date: Mon, 26 Apr 2021 16:54:11 +0000 Subject: [PATCH 03/16] Pin erlang Now that we run without erlang pinned by the repository itself, let's pin it on our side. Change-Id: Ifa491b102ad0db7f6dc645ca552d83b53bcf3b46 (cherry picked from commit b54789d093d8f9e35265d313e95f9f89aceb4f4b) --- docker/base/apt_preferences.ubuntu | 4 ++++ docker/kolla-toolbox/Dockerfile.j2 | 1 + docker/rabbitmq/Dockerfile.j2 | 1 + 3 files changed, 6 insertions(+) diff --git a/docker/base/apt_preferences.ubuntu b/docker/base/apt_preferences.ubuntu index 9792854d5e..c8e36affd0 100644 --- a/docker/base/apt_preferences.ubuntu +++ b/docker/base/apt_preferences.ubuntu @@ -1,3 +1,7 @@ Package: rabbitmq-server Pin: version 3.8.* Pin-Priority: 1000 + +Package: erlang* +Pin: version 1:23.* +Pin-Priority: 1000 diff --git a/docker/kolla-toolbox/Dockerfile.j2 b/docker/kolla-toolbox/Dockerfile.j2 index 06e9bb2302..b067b1de01 100644 --- a/docker/kolla-toolbox/Dockerfile.j2 +++ b/docker/kolla-toolbox/Dockerfile.j2 @@ -19,6 +19,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% if base_package_type == 'rpm' %} {% set kolla_toolbox_packages = [ 'crudini', + 'erlang-23.*', 'gcc', 'gdisk', 'git', diff --git a/docker/rabbitmq/Dockerfile.j2 b/docker/rabbitmq/Dockerfile.j2 index 28c1ef5f65..77de38b569 100644 --- a/docker/rabbitmq/Dockerfile.j2 +++ b/docker/rabbitmq/Dockerfile.j2 @@ -15,6 +15,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% if base_package_type == 'rpm' %} {% set rabbitmq_packages = [ 'hostname', + 'erlang-23.*', 'rabbitmq-server-3.8.*' ] %} From 0c20a44f9ad764492d98a6c6e21218dbee67e2a7 Mon Sep 17 00:00:00 2001 From: Marcin Juszkiewicz Date: Thu, 29 Apr 2021 11:12:21 +0200 Subject: [PATCH 04/16] CentOS on AArch64 has only Erlang 22 RabbitMQ 3.8.14 is the last version with Erlang 22 support Change-Id: I1d68a47f78b925af76c44c61af7da1465371e721 (cherry picked from commit 0ec6049a73ef7e322c06ee0e7b8582e5e69b8815) --- docker/kolla-toolbox/Dockerfile.j2 | 15 +++++++++++++-- docker/rabbitmq/Dockerfile.j2 | 15 +++++++++++++-- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/docker/kolla-toolbox/Dockerfile.j2 b/docker/kolla-toolbox/Dockerfile.j2 index b067b1de01..23cbd1a8ea 100644 --- a/docker/kolla-toolbox/Dockerfile.j2 +++ b/docker/kolla-toolbox/Dockerfile.j2 @@ -17,9 +17,9 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {{ macros.enable_extra_repos(['rabbitmq', 'erlang']) }} {% if base_package_type == 'rpm' %} + {% set kolla_toolbox_packages = [ 'crudini', - 'erlang-23.*', 'gcc', 'gdisk', 'git', @@ -34,9 +34,20 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build 'openssl-devel', 'openvswitch', 'python3-devel', - 'rabbitmq-server-3.8.*' ] %} + {% if base_arch == 'aarch64' %} + {% set kolla_toolbox_packages = kolla_toolbox_packages + [ + 'erlang-22.*', + 'rabbitmq-server-3.8.14' + ] %} + {% else %} + {% set kolla_toolbox_packages = kolla_toolbox_packages + [ + 'erlang-23.*', + 'rabbitmq-server-3.8.*' + ] %} + {% endif %} + {% elif base_package_type == 'deb' %} {% set kolla_toolbox_packages = [ 'build-essential', diff --git a/docker/rabbitmq/Dockerfile.j2 b/docker/rabbitmq/Dockerfile.j2 index 77de38b569..d629561870 100644 --- a/docker/rabbitmq/Dockerfile.j2 +++ b/docker/rabbitmq/Dockerfile.j2 @@ -13,12 +13,23 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% block rabbitmq_install %} {% if base_package_type == 'rpm' %} + {% set rabbitmq_packages = [ 'hostname', - 'erlang-23.*', - 'rabbitmq-server-3.8.*' ] %} + {% if base_arch == 'aarch64' %} + {% set rabbitmq_packages = rabbitmq_packages + [ + 'erlang-22.*', + 'rabbitmq-server-3.8.14' + ] %} + {% else %} + {% set rabbitmq_packages = rabbitmq_packages + [ + 'erlang-23.*', + 'rabbitmq-server-3.8.*' + ] %} + {% endif %} + {% elif base_package_type == 'deb' %} {% set rabbitmq_packages = [ 'logrotate', From 29bb43d53b611aa66d470c27550fae5cebced6ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Piliszek?= Date: Fri, 30 Apr 2021 18:41:11 +0000 Subject: [PATCH 05/16] Install RabbitMQ repo key on Ubuntu and Debian It was missed previously. CI did not detect it because it ignores apt security issues. That gets fixed in Ic5abc4b87fd76f87aba383abf43e95ba70629fcb Change-Id: I11256c674c42e35496146ff64fee1e8e04c30d2c (cherry picked from commit 444313975e6099055c5573363eea8af5a035fdbd) --- docker/base/Dockerfile.j2 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2 index baec98c392..90d2089a4e 100644 --- a/docker/base/Dockerfile.j2 +++ b/docker/base/Dockerfile.j2 @@ -341,7 +341,8 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom {# 901F9177AB97ACBE -- Treasure Data, Inc (Treasure Agent Official Signing key) #} {# A20F259AEB9C94BB -- Sensuapp (Freight) #} {# F1656F24C74CD1D8 -- MariaDB Signing Key #} - {# F77F1EDA57EBB1CC -- RabbitMQ Signing Key #} + {# F77F1EDA57EBB1CC -- Launchpad RabbitMQ Erlang PPA key #} + {# F6609E60DC62814E -- PackageCloud RabbitMQ repository key #} {% set base_apt_keys = [ '391A9AA2147192839E9DB0315EDB1B62EC4926EA', '46095ACC8548582C1A2699A9D27D666CD88E42B4', @@ -352,6 +353,7 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom 'A20F259AEB9C94BB', 'F1656F24C74CD1D8', 'F77F1EDA57EBB1CC', + 'F6609E60DC62814E', ] %} {% set remote_apt_keys = [ 'https://packages.grafana.com/gpg.key', @@ -361,6 +363,7 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom {% set base_apt_keys = [ '46095ACC8548582C1A2699A9D27D666CD88E42B4', 'F77F1EDA57EBB1CC', + 'F6609E60DC62814E', ] %} {% set remote_apt_keys = [ 'https://download.docker.com/linux/debian/gpg', From bc97aa584689469eacdb3b80f3d2706f9c6bff96 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 24 Nov 2020 09:27:31 +0000 Subject: [PATCH 06/16] swift: fix group membership in debuntu/binary The UCA packages seem to be removing the swift user from the kolla group. Explicitly add it after installation. Closes-Bug: #1905279 Change-Id: I51c7c6e2f520a582de0409025eb5d1a6da2952a2 (cherry picked from commit aa3208ea0f10889469f864aaccdb593547412d76) --- docker/swift/swift-base/Dockerfile.j2 | 5 +++++ releasenotes/notes/bug-1905279-41d664114ef26a16.yaml | 6 ++++++ 2 files changed, 11 insertions(+) create mode 100644 releasenotes/notes/bug-1905279-41d664114ef26a16.yaml diff --git a/docker/swift/swift-base/Dockerfile.j2 b/docker/swift/swift-base/Dockerfile.j2 index 98940dfacc..bf622fcd4a 100644 --- a/docker/swift/swift-base/Dockerfile.j2 +++ b/docker/swift/swift-base/Dockerfile.j2 @@ -24,6 +24,11 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% endif %} {{ macros.install_packages(swift_base_packages | customizable("packages")) }} + {# NOTE(mgoddard): UCA packages seem to modify the group membership for #} + {# the swift user, removing it from the kolla group. #} + {% if base_package_type == 'deb' %} +RUN usermod --append --groups kolla swift + {% endif %} {% elif install_type == 'source' %} {% if base_package_type == 'rpm' %} {% set swift_base_packages = [ diff --git a/releasenotes/notes/bug-1905279-41d664114ef26a16.yaml b/releasenotes/notes/bug-1905279-41d664114ef26a16.yaml new file mode 100644 index 0000000000..c195902653 --- /dev/null +++ b/releasenotes/notes/bug-1905279-41d664114ef26a16.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Fixes an issue with Swift containers failing to start in Ubuntu binary + images. `LP#1905279 + `__ From a596231e3e12f0b5bbc35a6541c4f0b8a01db98e Mon Sep 17 00:00:00 2001 From: Alexander Evseev Date: Tue, 16 Feb 2021 15:32:01 +0300 Subject: [PATCH 07/16] Add `crudini` to ovs-dpdk containers kolla-ansible runs ovs-dpdkctl.sh inside container, and ovs-dpdkctl.sh requires crudini. Change-Id: Iad84dd12432befe9250c50d4094b1e5e2e0cf8ee (cherry picked from commit 6585332a63b277ff0d63c6c7bd0e77d640579a0a) --- docker/ovsdpdk/ovsdpdk/Dockerfile.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/ovsdpdk/ovsdpdk/Dockerfile.j2 b/docker/ovsdpdk/ovsdpdk/Dockerfile.j2 index b125bd876a..d56088720c 100644 --- a/docker/ovsdpdk/ovsdpdk/Dockerfile.j2 +++ b/docker/ovsdpdk/ovsdpdk/Dockerfile.j2 @@ -19,7 +19,8 @@ RUN echo 'image not yet available for {{ base_distro }}' \ {% elif base_package_type == 'deb' %} {% set ovs_dpdk_packages = [ - 'openvswitch-switch-dpdk' + 'openvswitch-switch-dpdk', + 'crudini' ] %} {{ macros.install_packages(ovs_dpdk_packages | customizable("packages")) }} From 967555e17a42b0f3f6dcb530a84b3581f9c4c89c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Piliszek?= Date: Fri, 30 Apr 2021 15:12:53 +0000 Subject: [PATCH 08/16] [CI] Trust only infra mirrors Infra mirrors get their indices rebuilt to avoid broken indices (due to partial update). Unfortunately, this wipes out the cryptographic signatures. Our approach so far was disabling apt security features globally. However, this is not a valid choice for external repos. It hid an issue we introduced with new RabbitMQ repos missing proper keys installed in the image. This caused permanent failures outside of our CI. Our process should be as close as possible to users' experience. This patch makes CI trust only the mirrors that have their indices rebuilt (so infra mirrors). Change-Id: Ic5abc4b87fd76f87aba383abf43e95ba70629fcb (cherry picked from commit c364c8be85f4653bf7af1c7ac0378823553e7441) --- tests/templates/template_overrides.j2 | 32 +++++++++------------------ 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/tests/templates/template_overrides.j2 b/tests/templates/template_overrides.j2 index 5f5af79ce0..515da5b844 100644 --- a/tests/templates/template_overrides.j2 +++ b/tests/templates/template_overrides.j2 @@ -15,15 +15,6 @@ RUN echo registry={{ nodepool_npmjs_proxy }} > /etc/npmrc \ && ln -s /etc/npmrc /usr/etc/npmrc {% raw %} -{% if base_distro in ['debian', 'ubuntu'] %} -{% endraw %} - -# NOTE(hrw): Debian 'buster' and Ubuntu 18.04 refuse to use unsigned repos -RUN echo 'APT::Get::AllowUnauthenticated "true";' > /etc/apt/apt.conf.d/99allow-unauthenticated \ - && echo 'Acquire::AllowInsecureRepositories "true";' > /etc/apt/apt.conf.d/99allow-insecure-repos - -{% raw %} -{% endif %} {% endblock %} {% block base_centos_repo_overrides_post_copy %} @@ -64,17 +55,17 @@ RUN sed -i \ {% if base_distro == "debian" %} {% endraw %} -RUN sed -i -e "s|http://deb.debian.org|http://{{ nodepool_mirror_host }}|" \ - -e "s|http://security.debian.org|http://{{ nodepool_mirror_host }}|" \ +RUN sed -i -e "s|http://deb.debian.org|[trusted=yes] http://{{ nodepool_mirror_host }}|" \ + -e "s|http://security.debian.org|[trusted=yes] http://{{ nodepool_mirror_host }}|" \ /etc/apt/sources.list {% raw %} {% elif base_distro == "ubuntu" %} {% endraw %} -RUN sed -i -e "s|mirror://mirrors.ubuntu.com/mirrors.txt|http://{{ nodepool_mirror_host }}/ubuntu/|" \ - -e "s|http://ubuntu-cloud.archive.canonical.com/ubuntu|http://{{ nodepool_mirror_host }}/ubuntu-cloud-archive|" \ - -e "s|http://ports.ubuntu.com|http://{{ nodepool_mirror_host }}/ubuntu-ports|" \ +RUN sed -i -e "s|mirror://mirrors.ubuntu.com/mirrors.txt|[trusted=yes] http://{{ nodepool_mirror_host }}/ubuntu/|" \ + -e "s|http://ubuntu-cloud.archive.canonical.com/ubuntu|[trusted=yes] http://{{ nodepool_mirror_host }}/ubuntu-cloud-archive|" \ + -e "s|http://ports.ubuntu.com|[trusted=yes] http://{{ nodepool_mirror_host }}/ubuntu-ports|" \ /etc/apt/sources.list {% raw %} @@ -110,20 +101,17 @@ RUN sed -i \ /etc/yum.repos.d/epel*.repo {% raw %} {% elif base_package_type == 'deb' %} -{% endraw %} -RUN rm -f /etc/apt/apt.conf.d/99allow-unauthenticated /etc/apt/apt.conf.d/99allow-insecure-repos -{% raw %} {% if base_distro == "debian" %} {% endraw %} -RUN sed -i -e "s|http://{{ nodepool_mirror_host }}|http://deb.debian.org|" \ - -e "s|http://{{ nodepool_mirror_host }}|http://security.debian.org|" \ +RUN sed -i -e "s|\[trusted=yes\] http://{{ nodepool_mirror_host }}|http://deb.debian.org|" \ + -e "s|\[trusted=yes\] http://{{ nodepool_mirror_host }}|http://security.debian.org|" \ /etc/apt/sources.list {% raw %} {% elif base_distro == "ubuntu" %} {% endraw %} -RUN sed -i -e "s|http://{{ nodepool_mirror_host }}/ubuntu/|mirror://mirrors.ubuntu.com/mirrors.txt|" \ - -e "s|http://{{ nodepool_mirror_host }}/ubuntu-cloud-archive|http://ubuntu-cloud.archive.canonical.com/ubuntu|" \ - -e "s|http://{{ nodepool_mirror_host }}/ubuntu-ports|http://ports.ubuntu.com|" \ +RUN sed -i -e "s|\[trusted=yes\] http://{{ nodepool_mirror_host }}/ubuntu/|mirror://mirrors.ubuntu.com/mirrors.txt|" \ + -e "s|\[trusted=yes\] http://{{ nodepool_mirror_host }}/ubuntu-cloud-archive|http://ubuntu-cloud.archive.canonical.com/ubuntu|" \ + -e "s|\[trusted=yes\] http://{{ nodepool_mirror_host }}/ubuntu-ports|http://ports.ubuntu.com|" \ /etc/apt/sources.list {% raw %} {% endif %} From 1366332e59ffef4121dced126ef7d19a8b12a72f Mon Sep 17 00:00:00 2001 From: Marcin Juszkiewicz Date: Tue, 4 May 2021 11:30:28 +0200 Subject: [PATCH 09/16] CI: drop bashism from template overrides [[ is bash only In many distributions /bin/sh == bash but not in Debian and derived where it is dash (very simple POSIX shell). Change-Id: I2b084ea78b236623b174473f411bae04e624a3e9 (cherry picked from commit f0c3567740be342355b39a7c7d3ca341d5a4ec0f) --- tests/templates/template_overrides.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/templates/template_overrides.j2 b/tests/templates/template_overrides.j2 index 515da5b844..1bf9d72310 100644 --- a/tests/templates/template_overrides.j2 +++ b/tests/templates/template_overrides.j2 @@ -83,7 +83,7 @@ ENV PIP_TRUSTED_HOST= ENV PIP_EXTRA_INDEX_URL= {% endif %} -RUN if [[ -f /usr/etc/npmrc ]]; then \ +RUN if [ -f /usr/etc/npmrc ]; then \ unlink /usr/etc/npmrc; \ fi \ && rm -f /etc/npmrc From c916ec1eb409f1d111b47a0c187e5fbd81acc9e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Nasiadka?= Date: Mon, 31 May 2021 09:49:30 +0200 Subject: [PATCH 10/16] docs: Update Freenode to OFTC Change-Id: I5a6e7de57e0d123634c37807fd75bcc980abac73 (cherry picked from commit 5338b559ea12af998d89da3f339788a18e775dcd) --- doc/source/contributor/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/contributor/contributing.rst b/doc/source/contributor/contributing.rst index 264e1b86e1..218e72a3af 100644 --- a/doc/source/contributor/contributing.rst +++ b/doc/source/contributor/contributing.rst @@ -22,7 +22,7 @@ Communication ~~~~~~~~~~~~~ IRC Channel - ``#openstack-kolla`` (`channel logs`_) on Freenode + ``#openstack-kolla`` (`channel logs`_) on `OFTC `_ Weekly Meetings On Wednesdays at 15:00 UTC in the IRC channel (`meetings logs`_) From 2c8ea03a7954f37c6503f2cade49c8a53926647e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Piliszek?= Date: Thu, 10 Jun 2021 08:24:30 +0000 Subject: [PATCH 11/16] Enable Debian updates repo Backport below Wallaby switching bullseye to buster. Closes-Bug: #1931544 Change-Id: I8460a4159c8da8ae0b856867e068d30fbffb4d34 (cherry picked from commit 789c9c3b84e84796ac8211112fa29f415f560c88) --- docker/base/sources.list.debian | 3 +++ releasenotes/notes/bug-1931544-5a091735efb6d6dd.yaml | 9 +++++++++ 2 files changed, 12 insertions(+) create mode 100644 releasenotes/notes/bug-1931544-5a091735efb6d6dd.yaml diff --git a/docker/base/sources.list.debian b/docker/base/sources.list.debian index d74f36c332..5ec85ecd86 100644 --- a/docker/base/sources.list.debian +++ b/docker/base/sources.list.debian @@ -4,6 +4,9 @@ deb http://deb.debian.org/debian buster main # debian security updates deb http://security.debian.org/debian-security buster/updates main +# debian updates +deb http://deb.debian.org/debian buster-updates main + # debian backports deb http://deb.debian.org/debian buster-backports main diff --git a/releasenotes/notes/bug-1931544-5a091735efb6d6dd.yaml b/releasenotes/notes/bug-1931544-5a091735efb6d6dd.yaml new file mode 100644 index 0000000000..412923a153 --- /dev/null +++ b/releasenotes/notes/bug-1931544-5a091735efb6d6dd.yaml @@ -0,0 +1,9 @@ +--- +fixes: + - | + Fixes issues arising from the lack of Debian updates repo being enabled. + `LP#1931544 `__ +other: + - | + Debian images enable the Debian updates repo now. This is aligned with + the base Debian image. From ddea2f482adbb88f3effecff4c5674d0cb5ffa11 Mon Sep 17 00:00:00 2001 From: Marcin Juszkiewicz Date: Sat, 8 May 2021 11:03:57 +0200 Subject: [PATCH 12/16] gnocchi-base: numpy provides aarch64 binary wheels There is no need to install libraries to build Numpy as we have binary wheel provided by numpy upstream. Change-Id: I721001f877f6ec2f9a2e425c088a2e22177d2e5b (cherry picked from commit 9bad71d7c3fe3cdd1b128925554c4fdec1420a7f) (cherry picked from commit 858c2748692d0b0c8fed9613a0c0a2514136190d) --- docker/gnocchi/gnocchi-base/Dockerfile.j2 | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/docker/gnocchi/gnocchi-base/Dockerfile.j2 b/docker/gnocchi/gnocchi-base/Dockerfile.j2 index 91b203dabd..8c16047b78 100644 --- a/docker/gnocchi/gnocchi-base/Dockerfile.j2 +++ b/docker/gnocchi/gnocchi-base/Dockerfile.j2 @@ -52,16 +52,6 @@ RUN {{ macros.install_pip(gnocchi_base_pip_packages | customizable("pip_packages 'librados2-devel', ] %} - # on x86-64 pip grabs precompiled numpy/scipy - on aarch64/ppc64le - # it needs to be built - - {% if base_arch != 'x86_64' %} - {% set gnocchi_base_packages = gnocchi_base_packages + [ - 'blas-devel', - 'lapack-devel' - ] %} - {% endif %} - {{ macros.install_packages(gnocchi_base_packages | customizable("packages")) }} RUN mkdir -p /var/www/cgi-bin/gnocchi {% elif base_package_type == 'deb' %} @@ -70,16 +60,6 @@ RUN mkdir -p /var/www/cgi-bin/gnocchi 'librados-dev', ] %} - # on x86-64 pip grabs precompiled numpy/scipy - on aarch64/ppc64le - # it needs to be built - - {% if base_arch != 'x86_64' %} - {% set gnocchi_base_packages = gnocchi_base_packages + [ - 'libblas-dev', - 'liblapack-dev' - ] %} - {% endif %} - {{ macros.install_packages(gnocchi_base_packages | customizable("packages")) }} {% endif %} From d52bb8e7b94e532a3b8c7e9c5ce4560a8824a26d Mon Sep 17 00:00:00 2001 From: Doug Szumski Date: Mon, 7 Jun 2021 10:14:55 +0000 Subject: [PATCH 13/16] Pin td-agent to 4.0.* to fix missing logs The recent inclusion of Fluentd 1.12.x in Centos/Ubuntu packages causes a significant number of logs to fail to appear in the Elasticsearch or Monasca output plugins. The logs appear to sit in a local buffer, and are only flushed when Fluentd is restarted. The same issue persists in the most recent release (1.13.0), so for now we pin the td-agent package to 4.0.* which should restrict Fluentd to the 1.11.x release where we don't see this issue. Backport to Victoria. Change-Id: Iefcdd3100b7e3c5320bc5f1286a18251bdeab885 Closes-Bug: #1930867 (cherry picked from commit 6d0882ef799bb531d007c3e6839ef318fd512d5c) --- docker/base/apt_preferences.ubuntu | 5 +++++ docker/fluentd/Dockerfile.j2 | 3 ++- releasenotes/notes/bug-1930867.yaml | 6 ++++++ 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/bug-1930867.yaml diff --git a/docker/base/apt_preferences.ubuntu b/docker/base/apt_preferences.ubuntu index c8e36affd0..2928ee6620 100644 --- a/docker/base/apt_preferences.ubuntu +++ b/docker/base/apt_preferences.ubuntu @@ -5,3 +5,8 @@ Pin-Priority: 1000 Package: erlang* Pin: version 1:23.* Pin-Priority: 1000 + +# FIXME(mgoddard): Pinning to 4.0.* to avoid bug 1930867. +Package: td-agent* +Pin: version 4.0.* +Pin-Priority: 1000 diff --git a/docker/fluentd/Dockerfile.j2 b/docker/fluentd/Dockerfile.j2 index 52d01c658d..b5ad3d485f 100644 --- a/docker/fluentd/Dockerfile.j2 +++ b/docker/fluentd/Dockerfile.j2 @@ -27,8 +27,9 @@ LABEL fluentd_version="0.14" fluentd_binary="td-agent" ] %} {% if base_arch in ['aarch64', 'x86_64'] %} + # FIXME(mgoddard): Pinning to 4.0.* to avoid bug 1930867. {% set fluentd_packages = fluentd_packages + [ - 'td-agent' + 'td-agent-4.0.*' ] %} {% else %} {% set fluentd_packages = fluentd_packages + [ diff --git a/releasenotes/notes/bug-1930867.yaml b/releasenotes/notes/bug-1930867.yaml new file mode 100644 index 0000000000..fbdfca82dd --- /dev/null +++ b/releasenotes/notes/bug-1930867.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Fixes an issue with logs going missing in the Fluentd pipeline + by pinning td-agent to 4.0.*. + `LP#1930867 `__ From 08e95e5324925b785fa539d2abac21945e58487d Mon Sep 17 00:00:00 2001 From: James Pic Date: Wed, 2 Jun 2021 09:42:35 +0200 Subject: [PATCH 14/16] Fix _get_images_dir when pip-installed with --user There are two cases where pip installs kolla share into ~/.local/share/kolla: - with `pip install --user kolla` - with `pip install kolla` as non-root This patch helps kolla find the share dir in these cases. Closes-Bug: #1930544 Change-Id: I0b3eb6744a34a601266179fb358510a5c500e6ba (cherry picked from commit db78657c231663f3e8ebbc18de43c06ddaa94c97) --- kolla/image/build.py | 1 + releasenotes/notes/bug-1930544-a8926990f3a578a9.yaml | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 releasenotes/notes/bug-1930544-a8926990f3a578a9.yaml diff --git a/kolla/image/build.py b/kolla/image/build.py index 8fa331cf12..f558655595 100755 --- a/kolla/image/build.py +++ b/kolla/image/build.py @@ -790,6 +790,7 @@ def _get_images_dir(self): PROJECT_ROOT, os.path.join(sys.prefix, 'share/kolla'), os.path.join(sys.prefix, 'local/share/kolla'), + os.path.join(os.getenv('HOME', ''), '.local/share/kolla'), # NOTE(zioproto): When Kolla is used within a snap, the env var # $SNAP is the directory where the snap is mounted. # https://github.com/zioproto/snap-kolla diff --git a/releasenotes/notes/bug-1930544-a8926990f3a578a9.yaml b/releasenotes/notes/bug-1930544-a8926990f3a578a9.yaml new file mode 100644 index 0000000000..f10ce242af --- /dev/null +++ b/releasenotes/notes/bug-1930544-a8926990f3a578a9.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fix support for kolla install in ``~/.local``. `LP#1930544 + `__ From 29048206ed67323b318f35f5756d41e8e83f5b8d Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Tue, 15 Jun 2021 09:42:32 +0200 Subject: [PATCH 15/16] Fix build of masakari-monitors image The CentOS 8 Advanced Virtualization repository recently added libvirt 7.0.0. The current version of libvirt-python fails to build against it: running build /usr/bin/pkg-config --print-errors --atleast-version=0.9.11 libvirt /var/lib/kolla/venv/bin/python3.6 generator.py libvirt /usr/share/libvirt/api/libvirt-api.xml Found 480 functions in /usr/share/libvirt/api/libvirt-api.xml Found 0 functions in libvirt-override-api.xml Generated 389 wrapper functions Missing type converters: char ** *:1 char **:1 ERROR: failed virDomainAuthorizedSSHKeysGet ERROR: failed virDomainAuthorizedSSHKeysSet error: command '/var/lib/kolla/venv/bin/python3.6' failed with exit status 1 We need at least libvirt-python v6.10.0. Closes-Bug: #1931817 Change-Id: If69536387513ca221e03a80878016532a41aef87 (cherry picked from commit 715d799ab997cbfe9e444b3f2a72c5dbbbf6728e) --- docker/masakari/masakari-monitors/Dockerfile.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/masakari/masakari-monitors/Dockerfile.j2 b/docker/masakari/masakari-monitors/Dockerfile.j2 index d7098f7370..50fb62aa94 100644 --- a/docker/masakari/masakari-monitors/Dockerfile.j2 +++ b/docker/masakari/masakari-monitors/Dockerfile.j2 @@ -43,6 +43,7 @@ ADD masakari-monitors-archive /masakari-monitors-source ] %} RUN ln -s masakari-monitors-source/* masakari-monitors \ + {% if distro_package_manager == 'dnf' %}&& sed -i -e 's/libvirt-python===.*/libvirt-python===6.10.0/' /requirements/upper-constraints.txt {% endif %}\ && {{ macros.install_pip(masakari_monitors_pip_packages | customizable("pip_packages")) }} \ && mkdir -p /etc/masakari-monitors \ && chown -R masakari: /etc/masakari-monitors From 32eb32f1b66416a2ccd385bd4afee765bf1eb712 Mon Sep 17 00:00:00 2001 From: Scott Shambarger Date: Mon, 24 May 2021 10:51:23 -0700 Subject: [PATCH 16/16] horizon: Correct location of monitoring_policy file Patch to correctly copy monitoring_policy.json into /etc/openstack-dashboard. Policy was misplaced, and not being enforced. Note that by current default policy, admin doesn't not have Monitoring access. Closes-Bug: #1928408 Change-Id: I4faabdfa9c273fc61b536e6ce88b8d71ab2fc581 (cherry picked from commit f68dfb88fdbad51322727575457b3d78a70466b1) --- docker/horizon/extend_start.sh | 2 +- releasenotes/notes/bug-1928408-4a22a85570eee8d6.yaml | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/bug-1928408-4a22a85570eee8d6.yaml diff --git a/docker/horizon/extend_start.sh b/docker/horizon/extend_start.sh index a995bffa8c..b30bb9d7c7 100644 --- a/docker/horizon/extend_start.sh +++ b/docker/horizon/extend_start.sh @@ -160,7 +160,7 @@ function config_monasca_ui { "${SITE_PACKAGES}/openstack_dashboard/local/enabled/_50_admin_add_monitoring_panel.py" config_dashboard "${ENABLE_MONASCA:-no}" \ "${SITE_PACKAGES}/monitoring/conf/monitoring_policy.json" \ - "${SITE_PACKAGES}/openstack_dashboard/conf/monitoring_policy.json" + "/etc/openstack-dashboard/monitoring_policy.json" } function config_murano_dashboard { diff --git a/releasenotes/notes/bug-1928408-4a22a85570eee8d6.yaml b/releasenotes/notes/bug-1928408-4a22a85570eee8d6.yaml new file mode 100644 index 0000000000..4b24aa6682 --- /dev/null +++ b/releasenotes/notes/bug-1928408-4a22a85570eee8d6.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Fixes location of monitoring_policy in Horizon, so access + policy is correctly enforced. Note that by current default, + admin doesn't not have Monitoring access. + `LP#1928408 `__