From d21cab3a1e00686333966ff400bf5e0c3cf70aae Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Wed, 14 May 2025 14:04:38 +0000 Subject: [PATCH 01/13] Revert "[release] Use RDO's Epoxy packages" This reverts commit 64f171ccfa02bfeb9c92a641e186bd987f0099ae. Reason for revert: Let's switch to master Change-Id: I28f474015ffd000a3c2cf4c461c3c2f2b8959b0b --- kolla/common/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kolla/common/config.py b/kolla/common/config.py index c0aa8370a2..989adcc28b 100644 --- a/kolla/common/config.py +++ b/kolla/common/config.py @@ -40,7 +40,7 @@ OPENSTACK_RELEASE = 'master' # This is noarch repository so we will use it on all architectures -DELOREAN_DEPS = "https://trunk.rdoproject.org/centos9-epoxy/" \ +DELOREAN_DEPS = "https://trunk.rdoproject.org/centos9-master/" \ "delorean-deps.repo" # TODO(mandre) check for file integrity instead of downloading from an HTTPS From be41ef297fa97ced646f71db0abe2b0315e108e2 Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Tue, 20 May 2025 13:47:52 +0200 Subject: [PATCH 02/13] kolla-toolbox: Fix apt_preferences_rabbitmq.ubuntu link Change-Id: I1f9bc50b71f55dd5f579d1536a7ab63d5d554d1e --- docker/kolla-toolbox/apt_preferences_rabbitmq.ubuntu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/kolla-toolbox/apt_preferences_rabbitmq.ubuntu b/docker/kolla-toolbox/apt_preferences_rabbitmq.ubuntu index f1cc84d9be..977ded074d 120000 --- a/docker/kolla-toolbox/apt_preferences_rabbitmq.ubuntu +++ b/docker/kolla-toolbox/apt_preferences_rabbitmq.ubuntu @@ -1 +1 @@ -../rabbitmq/apt_preferences_rabbitmq.debian \ No newline at end of file +../rabbitmq/apt_preferences_rabbitmq.ubuntu \ No newline at end of file From c9b665fa8a1ba38f6798349019714ae174b9b85b Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Tue, 20 May 2025 13:49:02 +0200 Subject: [PATCH 03/13] rabbitmq: Update to 4.1 Change-Id: I4f852176918531ea19210961fd1b4fa63bc0ecca --- docker/kolla-toolbox/Dockerfile.j2 | 4 ++-- docker/rabbitmq/Dockerfile.j2 | 4 ++-- docker/rabbitmq/apt_preferences_rabbitmq.debian | 4 ++-- docker/rabbitmq/apt_preferences_rabbitmq.ubuntu | 4 ++-- releasenotes/notes/rabbitmq-41-d881d0b6bbf46446.yaml | 4 ++++ 5 files changed, 12 insertions(+), 8 deletions(-) create mode 100644 releasenotes/notes/rabbitmq-41-d881d0b6bbf46446.yaml diff --git a/docker/kolla-toolbox/Dockerfile.j2 b/docker/kolla-toolbox/Dockerfile.j2 index 0e057871ce..cc8ae1b55b 100644 --- a/docker/kolla-toolbox/Dockerfile.j2 +++ b/docker/kolla-toolbox/Dockerfile.j2 @@ -24,7 +24,7 @@ COPY apt_preferences_rabbitmq.{{ base_distro }} /etc/apt/preferences.d/rabbitmq {% if base_package_type == 'rpm' %} {% set kolla_toolbox_packages = [ 'crudini', - 'erlang-26.2.*', + 'erlang-27.*', 'gcc', 'gdisk', 'git', @@ -38,7 +38,7 @@ COPY apt_preferences_rabbitmq.{{ base_distro }} /etc/apt/preferences.d/rabbitmq 'openvswitch', 'python3.12', 'python3.12-devel', - 'rabbitmq-server-4.0.*' + 'rabbitmq-server-4.1.*' ] %} {% elif base_package_type == 'deb' %} diff --git a/docker/rabbitmq/Dockerfile.j2 b/docker/rabbitmq/Dockerfile.j2 index 194e212f55..3ff00eef7c 100644 --- a/docker/rabbitmq/Dockerfile.j2 +++ b/docker/rabbitmq/Dockerfile.j2 @@ -21,8 +21,8 @@ COPY apt_preferences_rabbitmq.{{ base_distro }} /etc/apt/preferences.d/rabbitmq {% if base_package_type == 'rpm' %} {% set rabbitmq_packages = [ - 'erlang-26.2.*', - 'rabbitmq-server-4.0.*' + 'erlang-27.*', + 'rabbitmq-server-4.1.*' ] %} {% elif base_package_type == 'deb' %} diff --git a/docker/rabbitmq/apt_preferences_rabbitmq.debian b/docker/rabbitmq/apt_preferences_rabbitmq.debian index 52b851f65a..0c254e0ac0 100644 --- a/docker/rabbitmq/apt_preferences_rabbitmq.debian +++ b/docker/rabbitmq/apt_preferences_rabbitmq.debian @@ -1,7 +1,7 @@ Package: rabbitmq-server -Pin: version 4.0.* +Pin: version 4.1.* Pin-Priority: 1000 Package: erlang -Pin: version 26.2.* +Pin: version 27.* Pin-Priority: 1000 diff --git a/docker/rabbitmq/apt_preferences_rabbitmq.ubuntu b/docker/rabbitmq/apt_preferences_rabbitmq.ubuntu index 52b851f65a..0c254e0ac0 100644 --- a/docker/rabbitmq/apt_preferences_rabbitmq.ubuntu +++ b/docker/rabbitmq/apt_preferences_rabbitmq.ubuntu @@ -1,7 +1,7 @@ Package: rabbitmq-server -Pin: version 4.0.* +Pin: version 4.1.* Pin-Priority: 1000 Package: erlang -Pin: version 26.2.* +Pin: version 27.* Pin-Priority: 1000 diff --git a/releasenotes/notes/rabbitmq-41-d881d0b6bbf46446.yaml b/releasenotes/notes/rabbitmq-41-d881d0b6bbf46446.yaml new file mode 100644 index 0000000000..8046fc722e --- /dev/null +++ b/releasenotes/notes/rabbitmq-41-d881d0b6bbf46446.yaml @@ -0,0 +1,4 @@ +--- +upgrade: + - | + ``rabbitmq`` version has been updated to ``4.1``. From 2666f1929b71b2617d2895eb0c3600841253f377 Mon Sep 17 00:00:00 2001 From: Takashi Kajinami Date: Tue, 27 May 2025 16:04:20 +0900 Subject: [PATCH 04/13] Drop environments for nose ... because stestr is now used instead of nose. Also drop the VIRTUAL_ENV environment which is now automatically set by tox. Change-Id: Ied6e20f5b1dc5de384f29d317bec998e86f26efa --- tox.ini | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tox.ini b/tox.ini index e57c38d0fb..4289657a21 100644 --- a/tox.ini +++ b/tox.ini @@ -33,12 +33,8 @@ setenv = commands = oslo_debug_helper -t kolla/tests {posargs} [testenv:cover] -setenv = VIRTUAL_ENV={envdir} - NOSE_WITH_COVERAGE=1 - NOSE_COVER_BRANCHES=1 - NOSE_COVER_HTML=1 - NOSE_COVER_HTML_DIR={toxinidir}/cover - PYTHON=coverage run --source kolla --parallel-mode +setenv = + PYTHON=coverage run --source kolla --parallel-mode commands = stestr run {posargs} coverage combine From bcf37e5016ecf1bc50dea111285cad887fd471b3 Mon Sep 17 00:00:00 2001 From: Michal Arbet Date: Tue, 27 May 2025 09:04:40 +0200 Subject: [PATCH 05/13] [mariadb] Fix incremental backup using correct base dir The original script used --incremental-history-name and compressed full backups as gzip streams. This failed because mariabackup expects an actual decompressed base directory for incremental backups. This patch: - Decompresses the latest full backup into a temporary directory - Uses that as --incremental-basedir - Aligns the backup flow with official Mariabackup documentation: https://mariadb.com/kb/en/incremental-backup-and-restore-with-mariabackup/ This makes the incremental process reliable and fully supported. Closes-Bug: #2111620 Change-Id: I562ce5e54752015863a20bf113bce74a69e02331 --- docker/mariadb/mariadb-server/backup.sh | 59 +++++++++++------ .../mariadb/mariadb-server/backup_replica.sh | 65 +++++++++++-------- .../notes/bug-2111620-e40e1dec6fed309f.yaml | 12 ++++ 3 files changed, 89 insertions(+), 47 deletions(-) create mode 100644 releasenotes/notes/bug-2111620-e40e1dec6fed309f.yaml diff --git a/docker/mariadb/mariadb-server/backup.sh b/docker/mariadb/mariadb-server/backup.sh index 30796fa4ec..be63cd9388 100644 --- a/docker/mariadb/mariadb-server/backup.sh +++ b/docker/mariadb/mariadb-server/backup.sh @@ -6,49 +6,66 @@ set -o pipefail # Execute a full backup backup_full() { echo "Taking a full backup" - LAST_FULL_DATE=$(date +%d-%m-%Y) + LAST_FULL_DATE=$(date +%d-%m-%Y-%s) + BACKUP_FILE="mysqlbackup-${LAST_FULL_DATE}.qp.xbc.xbs.gz" + BACKUP_PATH="$BACKUP_DIR/full-${LAST_FULL_DATE}" + mkdir -p "$BACKUP_PATH" + mariabackup \ --defaults-file=/etc/mysql/my.cnf \ --backup \ --stream=xbstream \ --history=$LAST_FULL_DATE | gzip > \ - $BACKUP_DIR/mysqlbackup-$(date +%d-%m-%Y-%s).qp.xbc.xbs.gz - echo $LAST_FULL_DATE > $BACKUP_DIR/last_full_date + "$BACKUP_PATH/$BACKUP_FILE" + + echo "$BACKUP_PATH/$BACKUP_FILE" > "$BACKUP_DIR/last_full_file" } # Execute an incremental backup backup_incremental() { echo "Taking an incremental backup" - if [ -r $BACKUP_DIR/last_full_date ]; then - LAST_FULL_DATE=$(cat $BACKUP_DIR/last_full_date) - fi - if [ -z $LAST_FULL_DATE ]; then - LAST_FULL_DATE=$(date +%d-%m-%Y) + + if [ ! -r "$BACKUP_DIR/last_full_file" ]; then + echo "Error: No full backup file found." + exit 1 fi + + FULL_BACKUP_FILE=$(cat "$BACKUP_DIR/last_full_file") + LAST_FULL_DATE=$(basename "$(dirname "$FULL_BACKUP_FILE")" | sed 's/^full-//') + NOW=$(date +%H-%M-%S-%d-%m-%Y) + INCR_DIR="$BACKUP_DIR/incr-${NOW}-since-${LAST_FULL_DATE}" + mkdir -p "$INCR_DIR" + + TMP_BASEDIR=$(mktemp -d) + echo "Decompressing full backup to temp dir: $TMP_BASEDIR" + gunzip -c "$FULL_BACKUP_FILE" | mbstream -x -C "$TMP_BASEDIR" + mariabackup \ --defaults-file=/etc/mysql/my.cnf \ --backup \ --stream=xbstream \ - --incremental-history-name=$LAST_FULL_DATE \ - --history=$LAST_FULL_DATE | gzip > \ - $BACKUP_DIR/incremental-$(date +%H)-mysqlbackup-$(date +%d-%m-%Y-%s).qp.xbc.xbs.gz + --incremental-basedir="$TMP_BASEDIR" \ + --history="incr-${NOW}" | gzip > \ + "$INCR_DIR/incremental-${NOW}-mysqlbackup-${LAST_FULL_DATE}.qp.xbc.xbs.gz" + + rm -rf "$TMP_BASEDIR" } BACKUP_DIR=/backup/ -cd $BACKUP_DIR +cd "$BACKUP_DIR" -if [ -n $BACKUP_TYPE ]; then - case $BACKUP_TYPE in +if [ -n "${BACKUP_TYPE:-}" ]; then + case "$BACKUP_TYPE" in "full") - backup_full - ;; + backup_full + ;; "incremental") - backup_incremental - ;; + backup_incremental + ;; *) - echo "Only full or incremental options are supported." - exit 1 - ;; + echo "Only full or incremental options are supported." + exit 1 + ;; esac else echo "You need to specify either full or incremental backup options." diff --git a/docker/mariadb/mariadb-server/backup_replica.sh b/docker/mariadb/mariadb-server/backup_replica.sh index 7f86a18a7c..ebdcbf6b0b 100644 --- a/docker/mariadb/mariadb-server/backup_replica.sh +++ b/docker/mariadb/mariadb-server/backup_replica.sh @@ -23,35 +23,48 @@ cd "${BACKUP_DIR}" backup_full() { echo "Taking a full backup" LAST_FULL_DATE=$(date +%d-%m-%Y-%s) + BACKUP_FILE="backup-full-${LAST_FULL_DATE}.mbs.gz" + BACKUP_PATH="${BACKUP_DIR}/full-${LAST_FULL_DATE}" + mkdir -p "${BACKUP_PATH}" + mariabackup \ --defaults-file="${REPLICA_MY_CNF}" \ --backup \ --stream=mbstream \ - --history="${LAST_FULL_DATE}" | gzip > \ - "${BACKUP_DIR}/mysqlbackup-${LAST_FULL_DATE}.qp.mbc.mbs.gz" && \ - echo "${LAST_FULL_DATE}" > "${BACKUP_DIR}/last_full_date" + --history="${LAST_FULL_DATE}" \ + | gzip > "${BACKUP_PATH}/${BACKUP_FILE}" + + echo "${BACKUP_PATH}/${BACKUP_FILE}" > "${BACKUP_DIR}/last_full_file" } # Execute an incremental backup backup_incremental() { - if [ -r "${BACKUP_DIR}/last_full_date" ]; then - LAST_FULL_DATE=$(cat "${BACKUP_DIR}/last_full_date") - else - LAST_FULL_DATE="" - fi - if [ ! -z "${LAST_FULL_DATE}" ]; then - echo "Taking an incremental backup" - mariabackup \ - --defaults-file="${REPLICA_MY_CNF}" \ - --backup \ - --stream=mbstream \ - --incremental-history-name="${LAST_FULL_DATE}" \ - --history="${LAST_FULL_DATE}" | gzip > \ - "${BACKUP_DIR}/incremental-$(date +%H)-mysqlbackup-${LAST_FULL_DATE}.qp.mbc.mbs.gz" - else - echo "Error: Full backup don't exist." + if [ ! -r "${BACKUP_DIR}/last_full_file" ]; then + echo "Error: No full backup file found." exit 1 fi + + FULL_BACKUP_FILE=$(cat "${BACKUP_DIR}/last_full_file") + LAST_FULL_DATE=$(basename "$(dirname "${FULL_BACKUP_FILE}")" | sed 's/^full-//') + NOW=$(date +%H-%M-%S-%d-%m-%Y) + INCR_DIR="${BACKUP_DIR}/incr-${NOW}-since-${LAST_FULL_DATE}" + mkdir -p "${INCR_DIR}" + + # Temp dir for full base restore + TMP_BASEDIR=$(mktemp -d) + + echo "Decompressing full backup to temp dir: ${TMP_BASEDIR}" + gunzip -c "${FULL_BACKUP_FILE}" | mbstream -x -C "${TMP_BASEDIR}" + + mariabackup \ + --defaults-file="${REPLICA_MY_CNF}" \ + --backup \ + --stream=mbstream \ + --incremental-basedir="${TMP_BASEDIR}" \ + --history="incr-${NOW}" \ + | gzip > "${INCR_DIR}/backup-incremental-${NOW}.mbs.gz" + + rm -rf "${TMP_BASEDIR}" } # Retry logic for database queries @@ -119,15 +132,15 @@ if [ -n "${BACKUP_TYPE}" ]; then get_and_set_replica_server case "${BACKUP_TYPE}" in "full") - backup_full - ;; + backup_full + ;; "incremental") - backup_incremental - ;; + backup_incremental + ;; *) - echo "Only full or incremental options are supported." - exit 1 - ;; + echo "Only full or incremental options are supported." + exit 1 + ;; esac else echo "You need to specify either full or incremental backup options." diff --git a/releasenotes/notes/bug-2111620-e40e1dec6fed309f.yaml b/releasenotes/notes/bug-2111620-e40e1dec6fed309f.yaml new file mode 100644 index 0000000000..e5c9f81987 --- /dev/null +++ b/releasenotes/notes/bug-2111620-e40e1dec6fed309f.yaml @@ -0,0 +1,12 @@ +--- +fixes: + - | + Fixes incremental MariaDB backups by switching to directory-based base backups, + following official Mariabackup recommendations. + `LP#2111620 `__ +upgrade: + - | + Backup files are now stored in timestamped directories and have new filenames. + This does not affect restore, as Kolla Ansible does not support automated restore; + users perform restores manually. A new full backup is recommended after upgrade + to ensure incremental backups work correctly. From b5849adc45e3a39e0837dea3ee8e1fb3cd057dd8 Mon Sep 17 00:00:00 2001 From: Michal Arbet Date: Thu, 29 May 2025 19:53:55 +0200 Subject: [PATCH 06/13] [base] Ensure full control over APT sources for Ubuntu Ubuntu images were occasionally failing to build due to unavailability or slowness of the official Ubuntu security repositories. Although Kolla uses `mirror://` URLs to leverage Ubuntu mirror infrastructure, the original `/etc/apt/sources.list.d/ubuntu.sources` file was still present, causing the system to fallback to default sources. This commit aligns Ubuntu behavior with Debian by explicitly removing the distro `.sources` file and relying solely on the sources configured by Kolla. This ensures consistent and reproducible builds both in CI and on local environments. Closes-Bug: #2112140 Change-Id: I00bcb30662bb12ed8711b28a64ef1941dcce893a --- docker/base/Dockerfile.j2 | 2 +- docker/base/sources.list.ubuntu | 4 +--- releasenotes/notes/bug-2112140-41a1924c260e531c.yaml | 5 +++++ 3 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 releasenotes/notes/bug-2112140-41a1924c260e531c.yaml diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2 index 6bd94a2cc8..eae52d1dbe 100644 --- a/docker/base/Dockerfile.j2 +++ b/docker/base/Dockerfile.j2 @@ -249,7 +249,7 @@ RUN cat /tmp/kolla_bashrc >> /etc/bash.bashrc \ {% block base_ubuntu_package_sources_list %} {% if base_distro == 'debian' or ( base_distro == 'ubuntu' and base_arch == 'x86_64' ) %} -RUN rm -f /etc/apt/sources.list.d/debian.sources +RUN rm -f /etc/apt/sources.list.d/{{ base_distro }}.sources COPY sources.list.{{ base_distro }} /etc/apt/sources.list {% else %} COPY sources.list.{{ base_distro }}.{{ base_arch }} /etc/apt/sources.list diff --git a/docker/base/sources.list.ubuntu b/docker/base/sources.list.ubuntu index 929aebbc01..f0d1f4e371 100644 --- a/docker/base/sources.list.ubuntu +++ b/docker/base/sources.list.ubuntu @@ -3,11 +3,9 @@ # Default repos deb mirror://mirrors.ubuntu.com/mirrors.txt noble main universe deb mirror://mirrors.ubuntu.com/mirrors.txt noble-updates main universe +deb mirror://mirrors.ubuntu.com/mirrors.txt noble-backports main universe deb mirror://mirrors.ubuntu.com/mirrors.txt noble-security main universe -# Backports have a lower priority and must be explicitly installed to be used -deb http://archive.ubuntu.com/ubuntu/ noble-backports main universe - # We need to add the repo for the updated packages they provide. The main ones # are qemu, libvirt, and openvswitch. deb http://ubuntu-cloud.archive.canonical.com/ubuntu noble-updates/epoxy main diff --git a/releasenotes/notes/bug-2112140-41a1924c260e531c.yaml b/releasenotes/notes/bug-2112140-41a1924c260e531c.yaml new file mode 100644 index 0000000000..5fec5c5aa0 --- /dev/null +++ b/releasenotes/notes/bug-2112140-41a1924c260e531c.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Ensures Kolla uses only configured APT mirror on Ubuntu for full source control. + `LP#2112140 `__ From ede71c02c4d38fe0b2513829d6d6ebf889119f1e Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Wed, 14 May 2025 16:00:58 +0200 Subject: [PATCH 07/13] debian: Use Epoxy extrepo Change-Id: I1ba004df309667460aa152928f789aaf6be6051f --- docker/base/Dockerfile.j2 | 2 +- releasenotes/notes/debian-epoxy-f68eb3e911f5fea3.yaml | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/debian-epoxy-f68eb3e911f5fea3.yaml diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2 index eae52d1dbe..2af0243834 100644 --- a/docker/base/Dockerfile.j2 +++ b/docker/base/Dockerfile.j2 @@ -264,7 +264,7 @@ COPY sources.list /etc/apt/sources.list.d/kolla-custom.list RUN apt update \ && apt install -y --no-install-recommends extrepo \ - && extrepo enable openstack_dalmatian \ + && extrepo enable openstack_epoxy \ && apt purge -y extrepo \ && apt --purge autoremove -y \ && apt clean diff --git a/releasenotes/notes/debian-epoxy-f68eb3e911f5fea3.yaml b/releasenotes/notes/debian-epoxy-f68eb3e911f5fea3.yaml new file mode 100644 index 0000000000..2f23c015da --- /dev/null +++ b/releasenotes/notes/debian-epoxy-f68eb3e911f5fea3.yaml @@ -0,0 +1,5 @@ +--- +upgrade: + - | + Debian container image builds now use Epoxy (2025.1) repositories + of Debian OpenStack. From c930c9fb9c435cf0681a15a84814ff91eb055b9f Mon Sep 17 00:00:00 2001 From: Michal Arbet Date: Wed, 21 May 2025 13:56:46 +0200 Subject: [PATCH 08/13] debian: Use bookworkm suite for rabbitmq APT repo Change-Id: Ifd664de82a81830eeda8ccc019b22ee93768635e --- kolla/template/repos.yaml | 4 ++-- kolla/tests/test_methods.py | 4 ++-- .../notes/debian-rabbitmq-bookworm-34b2e5d9d43241da.yaml | 5 +++++ 3 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 releasenotes/notes/debian-rabbitmq-bookworm-34b2e5d9d43241da.yaml diff --git a/kolla/template/repos.yaml b/kolla/template/repos.yaml index c549f76c51..907909eac6 100644 --- a/kolla/template/repos.yaml +++ b/kolla/template/repos.yaml @@ -88,7 +88,7 @@ debian: gpg_key: "proxysql.asc" rabbitmq: url: "https://ppa1.rabbitmq.com/rabbitmq/rabbitmq-server/deb/debian" - suite: "bullseye" + suite: "bookworm" component: "main" gpg_key: "rabbitmq.gpg" @@ -140,7 +140,7 @@ debian-aarch64: gpg_key: "proxysql.asc" rabbitmq: url: "https://ppa1.rabbitmq.com/rabbitmq/rabbitmq-server/deb/debian" - suite: "bullseye" + suite: "bookworm" component: "main" # NOTE(mnasiadka): Since rabbitmq is really noarch and community mirror is not # syncing binary-aarch64 - we're using amd64 here. diff --git a/kolla/tests/test_methods.py b/kolla/tests/test_methods.py index dcdc4f4e95..afa21e2ab2 100644 --- a/kolla/tests/test_methods.py +++ b/kolla/tests/test_methods.py @@ -92,7 +92,7 @@ def test_enable_repos_debian_arch(self): expectCmd += ">>/etc/apt/sources.list.d/rabbitmq.sources && " expectCmd += "echo 'Types: deb' " expectCmd += ">>/etc/apt/sources.list.d/rabbitmq.sources && " - expectCmd += "echo 'Suites: bullseye' " + expectCmd += "echo 'Suites: bookworm' " expectCmd += ">>/etc/apt/sources.list.d/rabbitmq.sources && " expectCmd += "echo 'Signed-By: /etc/kolla/apt-keys/rabbitmq.gpg' " expectCmd += ">>/etc/apt/sources.list.d/rabbitmq.sources && " @@ -139,7 +139,7 @@ def test_enable_repos_debian_multiple(self): expectCmd += ">>/etc/apt/sources.list.d/rabbitmq.sources && " expectCmd += "echo 'Types: deb' " expectCmd += ">>/etc/apt/sources.list.d/rabbitmq.sources && " - expectCmd += "echo 'Suites: bullseye' " + expectCmd += "echo 'Suites: bookworm' " expectCmd += ">>/etc/apt/sources.list.d/rabbitmq.sources && " expectCmd += "echo 'Signed-By: /etc/kolla/apt-keys/rabbitmq.gpg' " expectCmd += ">>/etc/apt/sources.list.d/rabbitmq.sources" diff --git a/releasenotes/notes/debian-rabbitmq-bookworm-34b2e5d9d43241da.yaml b/releasenotes/notes/debian-rabbitmq-bookworm-34b2e5d9d43241da.yaml new file mode 100644 index 0000000000..295de86f0a --- /dev/null +++ b/releasenotes/notes/debian-rabbitmq-bookworm-34b2e5d9d43241da.yaml @@ -0,0 +1,5 @@ +--- +upgrade: + - | + Debian container image builds now use bookworm suite + for RabbitMQ installation. From a4f68e18890fdc837230f20c6fcca2972746ab1a Mon Sep 17 00:00:00 2001 From: Michal Arbet Date: Thu, 29 May 2025 23:46:10 +0200 Subject: [PATCH 09/13] Bump proxysql to 3.0.x This patch bump proxysql 2.7.x to 3.0.x Change-Id: I30d5093c1997b1ac99a762983248b8d29e677ba3 --- doc/source/contributor/versions.rst | 2 +- docker/base/Dockerfile.j2 | 2 +- docker/base/proxysql.repo | 4 ++-- kolla/template/repos.yaml | 8 ++++---- .../notes/bump-proxysql-version-5669b13bcc3d88fe.yaml | 3 +++ 5 files changed, 11 insertions(+), 8 deletions(-) create mode 100644 releasenotes/notes/bump-proxysql-version-5669b13bcc3d88fe.yaml diff --git a/doc/source/contributor/versions.rst b/doc/source/contributor/versions.rst index da02236f84..9de20ec661 100644 --- a/doc/source/contributor/versions.rst +++ b/doc/source/contributor/versions.rst @@ -42,7 +42,7 @@ information about package sources. .. _`Kibana install guide`: https://www.elastic.co/guide/en/kibana/7.10/install.html .. _`Logstash install guide`: https://www.elastic.co/guide/en/logstash/7.9/installing-logstash.html .. _`TreasureData install guide`: https://www.fluentd.org/download -.. _`ProxySQL repository`: https://repo.proxysql.com/ProxySQL/proxysql-2.7.x/ +.. _`ProxySQL repository`: https://repo.proxysql.com/ProxySQL/proxysql-3.0.x/ .. _`Team RabbitMQ 'Cloudsmith' repo (Deb)`: https://www.rabbitmq.com/install-debian.html#apt-cloudsmith .. _`Team RabbitMQ 'Modern Erlang' PPA`: https://launchpad.net/~rabbitmq/+archive/ubuntu/rabbitmq-erlang diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2 index 2af0243834..5765ee8e04 100644 --- a/docker/base/Dockerfile.j2 +++ b/docker/base/Dockerfile.j2 @@ -309,7 +309,7 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom {'name': 'influxdb', 'url': 'https://repos.influxdata.com/influxdata-archive_compat.key'}, {'name': 'mariadb', 'url': 'https://downloads.mariadb.com/MariaDB/mariadb-keyring-2019.gpg', 'type': 'gpg'}, {'name': 'opensearch', 'url': 'https://artifacts.opensearch.org/publickeys/opensearch.pgp'}, - {'name': 'proxysql', 'url': 'https://repo.proxysql.com/ProxySQL/proxysql-2.7.x/repo_pub_key'}, + {'name': 'proxysql', 'url': 'https://repo.proxysql.com/ProxySQL/proxysql-3.0.x/repo_pub_key'}, {'name': 'treasuredata', 'url': 'https://packages.treasuredata.com/GPG-KEY-td-agent'}, ] %} diff --git a/docker/base/proxysql.repo b/docker/base/proxysql.repo index 938f565219..3c09fa7ffa 100644 --- a/docker/base/proxysql.repo +++ b/docker/base/proxysql.repo @@ -1,6 +1,6 @@ [proxysql] name = ProxySQL -baseurl = https://repo.proxysql.com/ProxySQL/proxysql-2.7.x/almalinux/$releasever -gpgkey = https://repo.proxysql.com/ProxySQL/proxysql-2.7.x/repo_pub_key +baseurl = https://repo.proxysql.com/ProxySQL/proxysql-3.0.x/almalinux/$releasever +gpgkey = https://repo.proxysql.com/ProxySQL/proxysql-3.0.x/repo_pub_key gpgcheck = 1 enabled = 0 diff --git a/kolla/template/repos.yaml b/kolla/template/repos.yaml index 907909eac6..6b66aec1df 100644 --- a/kolla/template/repos.yaml +++ b/kolla/template/repos.yaml @@ -82,7 +82,7 @@ debian: component: "main" gpg_key: "opensearch.asc" proxysql: - url: "https://repo.proxysql.com/ProxySQL/proxysql-2.7.x/bookworm/" + url: "https://repo.proxysql.com/ProxySQL/proxysql-3.0.x/bookworm/" suite: "./" component: "" gpg_key: "proxysql.asc" @@ -134,7 +134,7 @@ debian-aarch64: component: "main" gpg_key: "opensearch.asc" proxysql: - url: "https://repo.proxysql.com/ProxySQL/proxysql-2.7.x/bookworm/" + url: "https://repo.proxysql.com/ProxySQL/proxysql-3.0.x/bookworm/" suite: "./" component: "" gpg_key: "proxysql.asc" @@ -228,7 +228,7 @@ ubuntu: component: "main" gpg_key: "opensearch.asc" proxysql: - url: "https://repo.proxysql.com/ProxySQL/proxysql-2.7.x/noble/" + url: "https://repo.proxysql.com/ProxySQL/proxysql-3.0.x/noble/" suite: "./" component: "" gpg_key: "proxysql.asc" @@ -281,7 +281,7 @@ ubuntu-aarch64: component: "main" gpg_key: "opensearch.asc" proxysql: - url: "https://repo.proxysql.com/ProxySQL/proxysql-2.7.x/noble/" + url: "https://repo.proxysql.com/ProxySQL/proxysql-3.0.x/noble/" suite: "./" component: "" gpg_key: "proxysql.asc" diff --git a/releasenotes/notes/bump-proxysql-version-5669b13bcc3d88fe.yaml b/releasenotes/notes/bump-proxysql-version-5669b13bcc3d88fe.yaml new file mode 100644 index 0000000000..791e41d832 --- /dev/null +++ b/releasenotes/notes/bump-proxysql-version-5669b13bcc3d88fe.yaml @@ -0,0 +1,3 @@ +--- +upgrade: + - Upgraded ProxySQL to version ``3.0.x``. From bf86273cafbbff0ba3eb8227d3e3be1854866e5b Mon Sep 17 00:00:00 2001 From: Piotr Milewski Date: Mon, 12 May 2025 09:12:11 +0200 Subject: [PATCH 10/13] Update OpenSearch and OpenSearch Dashboards to version 3.x Depends-On: https://review.opendev.org/c/openstack/kolla-ansible/+/949478 Change-Id: I4507fb60d361ea504459a4b502d6f6c42a327064 --- doc/source/contributor/versions.rst | 2 +- docker/base/Dockerfile.j2 | 6 ++-- docker/base/opensearch.repo | 16 +++++----- kolla/template/repos.yaml | 32 +++++++++---------- .../opensearch-upgrade-9e263a65f015a376.yaml | 4 +++ 5 files changed, 32 insertions(+), 28 deletions(-) create mode 100644 releasenotes/notes/opensearch-upgrade-9e263a65f015a376.yaml diff --git a/doc/source/contributor/versions.rst b/doc/source/contributor/versions.rst index 9de20ec661..9d4e2624f9 100644 --- a/doc/source/contributor/versions.rst +++ b/doc/source/contributor/versions.rst @@ -21,7 +21,7 @@ information about package sources. Logstash 7.x `Logstash install guide`_ MariaDB 10.11 (LTS) `MariaDB Community downloads`_ Galera 26.4 (LTS) `MariaDB Community downloads`_ - OpenSearch 2.x `OpenSearch install guide`_ + OpenSearch 3.x `OpenSearch install guide`_ ProxySQL 2.7.x `ProxySQL repository`_ Rabbitmq 4.0.x - CentOS/Rocky: `Team RabbitMQ 'Cloudsmith' repo (RPM)`_ diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2 index 5765ee8e04..b3181558cd 100644 --- a/docker/base/Dockerfile.j2 +++ b/docker/base/Dockerfile.j2 @@ -155,8 +155,8 @@ RUN rm -f /etc/rpm/macros.image-language-conf \ 'centos9-storage', 'epel', 'influxdb', - 'opensearch-2.x', - 'opensearch-dashboards-2.x', + 'opensearch-3.x', + 'opensearch-dashboards-3.x', ] %} {% if base_arch == 'aarch64' %} @@ -308,7 +308,7 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom {'name': 'grafana', 'url': 'https://rpm.grafana.com/gpg.key'}, {'name': 'influxdb', 'url': 'https://repos.influxdata.com/influxdata-archive_compat.key'}, {'name': 'mariadb', 'url': 'https://downloads.mariadb.com/MariaDB/mariadb-keyring-2019.gpg', 'type': 'gpg'}, - {'name': 'opensearch', 'url': 'https://artifacts.opensearch.org/publickeys/opensearch.pgp'}, + {'name': 'opensearch', 'url': 'https://artifacts.opensearch.org/publickeys/opensearch-release.pgp'}, {'name': 'proxysql', 'url': 'https://repo.proxysql.com/ProxySQL/proxysql-3.0.x/repo_pub_key'}, {'name': 'treasuredata', 'url': 'https://packages.treasuredata.com/GPG-KEY-td-agent'}, ] %} diff --git a/docker/base/opensearch.repo b/docker/base/opensearch.repo index 19f91c3bcb..63caa75f80 100644 --- a/docker/base/opensearch.repo +++ b/docker/base/opensearch.repo @@ -1,19 +1,19 @@ -[opensearch-2.x] -name=OpenSearch 2.x -baseurl=https://artifacts.opensearch.org/releases/bundle/opensearch/2.x/yum +[opensearch-3.x] +name=OpenSearch 3.x +baseurl=https://artifacts.opensearch.org/releases/bundle/opensearch/3.x/yum enabled=0 repo_gpgcheck=1 gpgcheck=1 -gpgkey=https://artifacts.opensearch.org/publickeys/opensearch.pgp +gpgkey=https://artifacts.opensearch.org/publickeys/opensearch-release.pgp autorefresh=1 type=rpm-md -[opensearch-dashboards-2.x] -name=OpenSearch Dashboards 2.x -baseurl=https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/2.x/yum +[opensearch-dashboards-3.x] +name=OpenSearch Dashboards 3.x +baseurl=https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/3.x/yum enabled=0 repo_gpgcheck=1 gpgcheck=1 -gpgkey=https://artifacts.opensearch.org/publickeys/opensearch.pgp +gpgkey=https://artifacts.opensearch.org/publickeys/opensearch-release.pgp autorefresh=1 type=rpm-md diff --git a/kolla/template/repos.yaml b/kolla/template/repos.yaml index 6b66aec1df..d0bfed99f6 100644 --- a/kolla/template/repos.yaml +++ b/kolla/template/repos.yaml @@ -11,8 +11,8 @@ centos: hacluster: "highavailability" influxdb: "influxdb" mariadb: "mariadb" - opensearch: "opensearch-2.x" - opensearch-dashboards: "opensearch-dashboards-2.x" + opensearch: "opensearch-3.x" + opensearch-dashboards: "opensearch-dashboards-3.x" openvswitch: "centos-nfv-openvswitch" opstools: "centos-opstools" proxysql: "proxysql" @@ -30,8 +30,8 @@ centos-aarch64: hacluster: "highavailability" influxdb: "influxdb" mariadb: "mariadb" - opensearch: "opensearch-2.x" - opensearch-dashboards: "opensearch-dashboards-2.x" + opensearch: "opensearch-3.x" + opensearch-dashboards: "opensearch-dashboards-3.x" openvswitch: "centos-nfv-openvswitch" opstools: "centos-opstools" proxysql: "proxysql" @@ -72,12 +72,12 @@ debian: component: "main" gpg_key: "mariadb.gpg" opensearch: - url: "https://artifacts.opensearch.org/releases/bundle/opensearch/2.x/apt/" + url: "https://artifacts.opensearch.org/releases/bundle/opensearch/3.x/apt/" suite: "stable" component: "main" gpg_key: "opensearch.asc" opensearch-dashboards: - url: "https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/2.x/apt/" + url: "https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/3.x/apt/" suite: "stable" component: "main" gpg_key: "opensearch.asc" @@ -124,12 +124,12 @@ debian-aarch64: component: "main" gpg_key: "mariadb.gpg" opensearch: - url: "https://artifacts.opensearch.org/releases/bundle/opensearch/2.x/apt/" + url: "https://artifacts.opensearch.org/releases/bundle/opensearch/3.x/apt/" suite: "stable" component: "main" gpg_key: "opensearch.asc" opensearch-dashboards: - url: "https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/2.x/apt/" + url: "https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/3.x/apt/" suite: "stable" component: "main" gpg_key: "opensearch.asc" @@ -159,8 +159,8 @@ rocky: hacluster: "highavailability" influxdb: "influxdb" mariadb: "mariadb" - opensearch: "opensearch-2.x" - opensearch-dashboards: "opensearch-dashboards-2.x" + opensearch: "opensearch-3.x" + opensearch-dashboards: "opensearch-dashboards-3.x" openvswitch: "centos-nfv-openvswitch" opstools: "centos-opstools" proxysql: "proxysql" @@ -178,8 +178,8 @@ rocky-aarch64: influxdb: "influxdb" hacluster: "highavailability" mariadb: "mariadb" - opensearch: "opensearch-2.x" - opensearch-dashboards: "opensearch-dashboards-2.x" + opensearch: "opensearch-3.x" + opensearch-dashboards: "opensearch-dashboards-3.x" openvswitch: "centos-nfv-openvswitch" opstools: "centos-opstools" proxysql: "proxysql" @@ -218,12 +218,12 @@ ubuntu: component: "main" gpg_key: "mariadb.gpg" opensearch: - url: "https://artifacts.opensearch.org/releases/bundle/opensearch/2.x/apt/" + url: "https://artifacts.opensearch.org/releases/bundle/opensearch/3.x/apt/" suite: "stable" component: "main" gpg_key: "opensearch.asc" opensearch-dashboards: - url: "https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/2.x/apt/" + url: "https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/3.x/apt/" suite: "stable" component: "main" gpg_key: "opensearch.asc" @@ -271,12 +271,12 @@ ubuntu-aarch64: component: "main" gpg_key: "mariadb.gpg" opensearch: - url: "https://artifacts.opensearch.org/releases/bundle/opensearch/2.x/apt/" + url: "https://artifacts.opensearch.org/releases/bundle/opensearch/3.x/apt/" suite: "stable" component: "main" gpg_key: "opensearch.asc" opensearch-dashboards: - url: "https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/2.x/apt/" + url: "https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/3.x/apt/" suite: "stable" component: "main" gpg_key: "opensearch.asc" diff --git a/releasenotes/notes/opensearch-upgrade-9e263a65f015a376.yaml b/releasenotes/notes/opensearch-upgrade-9e263a65f015a376.yaml new file mode 100644 index 0000000000..d7e00e3315 --- /dev/null +++ b/releasenotes/notes/opensearch-upgrade-9e263a65f015a376.yaml @@ -0,0 +1,4 @@ +--- +upgrade: + - | + Update OpenSearch and OpenSearch Dashboards to version 3.x. From 2b42c958effd29d7bb397c5ef79de4a58911d14f Mon Sep 17 00:00:00 2001 From: "Dr. Jens Harbott" Date: Fri, 30 May 2025 20:35:52 +0200 Subject: [PATCH 11/13] zuul: make failing job temporarily non-voting We have multiple CI issues currently, make the kolla-ansible-debian-upgrade job non-voting so that we can merge fixes in order. Signed-off-by: Dr. Jens Harbott Change-Id: I2d7137f3d01eb28f7693113fb690069e2760820f --- .zuul.d/debian.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.zuul.d/debian.yaml b/.zuul.d/debian.yaml index 715e7acdc7..2a2e526a34 100644 --- a/.zuul.d/debian.yaml +++ b/.zuul.d/debian.yaml @@ -5,7 +5,8 @@ - kolla-build-debian - kolla-build-debian-podman - kolla-ansible-debian - - kolla-ansible-debian-upgrade + - kolla-ansible-debian-upgrade: + voting: false check-arm64: jobs: - kolla-build-debian-aarch64 @@ -15,7 +16,7 @@ - kolla-build-debian - kolla-build-debian-podman - kolla-ansible-debian - - kolla-ansible-debian-upgrade + # - kolla-ansible-debian-upgrade periodic: jobs: - kolla-publish-debian-quay From b643e8f9fd3e9d8e881e112e53183db743f1e5d3 Mon Sep 17 00:00:00 2001 From: Michal Arbet Date: Wed, 4 Jun 2025 09:40:03 +0200 Subject: [PATCH 12/13] Fix ProxySQL rule sanity check to support username-based routing Kolla Ansible now generates ProxySQL query rules not only based on the schema name (schemaname), but also on the username. This is needed to properly route queries that are executed before a database schema is selected, such as `SET AUTOCOMMIT` or `ROLLBACK`. To support this, the ProxySQL config sanity check must correctly deduplicate rules based on both `schemaname` and `username`, and avoid false positives when both types are used simultaneously. This change adjusts the logic in `kolla_proxysql_config_sync` to: - recognize rules by key `schema:` or `user:` - assign unique rule IDs only to truly distinct rules - log and skip exact duplicates with appropriate context This fix is required to support the Kolla Ansible patch below. Needed-By: https://review.opendev.org/c/openstack/kolla-ansible/+/951599 Change-Id: I99acd1984ee555a5b6c731e6ee460a33677060d0 --- docker/proxysql/kolla_proxysql_config_sync | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/docker/proxysql/kolla_proxysql_config_sync b/docker/proxysql/kolla_proxysql_config_sync index d6d531f30e..094642c8a1 100644 --- a/docker/proxysql/kolla_proxysql_config_sync +++ b/docker/proxysql/kolla_proxysql_config_sync @@ -93,14 +93,24 @@ class ProxySQLConfig: rules = list() rule_id = 1 for rule in self.config['mysql_query_rules']: - if rule['schemaname'] not in rules_added: - rules_added.append(rule['schemaname']) + if 'schemaname' in rule: + key = f"schema:{rule['schemaname']}" + elif 'username' in rule: + key = f"user:{rule['username']}" + else: + LOG.warning( + f"Rule without schemaname or username found, skipping: {rule}" + ) + continue + + if key not in rules_added: + rules_added.append(key) rule['rule_id'] = rule_id rules.append(rule) rule_id += 1 else: - LOG.warning("Rule witch schemaname {} already exist, ignoring." - .format(rule['schemaname'])) + LOG.warning("Duplicate rule for {}, ignoring.".format(key)) + self.config['mysql_query_rules'] = rules def _write_dict(self, key, value): From 6a1d383c48eda9fd9c1c55474d29bdba0f06a52f Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Thu, 5 Jun 2025 16:06:40 +0100 Subject: [PATCH 13/13] Remove dibbler from neutron-l3-agent The dibbler-client package has been removed from upstream Rocky repos, breaking neutron-l3-agent container image builds on RL9. Since the 2025.1 release, package is no longer used in Neutron has been removed because the project is stale and functionality was untested [1]. This change removes the package from the image build for all distributions. [1] https://review.opendev.org/c/openstack/neutron/+/934283 Closes-Bug: #2112575 Change-Id: I35b233a499c6bf1861b376fc441dc8081cfaf5d6 Signed-off-by: Alex Welsh --- docker/neutron/neutron-l3-agent/Dockerfile.j2 | 4 +--- releasenotes/notes/remove-dibbler-537b80d88f9984f5.yaml | 6 ++++++ 2 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 releasenotes/notes/remove-dibbler-537b80d88f9984f5.yaml diff --git a/docker/neutron/neutron-l3-agent/Dockerfile.j2 b/docker/neutron/neutron-l3-agent/Dockerfile.j2 index bb36adf176..2787dae5b7 100644 --- a/docker/neutron/neutron-l3-agent/Dockerfile.j2 +++ b/docker/neutron/neutron-l3-agent/Dockerfile.j2 @@ -10,14 +10,12 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% if base_package_type == 'rpm' %} {% set neutron_l3_agent_packages = [ 'libreswan', - 'radvd', - 'dibbler-client', + 'radvd' ] %} {% elif base_package_type == 'deb' %} {% set neutron_l3_agent_packages = [ 'strongswan', 'radvd', - 'dibbler-client', 'libnetfilter-log1' ] %} {% endif %} diff --git a/releasenotes/notes/remove-dibbler-537b80d88f9984f5.yaml b/releasenotes/notes/remove-dibbler-537b80d88f9984f5.yaml new file mode 100644 index 0000000000..37e3cccc90 --- /dev/null +++ b/releasenotes/notes/remove-dibbler-537b80d88f9984f5.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Fix ``neutron-l3-agent`` image builds on Rocky Linux by removing + ``dibbler-client`` dependency. The package is no longer used in Neutron. + `LP#2112575 `__