Synchronise master with upstream

.zuul.d/debian.yaml

@@ -5,7 +5,8 @@
         - kolla-build-debian
         - kolla-build-debian-podman
         - kolla-ansible-debian
-        - kolla-ansible-debian-upgrade
+        - kolla-ansible-debian-upgrade:
+            voting: false
     check-arm64:
       jobs:
         - kolla-build-debian-aarch64
@@ -15,7 +16,7 @@
         - kolla-build-debian
         - kolla-build-debian-podman
         - kolla-ansible-debian
-        - kolla-ansible-debian-upgrade
+        # - kolla-ansible-debian-upgrade
     periodic:
       jobs:
         - kolla-publish-debian-quay

doc/source/contributor/versions.rst

@@ -21,7 +21,7 @@ information about package sources.
  Logstash       7.x                `Logstash install guide`_
  MariaDB        10.11 (LTS)        `MariaDB Community downloads`_
  Galera         26.4 (LTS)         `MariaDB Community downloads`_
- OpenSearch     2.x                `OpenSearch install guide`_
+ OpenSearch     3.x                `OpenSearch install guide`_
  ProxySQL       2.7.x              `ProxySQL repository`_
  Rabbitmq       4.0.x              - CentOS/Rocky:
                                      `Team RabbitMQ 'Cloudsmith' repo (RPM)`_
@@ -42,7 +42,7 @@ information about package sources.
 .. _`Kibana install guide`: https://www.elastic.co/guide/en/kibana/7.10/install.html
 .. _`Logstash install guide`: https://www.elastic.co/guide/en/logstash/7.9/installing-logstash.html
 .. _`TreasureData install guide`: https://www.fluentd.org/download
-.. _`ProxySQL repository`: https://repo.proxysql.com/ProxySQL/proxysql-2.7.x/
+.. _`ProxySQL repository`: https://repo.proxysql.com/ProxySQL/proxysql-3.0.x/
 
 .. _`Team RabbitMQ 'Cloudsmith' repo (Deb)`: https://www.rabbitmq.com/install-debian.html#apt-cloudsmith
 .. _`Team RabbitMQ 'Modern Erlang' PPA`: https://launchpad.net/~rabbitmq/+archive/ubuntu/rabbitmq-erlang

docker/base/Dockerfile.j2

@@ -155,8 +155,8 @@ RUN rm -f /etc/rpm/macros.image-language-conf \
     'centos9-storage',
     'epel',
     'influxdb',
-    'opensearch-2.x',
-    'opensearch-dashboards-2.x',
+    'opensearch-3.x',
+    'opensearch-dashboards-3.x',
 ] %}
 
 {% if base_arch == 'aarch64' %}
@@ -249,7 +249,7 @@ RUN cat /tmp/kolla_bashrc >> /etc/bash.bashrc \
 
 {% block base_ubuntu_package_sources_list %}
 {% if base_distro == 'debian' or ( base_distro == 'ubuntu' and base_arch == 'x86_64' ) %}
-RUN rm -f /etc/apt/sources.list.d/debian.sources
+RUN rm -f /etc/apt/sources.list.d/{{ base_distro }}.sources
 COPY sources.list.{{ base_distro }} /etc/apt/sources.list
 {% else %}
 COPY sources.list.{{ base_distro }}.{{ base_arch }} /etc/apt/sources.list
@@ -264,7 +264,7 @@ COPY sources.list /etc/apt/sources.list.d/kolla-custom.list
 
 RUN apt update \
  && apt install -y --no-install-recommends extrepo \
- && extrepo enable openstack_dalmatian \
+ && extrepo enable openstack_epoxy \
  && apt purge -y extrepo \
  && apt --purge autoremove -y \
  && apt clean
@@ -308,8 +308,8 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom
    {'name': 'grafana', 'url': 'https://rpm.grafana.com/gpg.key'},
    {'name': 'influxdb', 'url': 'https://repos.influxdata.com/influxdata-archive_compat.key'},
    {'name': 'mariadb', 'url': 'https://downloads.mariadb.com/MariaDB/mariadb-keyring-2019.gpg', 'type': 'gpg'},
-   {'name': 'opensearch', 'url': 'https://artifacts.opensearch.org/publickeys/opensearch.pgp'},
-   {'name': 'proxysql', 'url': 'https://repo.proxysql.com/ProxySQL/proxysql-2.7.x/repo_pub_key'},
+   {'name': 'opensearch', 'url': 'https://artifacts.opensearch.org/publickeys/opensearch-release.pgp'},
+   {'name': 'proxysql', 'url': 'https://repo.proxysql.com/ProxySQL/proxysql-3.0.x/repo_pub_key'},
    {'name': 'treasuredata', 'url': 'https://packages.treasuredata.com/GPG-KEY-td-agent'},
 ] %}
 

docker/base/opensearch.repo

@@ -1,19 +1,19 @@
-[opensearch-2.x]
-name=OpenSearch 2.x
-baseurl=https://artifacts.opensearch.org/releases/bundle/opensearch/2.x/yum
+[opensearch-3.x]
+name=OpenSearch 3.x
+baseurl=https://artifacts.opensearch.org/releases/bundle/opensearch/3.x/yum
 enabled=0
 repo_gpgcheck=1
 gpgcheck=1
-gpgkey=https://artifacts.opensearch.org/publickeys/opensearch.pgp
+gpgkey=https://artifacts.opensearch.org/publickeys/opensearch-release.pgp
 autorefresh=1
 type=rpm-md
 
-[opensearch-dashboards-2.x]
-name=OpenSearch Dashboards 2.x
-baseurl=https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/2.x/yum
+[opensearch-dashboards-3.x]
+name=OpenSearch Dashboards 3.x
+baseurl=https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/3.x/yum
 enabled=0
 repo_gpgcheck=1
 gpgcheck=1
-gpgkey=https://artifacts.opensearch.org/publickeys/opensearch.pgp
+gpgkey=https://artifacts.opensearch.org/publickeys/opensearch-release.pgp
 autorefresh=1
 type=rpm-md

docker/base/proxysql.repo

@@ -1,6 +1,6 @@
 [proxysql]
 name = ProxySQL
-baseurl = https://repo.proxysql.com/ProxySQL/proxysql-2.7.x/almalinux/$releasever
-gpgkey = https://repo.proxysql.com/ProxySQL/proxysql-2.7.x/repo_pub_key
+baseurl = https://repo.proxysql.com/ProxySQL/proxysql-3.0.x/almalinux/$releasever
+gpgkey = https://repo.proxysql.com/ProxySQL/proxysql-3.0.x/repo_pub_key
 gpgcheck = 1
 enabled = 0

docker/base/sources.list.ubuntu

@@ -3,11 +3,9 @@
 # Default repos
 deb mirror://mirrors.ubuntu.com/mirrors.txt noble main universe
 deb mirror://mirrors.ubuntu.com/mirrors.txt noble-updates main universe
+deb mirror://mirrors.ubuntu.com/mirrors.txt noble-backports main universe
 deb mirror://mirrors.ubuntu.com/mirrors.txt noble-security main universe
 
-# Backports have a lower priority and must be explicitly installed to be used
-deb http://archive.ubuntu.com/ubuntu/ noble-backports main universe
-
 # We need to add the repo for the updated packages they provide. The main ones
 # are qemu, libvirt, and openvswitch.
 deb http://ubuntu-cloud.archive.canonical.com/ubuntu noble-updates/epoxy main

docker/kolla-toolbox/Dockerfile.j2

@@ -24,7 +24,7 @@ COPY apt_preferences_rabbitmq.{{ base_distro }} /etc/apt/preferences.d/rabbitmq
 {% if base_package_type == 'rpm' %}
     {% set kolla_toolbox_packages = [
         'crudini',
-        'erlang-26.2.*',
+        'erlang-27.*',
         'gcc',
         'gdisk',
         'git',
@@ -38,7 +38,7 @@ COPY apt_preferences_rabbitmq.{{ base_distro }} /etc/apt/preferences.d/rabbitmq
         'openvswitch',
         'python3.12',
         'python3.12-devel',
-        'rabbitmq-server-4.0.*'
+        'rabbitmq-server-4.1.*'
     ] %}
 
 {% elif base_package_type == 'deb' %}

docker/kolla-toolbox/apt_preferences_rabbitmq.ubuntu

@@ -1 +1 @@
-../rabbitmq/apt_preferences_rabbitmq.debian
+../rabbitmq/apt_preferences_rabbitmq.ubuntu

docker/mariadb/mariadb-server/backup.sh

@@ -6,49 +6,66 @@ set -o pipefail
 # Execute a full backup
 backup_full() {
     echo "Taking a full backup"
-    LAST_FULL_DATE=$(date +%d-%m-%Y)
+    LAST_FULL_DATE=$(date +%d-%m-%Y-%s)
+    BACKUP_FILE="mysqlbackup-${LAST_FULL_DATE}.qp.xbc.xbs.gz"
+    BACKUP_PATH="$BACKUP_DIR/full-${LAST_FULL_DATE}"
+    mkdir -p "$BACKUP_PATH"
+
     mariabackup \
         --defaults-file=/etc/mysql/my.cnf \
         --backup \
         --stream=xbstream \
         --history=$LAST_FULL_DATE | gzip > \
-        $BACKUP_DIR/mysqlbackup-$(date +%d-%m-%Y-%s).qp.xbc.xbs.gz
-    echo $LAST_FULL_DATE > $BACKUP_DIR/last_full_date
+        "$BACKUP_PATH/$BACKUP_FILE"
+
+    echo "$BACKUP_PATH/$BACKUP_FILE" > "$BACKUP_DIR/last_full_file"
 }
 
 # Execute an incremental backup
 backup_incremental() {
     echo "Taking an incremental backup"
-    if [ -r $BACKUP_DIR/last_full_date ]; then
-        LAST_FULL_DATE=$(cat $BACKUP_DIR/last_full_date)
-    fi
-    if [ -z $LAST_FULL_DATE ]; then
-        LAST_FULL_DATE=$(date +%d-%m-%Y)
+
+    if [ ! -r "$BACKUP_DIR/last_full_file" ]; then
+        echo "Error: No full backup file found."
+        exit 1
     fi
+
+    FULL_BACKUP_FILE=$(cat "$BACKUP_DIR/last_full_file")
+    LAST_FULL_DATE=$(basename "$(dirname "$FULL_BACKUP_FILE")" | sed 's/^full-//')
+    NOW=$(date +%H-%M-%S-%d-%m-%Y)
+    INCR_DIR="$BACKUP_DIR/incr-${NOW}-since-${LAST_FULL_DATE}"
+    mkdir -p "$INCR_DIR"
+
+    TMP_BASEDIR=$(mktemp -d)
+    echo "Decompressing full backup to temp dir: $TMP_BASEDIR"
+    gunzip -c "$FULL_BACKUP_FILE" | mbstream -x -C "$TMP_BASEDIR"
+
     mariabackup \
         --defaults-file=/etc/mysql/my.cnf \
         --backup \
         --stream=xbstream \
-        --incremental-history-name=$LAST_FULL_DATE \
-        --history=$LAST_FULL_DATE | gzip > \
-        $BACKUP_DIR/incremental-$(date +%H)-mysqlbackup-$(date +%d-%m-%Y-%s).qp.xbc.xbs.gz
+        --incremental-basedir="$TMP_BASEDIR" \
+        --history="incr-${NOW}" | gzip > \
+        "$INCR_DIR/incremental-${NOW}-mysqlbackup-${LAST_FULL_DATE}.qp.xbc.xbs.gz"
+
+    rm -rf "$TMP_BASEDIR"
 }
 
 BACKUP_DIR=/backup/
-cd $BACKUP_DIR
+cd "$BACKUP_DIR"
 
-if [ -n $BACKUP_TYPE ]; then
-    case $BACKUP_TYPE in
+if [ -n "${BACKUP_TYPE:-}" ]; then
+    case "$BACKUP_TYPE" in
         "full")
-        backup_full
-        ;;
+            backup_full
+            ;;
         "incremental")
-        backup_incremental
-        ;;
+            backup_incremental
+            ;;
         *)
-        echo "Only full or incremental options are supported."
-        exit 1
-        ;;
+            echo "Only full or incremental options are supported."
+            exit 1
+            ;;
     esac
 else
     echo "You need to specify either full or incremental backup options."

docker/mariadb/mariadb-server/backup_replica.sh

@@ -23,35 +23,48 @@ cd "${BACKUP_DIR}"
 backup_full() {
     echo "Taking a full backup"
     LAST_FULL_DATE=$(date +%d-%m-%Y-%s)
+    BACKUP_FILE="backup-full-${LAST_FULL_DATE}.mbs.gz"
+    BACKUP_PATH="${BACKUP_DIR}/full-${LAST_FULL_DATE}"
+    mkdir -p "${BACKUP_PATH}"
+
     mariabackup \
         --defaults-file="${REPLICA_MY_CNF}" \
         --backup \
         --stream=mbstream \
-        --history="${LAST_FULL_DATE}" | gzip > \
-        "${BACKUP_DIR}/mysqlbackup-${LAST_FULL_DATE}.qp.mbc.mbs.gz" && \
-    echo "${LAST_FULL_DATE}" > "${BACKUP_DIR}/last_full_date"
+        --history="${LAST_FULL_DATE}" \
+        | gzip > "${BACKUP_PATH}/${BACKUP_FILE}"
+
+    echo "${BACKUP_PATH}/${BACKUP_FILE}" > "${BACKUP_DIR}/last_full_file"
 }
 
 # Execute an incremental backup
 backup_incremental() {
-    if [ -r "${BACKUP_DIR}/last_full_date" ]; then
-        LAST_FULL_DATE=$(cat "${BACKUP_DIR}/last_full_date")
-    else
-        LAST_FULL_DATE=""
-    fi
-    if [ ! -z "${LAST_FULL_DATE}" ]; then
-        echo "Taking an incremental backup"
-        mariabackup \
-            --defaults-file="${REPLICA_MY_CNF}" \
-            --backup \
-            --stream=mbstream \
-            --incremental-history-name="${LAST_FULL_DATE}" \
-            --history="${LAST_FULL_DATE}" | gzip > \
-            "${BACKUP_DIR}/incremental-$(date +%H)-mysqlbackup-${LAST_FULL_DATE}.qp.mbc.mbs.gz"
-    else
-        echo "Error: Full backup don't exist."
+    if [ ! -r "${BACKUP_DIR}/last_full_file" ]; then
+        echo "Error: No full backup file found."
         exit 1
     fi
+
+    FULL_BACKUP_FILE=$(cat "${BACKUP_DIR}/last_full_file")
+    LAST_FULL_DATE=$(basename "$(dirname "${FULL_BACKUP_FILE}")" | sed 's/^full-//')
+    NOW=$(date +%H-%M-%S-%d-%m-%Y)
+    INCR_DIR="${BACKUP_DIR}/incr-${NOW}-since-${LAST_FULL_DATE}"
+    mkdir -p "${INCR_DIR}"
+
+    # Temp dir for full base restore
+    TMP_BASEDIR=$(mktemp -d)
+
+    echo "Decompressing full backup to temp dir: ${TMP_BASEDIR}"
+    gunzip -c "${FULL_BACKUP_FILE}" | mbstream -x -C "${TMP_BASEDIR}"
+
+    mariabackup \
+        --defaults-file="${REPLICA_MY_CNF}" \
+        --backup \
+        --stream=mbstream \
+        --incremental-basedir="${TMP_BASEDIR}" \
+        --history="incr-${NOW}" \
+        | gzip > "${INCR_DIR}/backup-incremental-${NOW}.mbs.gz"
+
+    rm -rf "${TMP_BASEDIR}"
 }
 
 # Retry logic for database queries
@@ -119,15 +132,15 @@ if [ -n "${BACKUP_TYPE}" ]; then
     get_and_set_replica_server
     case "${BACKUP_TYPE}" in
         "full")
-        backup_full
-        ;;
+            backup_full
+            ;;
         "incremental")
-        backup_incremental
-        ;;
+            backup_incremental
+            ;;
         *)
-        echo "Only full or incremental options are supported."
-        exit 1
-        ;;
+            echo "Only full or incremental options are supported."
+            exit 1
+            ;;
     esac
 else
     echo "You need to specify either full or incremental backup options."

docker/neutron/neutron-l3-agent/Dockerfile.j2

@@ -10,14 +10,12 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 {% if base_package_type == 'rpm' %}
    {% set neutron_l3_agent_packages = [
       'libreswan',
-      'radvd',
-      'dibbler-client',
+      'radvd'
    ] %}
 {% elif base_package_type == 'deb' %}
    {% set neutron_l3_agent_packages = [
       'strongswan',
       'radvd',
-      'dibbler-client',
       'libnetfilter-log1'
    ] %}
 {% endif %}

docker/proxysql/kolla_proxysql_config_sync

@@ -93,14 +93,24 @@ class ProxySQLConfig:
         rules = list()
         rule_id = 1
         for rule in self.config['mysql_query_rules']:
-            if rule['schemaname'] not in rules_added:
-                rules_added.append(rule['schemaname'])
+            if 'schemaname' in rule:
+                key = f"schema:{rule['schemaname']}"
+            elif 'username' in rule:
+                key = f"user:{rule['username']}"
+            else:
+                LOG.warning(
+                    f"Rule without schemaname or username found, skipping: {rule}"
+                )
+                continue
+
+            if key not in rules_added:
+                rules_added.append(key)
                 rule['rule_id'] = rule_id
                 rules.append(rule)
                 rule_id += 1
             else:
-                LOG.warning("Rule witch schemaname {} already exist, ignoring."
-                            .format(rule['schemaname']))
+                LOG.warning("Duplicate rule for {}, ignoring.".format(key))
+
         self.config['mysql_query_rules'] = rules
 
     def _write_dict(self, key, value):

docker/rabbitmq/Dockerfile.j2

@@ -21,8 +21,8 @@ COPY apt_preferences_rabbitmq.{{ base_distro }} /etc/apt/preferences.d/rabbitmq
 {% if base_package_type == 'rpm' %}
 
     {% set rabbitmq_packages = [
-        'erlang-26.2.*',
-        'rabbitmq-server-4.0.*'
+        'erlang-27.*',
+        'rabbitmq-server-4.1.*'
     ] %}
 
 {% elif base_package_type == 'deb' %}

docker/rabbitmq/apt_preferences_rabbitmq.debian

@@ -1,7 +1,7 @@
 Package: rabbitmq-server
-Pin: version 4.0.*
+Pin: version 4.1.*
 Pin-Priority: 1000
 
 Package: erlang
-Pin: version 26.2.*
+Pin: version 27.*
 Pin-Priority: 1000