wallaby: Sync upstream stable/wallaby

docker/base/Dockerfile.j2

@@ -93,12 +93,14 @@ COPY dnf.conf /etc/dnf/dnf.conf
 {% elif base_arch == 'aarch64' %}
     {% set base_yum_repo_files = [
         'elasticsearch.repo',
+        'erlang-solutions.repo',
         'grafana.repo',
         'rabbitmq_rabbitmq-server.repo',
         'td.repo',
     ] %}
 
     {% set base_yum_repo_keys = [
+        'https://packages.erlang-solutions.com/rpm/erlang_solutions.asc',
         'https://packages.grafana.com/gpg.key',
         'https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc',
     ] %}
@@ -368,7 +370,7 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom
       'F77F1EDA57EBB1CC',
       'F6609E60DC62814E',
     ] %}
-    {% set remote_apt_keys = [
+    {% set base_remote_apt_keys = [
       'https://packages.grafana.com/gpg.key',
       'https://repos.influxdata.com/influxdb.key',
     ] %}
@@ -379,7 +381,7 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom
       'F77F1EDA57EBB1CC',
       'F6609E60DC62814E',
     ] %}
-    {% set remote_apt_keys = [
+    {% set base_remote_apt_keys = [
       'https://download.docker.com/linux/debian/gpg',
       'https://packages.grafana.com/gpg.key',
       'https://packages.treasuredata.com/GPG-KEY-td-agent',
@@ -394,7 +396,7 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom
             {%- if not loop.last %} \
             {% endif -%}
         {% endfor %}
-        {% for key in remote_apt_keys | customizable('remote_apt_keys') %}
+        {% for key in base_remote_apt_keys | customizable('remote_apt_keys') %}
             {%- if loop.first %} RUN {% else %} && {% endif %}curl {{ key }} | apt-key add -
             {%- if not loop.last %} \
             {% endif -%}

docker/base/apt_preferences.debian

@@ -7,11 +7,6 @@ Package: erlang*
 Pin: version 1:23.*
 Pin-Priority: 1000
 
-# FIXME(mgoddard): Pinning to 4.0.* to avoid bug 1930867.
-Package: td-agent*
-Pin: version 4.0.*
-Pin-Priority: 1000
-
 # NOTE(mgoddard): logstash 7.9.x is the last version that supports
 # Elasticsearch OSS.
 Package: logstash-oss

docker/base/apt_preferences.ubuntu

@@ -6,11 +6,6 @@ Package: erlang*
 Pin: version 1:23.*
 Pin-Priority: 1000
 
-# FIXME(mgoddard): Pinning to 4.0.* to avoid bug 1930867.
-Package: td-agent*
-Pin: version 4.0.*
-Pin-Priority: 1000
-
 # NOTE(mgoddard): logstash 7.9.x is the last version that supports
 # Elasticsearch OSS.
 Package: logstash-oss

docker/base/erlang-solutions.repo

@@ -0,0 +1,6 @@
+[erlang-solutions]
+name=erlang-solutions
+baseurl=https://packages.erlang-solutions.com/rpm/centos/$releasever/$basearch
+gpgcheck=1
+gpgkey=https://packages.erlang-solutions.com/rpm/erlang_solutions.asc
+enabled=0

docker/collectd/Dockerfile.j2

@@ -30,10 +30,8 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
       'collectd-dbi',
       'collectd-disk',
       'collectd-dns',
-      'collectd-dpdk_telemetry',
       'collectd-generic-jmx',
       'collectd-ipmi',
-      'collectd-libpod-stats',
       'collectd-log_logstash',
       'collectd-logparser',
       'collectd-mcelog',
@@ -47,7 +45,6 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
       'collectd-procevent',
       'collectd-python',
       'collectd-sensors',
-      'collectd-sensubility',
       'collectd-smart',
       'collectd-snmp',
       'collectd-snmp-agent',
@@ -57,7 +54,6 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
       'collectd-write_http',
       'collectd-write_kafka',
       'collectd-write_prometheus',
-      'python3-sqlalchemy-collectd'
   ] %}
 
   {% if base_arch in ['x86_64', 'ppc64le'] %}
@@ -69,18 +65,13 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
   {% if base_arch =='x86_64' %}
       {% set collectd_packages = collectd_packages + [
         'collectd-hugepages',
+        'collectd-pcie-errors',
         'collectd-pmu',
         'collectd-rdt',
         'collectd-turbostat'
       ] %}
   {% endif %}
 
-  {% if base_arch != 's390x' %}
-      {% set collectd_packages = collectd_packages + [
-        'collectd-pcie-errors'
-      ] %}
-  {% endif %}
-
 {% endif %}
 
 {{ macros.install_packages(collectd_packages | customizable("packages")) }}

docker/fluentd/Dockerfile.j2

@@ -27,9 +27,8 @@ LABEL fluentd_version="0.14" fluentd_binary="td-agent"
     ] %}
 
     {% if base_arch in ['aarch64', 'x86_64'] %}
-        # FIXME(mgoddard): Pinning to 4.0.* to avoid bug 1930867.
         {% set fluentd_packages = fluentd_packages + [
-            'td-agent-4.0.*'
+            'td-agent'
         ] %}
     {% else %}
         {% set fluentd_packages = fluentd_packages + [
@@ -62,7 +61,14 @@ LABEL fluentd_version="0.14" fluentd_binary="td-agent"
 {% endif %}
 
 {{ macros.configure_user(name=fluentd_user, groups='mysql') }}
-{{ macros.install_packages(fluentd_packages | customizable("packages")) }}
+
+# NOTE(frickler): Downgrading elasticsearch gems for compatibility with OSS versions of ES
+{{ macros.install_packages(fluentd_packages | customizable("packages")) }}{% if base_arch in ['aarch64', 'x86_64'] %} \
+    && td-agent-gem install elasticsearch:7.13.0 \
+    && td-agent-gem uninstall "elasticsearch:>7.13.0" \
+    && td-agent-gem uninstall "elasticsearch-api:>7.13.0" \
+    && td-agent-gem uninstall "elasticsearch-transport:>7.13.0"
+{% endif %}
 
 # Distro specific files and operations
 # Fluentd on rpm-based non-x86_64 is installed from rpm

docker/ironic/ironic-pxe/Dockerfile.j2

@@ -25,15 +25,19 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
     {% if base_arch != 's390x' %}
         {% set ironic_pxe_packages = ironic_pxe_packages + [
             'grub2-tools',
-            'grub2-efi-aa64-modules'
+            'grub2-efi-*64',
+            'grub2-efi-aa64-modules',
+            'shim-*64',
         ] %}
     {% endif %}
 
 {{ macros.install_packages(ironic_pxe_packages | customizable("packages")) }}
 {% elif base_package_type == 'deb' %}
     {% set ironic_pxe_packages = [
+        'grub-efi-*64-signed',
         'ipxe',
         'pxelinux',
+        'shim-signed',
         'syslinux-common',
         'tftpd-hpa'
     ] %}
@@ -42,10 +46,6 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
         {% set ironic_pxe_packages = ironic_pxe_packages + [
             'syslinux'
         ] %}
-    {% elif base_arch == 'aarch64' %}
-        {% set ironic_pxe_packages = ironic_pxe_packages + [
-            'grub-efi-arm64'
-        ] %}
     {% endif %}
 
 {{ macros.install_packages(ironic_pxe_packages | customizable("packages")) }}

docker/ironic/ironic-pxe/extend_start.sh

@@ -1,7 +1,8 @@
 #!/bin/bash
 
 
-function prepare_pxe {
+# For x86 legacy BIOS boot mode
+function prepare_pxe_pxelinux {
     chown -R ironic: /tftpboot
     for pxe_file in /var/lib/tftpboot/pxelinux.0 /var/lib/tftpboot/chain.c32 /usr/lib/syslinux/pxelinux.0 \
                     /usr/lib/syslinux/chain.c32 /usr/lib/PXELINUX/pxelinux.0 \
@@ -12,32 +13,60 @@ function prepare_pxe {
     done
 }
 
+# For UEFI boot mode
+function prepare_pxe_grub {
+    if [[ "${KOLLA_BASE_DISTRO}" =~ debian|ubuntu  ]]; then
+        shim_src_file="/usr/lib/shim/shim*64.efi.signed"
+        grub_src_file="/usr/lib/grub/*-efi-signed/grubnet*64.efi.signed"
+    elif [[ "${KOLLA_BASE_DISTRO}" =~ centos|rhel ]]; then
+        shim_src_file="/boot/efi/EFI/centos/shim*64.efi"
+        grub_src_file="/boot/efi/EFI/centos/grub*64.efi"
+    fi
+
+    if [[ "${KOLLA_BASE_ARCH}" == "x86_64" ]]; then
+        shim_dst_file="bootx64.efi"
+        grub_dst_file="grubx64.efi"
+    elif [[ "${KOLLA_BASE_ARCH}" == "aarch64" ]]; then
+        shim_dst_file="bootaa64.efi"
+        grub_dst_file="grubaa64.efi"
+    fi
+
+    cp $shim_src_file /tftpboot/$shim_dst_file
+    cp $grub_src_file /tftpboot/$grub_dst_file
+}
+
 function prepare_ipxe {
+    # NOTE(mgoddard): Ironic uses snponly.efi as the default for
+    # uefi_ipxe_bootfile_name since Xena. In Wallaby and earlier releases it
+    # was ipxe.efi. Ensure that both exist, using symlinks where the files are
+    # named differently to allow the original names to be used in ironic.conf.
     if [[ "${KOLLA_BASE_DISTRO}" =~ debian|ubuntu ]]; then
         cp /usr/lib/ipxe/{undionly.kpxe,ipxe.efi} /tftpboot
+        # NOTE(mgoddard): The 'else' can be removed  when snponly.efi is
+        # available in Jammy 22.04.
+        if [[ -f /usr/lib/ipxe/snponly.efi ]]; then
+            cp /usr/lib/ipxe/snponly.efi /tftpboot/snponly.efi
+        elif [[ ! -e /tftpboot/snponly.efi ]]; then
+            ln -s /tftpboot/ipxe.efi /tftpboot/snponly.efi
+        fi
     elif [[ "${KOLLA_BASE_DISTRO}" =~ centos|rhel ]]; then
         cp /usr/share/ipxe/{undionly.kpxe,ipxe*.efi} /tftpboot
+        if [[ ! -e /tftpboot/ipxe.efi ]]; then
+            ln -s /tftpboot/ipxe-${KOLLA_BASE_ARCH}.efi /tftpboot/ipxe.efi
+        fi
+        if [[ ! -e /tftpboot/snponly.efi ]]; then
+            ln -s /tftpboot/ipxe-snponly-${KOLLA_BASE_ARCH}.efi /tftpboot/snponly.efi
+        fi
     fi
 }
 
 # Bootstrap and exit if KOLLA_BOOTSTRAP variable is set. This catches all cases
 # of the KOLLA_BOOTSTRAP variable being set, including empty.
 if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then
-    prepare_pxe
+    prepare_pxe_pxelinux
+    prepare_pxe_grub
     prepare_ipxe
     exit 0
 fi
 
-if [[ -d /usr/lib/grub/arm64-efi ]]; then
-    modules="boot chain configfile efinet ext2 fat gettext help hfsplus loadenv \
-    lsefi normal part_gpt part_msdos read search search_fs_file search_fs_uuid \
-    search_label terminal terminfo tftp linux"
-
-    if [[ "${KOLLA_BASE_DISTRO}" =~ debian|ubuntu ]]; then
-        grub-mkimage -v -o /tftpboot/grubaa64.efi -O arm64-efi -p "grub" $modules
-    elif [[ "${KOLLA_BASE_DISTRO}" =~ centos|rhel ]]; then
-        grub2-mkimage -v -o /tftpboot/grubaa64.efi -O arm64-efi -p "EFI/centos" $modules
-    fi
-fi
-
 . /usr/local/bin/kolla_httpd_setup

docker/kolla-toolbox/Dockerfile.j2

@@ -14,12 +14,17 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 {{ macros.configure_user(name='ansible') }}
 {{ macros.configure_user(name='rabbitmq') }}
 
-{{ macros.enable_extra_repos(['rabbitmq', 'erlang', 'openvswitch', 'powertools']) }}
+{% if base_arch == 'aarch64' %}
+{{ macros.enable_extra_repos(['epel']) }}
+{% endif %}
+
+{{ macros.enable_extra_repos(['erlang', 'openvswitch', 'powertools', 'rabbitmq']) }}
 
 {% if base_package_type == 'rpm' %}
 
     {% set kolla_toolbox_packages = [
         'crudini',
+        'erlang-23.*',
         'gcc',
         'gdisk',
         'git',
@@ -34,19 +39,8 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
         'openssl-devel',
         'openvswitch',
         'python3-devel',
-        ] %}
-
-    {% if base_arch == 'aarch64' %}
-        {% set kolla_toolbox_packages = kolla_toolbox_packages + [
-            'erlang-22.*',
-            'rabbitmq-server-3.8.14'
-            ] %}
-    {% else %}
-        {% set kolla_toolbox_packages = kolla_toolbox_packages + [
-            'erlang-23.*',
-            'rabbitmq-server-3.8.*'
-            ] %}
-    {% endif %}
+        'rabbitmq-server-3.8.*'
+    ] %}
 
 {% elif base_package_type == 'deb' %}
     {% set kolla_toolbox_packages = [

docker/magnum/magnum-base/Dockerfile.j2

@@ -32,6 +32,7 @@ RUN ln -s magnum-base-source/* magnum \
     && {{ macros.install_pip(magnum_base_pip_packages | customizable("pip_packages")) }} \
     && mkdir -p /etc/magnum \
     && cp -r /magnum/etc/magnum/* /etc/magnum \
+    && mv /etc/magnum/keystone_auth_default_policy.sample /etc/magnum/keystone_auth_default_policy.json \
     && chown -R magnum: /etc/magnum
 
 {% endif %}

docker/openstack-base/Dockerfile.j2

@@ -24,7 +24,6 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
             'mod_ssl',
             'openssl',
             'python3-alembic',
-            'python3-anyjson',
             'python3-aodhclient',
             'python3-barbicanclient',
             'python3-cachetools',
@@ -88,7 +87,6 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
             'python3-routes',
             'python3-saharaclient',
             'python3-sqlalchemy',
-            'python3-sqlalchemy-collectd',
             'python3-sqlparse',
             'python3-swiftclient',
             'python3-sysv_ipc',
@@ -210,7 +208,6 @@ ENV DEBIAN_FRONTEND noninteractive
         'WSME',
         'alembic',
         'amqp',
-        'anyjson',
         'aodhclient',
         'appdirs',
         'automaton',

docker/rabbitmq/Dockerfile.j2

@@ -9,27 +9,22 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 
 {{ macros.configure_user(name='rabbitmq') }}
 
-{{ macros.enable_extra_repos(['rabbitmq', 'erlang', 'powertools']) }}
+{# NOTE(hrw): erlang packages from Erlang Solutions needs libwx from EPEL #}
+{% if base_arch == 'aarch64' %}
+{{ macros.enable_extra_repos(['epel']) }}
+{% endif %}
+
+{{ macros.enable_extra_repos(['erlang', 'powertools', 'rabbitmq']) }}
 
 {% block rabbitmq_install %}
 {% if base_package_type == 'rpm' %}
 
     {% set rabbitmq_packages = [
+        'erlang-23.*',
         'hostname',
+        'rabbitmq-server-3.8.*'
     ] %}
 
-    {% if base_arch == 'aarch64' %}
-        {% set rabbitmq_packages = rabbitmq_packages + [
-            'erlang-22.*',
-            'rabbitmq-server-3.8.14'
-        ] %}
-    {% else %}
-        {% set rabbitmq_packages = rabbitmq_packages + [
-            'erlang-23.*',
-            'rabbitmq-server-3.8.*'
-        ] %}
-    {% endif %}
-
 {% elif base_package_type == 'deb' %}
     {% set rabbitmq_packages = [
         'logrotate',

docker/storm/storm-base/Dockerfile.j2

@@ -12,6 +12,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 {% if base_package_type == 'rpm' %}
     {% set storm_packages = [
         'java-1.8.0-openjdk-headless',
+        'zip',
     ] %}
 {% elif base_package_type == 'deb' %}
 
@@ -23,6 +24,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 
     {% set storm_packages = [
         'openjdk-' + java_version + '-jre-headless',
+        'zip',
     ] %}
 {% endif %}
 
@@ -40,6 +42,9 @@ RUN curl -o /tmp/storm.tgz ${storm_url} \
     && tar --strip 1 -xvf /tmp/storm.tgz -C /opt/storm \
     && rm -f /tmp/storm.tgz
 
+# Mitigation for CVE-2021-44228 and CVE-2021-45046: remove the JndiLookup class
+# from the classpath
+RUN zip -q -d /opt/storm/lib/log4j-core-*.jar org/apache/logging/log4j/core/lookup/JndiLookup.class
 {% endblock %}
 
 {% block storm_python_version %}