Synchronise victoria with upstream

docker/base/Dockerfile.j2

@@ -351,8 +351,8 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom
     {# 901F9177AB97ACBE                         -- Treasure Data, Inc (Treasure Agent Official Signing key) <support@treasure-data.com> #}
     {# A20F259AEB9C94BB                         -- Sensuapp (Freight) <support@hw-ops.com> #}
     {# F1656F24C74CD1D8                         -- MariaDB Signing Key <signing-key@mariadb.org> #}
-    {# F77F1EDA57EBB1CC                         -- Launchpad RabbitMQ Erlang PPA key #}
-    {# F6609E60DC62814E                         -- PackageCloud RabbitMQ repository key #}
+    {# E495BB49CC4BBE5B                         -- Cloudsmith RabbitMQ Erlang repository key #}
+    {# 9F4587F226208342                         -- Cloudsmith RabbitMQ repository key #}
     {% set base_apt_keys = [
       '391A9AA2147192839E9DB0315EDB1B62EC4926EA',
       '46095ACC8548582C1A2699A9D27D666CD88E42B4',
@@ -362,8 +362,8 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom
       '901F9177AB97ACBE',
       'A20F259AEB9C94BB',
       'F1656F24C74CD1D8',
-      'F77F1EDA57EBB1CC',
-      'F6609E60DC62814E',
+      'E495BB49CC4BBE5B',
+      '9F4587F226208342',
     ] %}
     {% set base_remote_apt_keys = [
       'https://packages.grafana.com/gpg.key',
@@ -373,8 +373,8 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom
     {% set base_apt_keys = [
       '46095ACC8548582C1A2699A9D27D666CD88E42B4',
       'F1656F24C74CD1D8',
-      'F77F1EDA57EBB1CC',
-      'F6609E60DC62814E',
+      'E495BB49CC4BBE5B',
+      '9F4587F226208342',
     ] %}
     {% set base_remote_apt_keys = [
       'https://download.docker.com/linux/debian/gpg',

docker/base/healthcheck_curl

@@ -5,7 +5,7 @@
 : ${HEALTHCHECK_CURL_OUTPUT:='/dev/null'}
 
 export NSS_SDB_USE_CACHE=no
-curl -g -k -q -s -S --fail -o "${HEALTHCHECK_CURL_OUTPUT}" \
+curl -q -g -k -s -S --fail -o "${HEALTHCHECK_CURL_OUTPUT}" \
     --max-time "${HEALTHCHECK_CURL_MAX_TIME}" \
     --user-agent "${HEALTHCHECK_CURL_USER_AGENT}" \
     --write-out "${HEALTHCHECK_CURL_WRITE_OUT}" \

docker/elasticsearch/elasticsearch/Dockerfile.j2

@@ -12,8 +12,12 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 {{ macros.enable_extra_repos(['elasticsearch']) }}
 
 {% if base_package_type == 'rpm' %}
+
+# NOTE(hrw): post-install script of elasticsearch fails when trying to
+# install elasticsearch and java together.
+{{ macros.install_packages(['java-11-openjdk-headless']) }}
+
     {% set elasticsearch_packages = [
-        'java-11-openjdk-headless',
         'elasticsearch-oss',
     ] %}
 

docker/macros.j2

@@ -34,7 +34,7 @@
 
 {% macro install_pip(packages, constraints = true) %}
 {%- if packages is sequence and packages|length > 0 -%}
-    python{{ distro_python_version }} -m pip --no-cache-dir install --upgrade{{ ' ' }}
+    SETUPTOOLS_USE_DISTUTILS=stdlib python{{ distro_python_version }} -m pip --no-cache-dir install --upgrade{{ ' ' }}
     {%- if constraints %}-c /requirements/upper-constraints.txt {% endif -%}
     {{ packages | join(' ') }}
 {%- else -%}

docker/masakari/masakari-monitors/Dockerfile.j2

@@ -19,16 +19,20 @@ RUN echo '{{ install_type }} not yet available for {{ base_distro }}' \
     {% if base_package_type == 'rpm' %}
 
         {% set masakari_monitors_packages = [
-            'libvirt-devel',
+            'cyrus-sasl-md5',
+            'cyrus-sasl-scram',
+            'python3-libvirt',
             'pacemaker-cli',
             'tcpdump',
         ] %}
 
     {% elif base_package_type == 'deb' %}
 
         {% set masakari_monitors_packages = [
-            'libvirt-dev',
+            'libsasl2-modules-gssapi-mit',
+            'python3-libvirt',
             'pacemaker-cli-utils',
+            'sasl2-bin',
             'tcpdump',
         ] %}
 
@@ -42,8 +46,10 @@ ADD masakari-monitors-archive /masakari-monitors-source
     '/masakari-monitors'
 ] %}
 
+# NOTE(hrw): distros may provide other version of libvirt
 RUN ln -s masakari-monitors-source/* masakari-monitors \
-    {% if distro_package_manager == 'dnf' %}&& sed -i -e 's/libvirt-python===.*/libvirt-python===6.10.0/' /requirements/upper-constraints.txt {% endif %}\
+    && sed -i -e "/^libvirt-python/d" /requirements/upper-constraints.txt \
+    && sed -i -e "/^libvirt-python/d" /masakari-monitors/requirements.txt \
     && {{ macros.install_pip(masakari_monitors_pip_packages | customizable("pip_packages")) }} \
     && mkdir -p /etc/masakari-monitors \
     && chown -R masakari: /etc/masakari-monitors

kolla/common/utils.py

@@ -21,21 +21,33 @@ def make_a_logger(conf=None, image_name=None):
         log = logging.getLogger(".".join([__name__, image_name]))
     else:
         log = logging.getLogger(__name__)
+
+    if conf is not None and conf.debug:
+        loglevel = logging.DEBUG
+    else:
+        loglevel = logging.INFO
+
     if not log.handlers:
-        if conf is None or not conf.logs_dir or not image_name:
-            handler = logging.StreamHandler(sys.stderr)
-            log.propagate = False
+        stream_handler = logging.StreamHandler(sys.stderr)
+        stream_handler.setFormatter(logging.Formatter(logging.BASIC_FORMAT))
+        # NOTE(hrw): quiet mode matters only on console
+        if conf is not None and conf.quiet:
+            stream_handler.setLevel(logging.CRITICAL)
         else:
+            stream_handler.setLevel(loglevel)
+        log.addHandler(stream_handler)
+        log.propagate = False
+
+        if conf is not None and conf.logs_dir and image_name:
             filename = os.path.join(conf.logs_dir, "%s.log" % image_name)
             handler = logging.FileHandler(filename, delay=True)
-        handler.setFormatter(logging.Formatter(logging.BASIC_FORMAT))
-        log.addHandler(handler)
-    if conf is not None and conf.debug:
-        log.setLevel(logging.DEBUG)
-    elif conf is not None and conf.quiet and image_name:
-        log.setLevel(logging.CRITICAL)
-    else:
-        log.setLevel(logging.INFO)
+            # NOTE(hrw): logfile will be INFO or DEBUG
+            handler.setLevel(loglevel)
+            handler.setFormatter(logging.Formatter(logging.BASIC_FORMAT))
+            log.addHandler(handler)
+
+    # NOTE(hrw): needs to be high, handlers have own levels
+    log.setLevel(logging.DEBUG)
     return log
 
 

kolla/image/build.py

@@ -405,6 +405,15 @@ def followups(self):
     def process_source(self, image, source):
         dest_archive = os.path.join(image.path, source['name'] + '-archive')
 
+        # NOTE(mgoddard): Change ownership of files to root:root. This
+        # avoids an issue introduced by the fix for git CVE-2022-24765,
+        # which breaks PBR when the source checkout is not owned by the
+        # user installing it. LP#1969096
+        def reset_userinfo(tarinfo):
+            tarinfo.uid = tarinfo.gid = 0
+            tarinfo.uname = tarinfo.gname = "root"
+            return tarinfo
+
         if source.get('type') == 'url':
             self.logger.debug("Getting archive from %s", source['source'])
             try:
@@ -458,7 +467,8 @@ def process_source(self, image, source):
             if os.path.isdir(source['source']):
                 with tarfile.open(dest_archive, 'w') as tar:
                     tar.add(source['source'],
-                            arcname=os.path.basename(source['source']))
+                            arcname=os.path.basename(source['source']),
+                            filter=reset_userinfo)
             else:
                 shutil.copyfile(source['source'], dest_archive)
 

kolla/template/filters.py

@@ -12,11 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from jinja2 import contextfilter
+# NOTE: jinja2 3.1.0 dropped contextfilter in favour of pass_context.
+try:
+    from jinja2 import pass_context
+except ImportError:
+    from jinja2 import contextfilter as pass_context
+
 from jinja2 import Undefined
 
 
-@contextfilter
+@pass_context
 def customizable(context, val_list, call_type):
     # NOTE(mgoddard): Don't try to customise undefined values. There are cases
     # where this might happen, for example using a generic template overrides

kolla/template/methods.py

@@ -15,7 +15,11 @@
 import os
 import yaml
 
-from jinja2 import contextfunction
+# NOTE: jinja2 3.1.0 dropped contextfunction in favour of pass_context.
+try:
+    from jinja2 import pass_context
+except ImportError:
+    from jinja2 import contextfunction as pass_context
 
 
 def debian_package_install(packages, clean_package_cache=True):
@@ -71,7 +75,7 @@ def debian_package_install(packages, clean_package_cache=True):
     return ' && '.join(cmds)
 
 
-@contextfunction
+@pass_context
 def handle_repos(context, reponames, mode):
     """NOTE(hrw): we need to handle CentOS, Debian and Ubuntu with one macro.
 

kolla/template/repos.yaml

@@ -47,53 +47,51 @@ rhel:
   opstools: "centos-opstools"
   rabbitmq: "centos-rabbitmq-38"
 
-# NOTE(mnasiadka): Erlang repo - Debian Buster/Bullseye needs to use bionic as per RabbitMQ docs
 debian:
-
   elasticsearch: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu bionic main"
+  erlang: "deb https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-erlang/deb/debian buster main"
   grafana: "deb https://packages.grafana.com/oss/deb stable main"
   influxdb: "deb https://repos.influxdata.com/debian buster stable"
   logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
   kibana: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
   mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/debian buster main"
-  rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/debian/ buster main"
+  rabbitmq: "deb https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-server/deb/debian buster main"
   td-agent: "deb http://packages.treasuredata.com/4/debian/buster buster contrib"
 
 debian-aarch64:
   elasticsearch: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu bionic main"
+  erlang: "deb https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-erlang/deb/debian buster main"
   grafana: "deb https://packages.grafana.com/oss/deb stable main"
   influxdb: "deb https://repos.influxdata.com/debian buster stable"
   logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
   kibana: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
   libvirt: "deb https://obs.linaro.org/repos/home:/marcin.juszkiewicz/debian-buster ./"
   mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/debian buster main"
-  rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/debian/ buster main"
+  rabbitmq: "deb https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-server/deb/debian buster main"
   td-agent: "deb http://packages.treasuredata.com/4/debian/buster buster contrib"
 
 ubuntu:
   elasticsearch: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu focal main"
+  erlang: "deb https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-erlang/deb/ubuntu focal main"
   grafana: "deb https://packages.grafana.com/oss/deb stable main"
   influxdb: "deb https://repos.influxdata.com/ubuntu focal stable"
   logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
   kibana: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
   mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu focal main"
   qdrouterd: "deb http://ppa.launchpad.net/qpid/released/ubuntu/ focal main"
-  rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ focal main"
+  rabbitmq: "deb https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-server/deb/ubuntu focal main"
   td-agent: "deb http://packages.treasuredata.com/4/ubuntu/focal/ focal contrib"
 
 ubuntu-aarch64:
   elasticsearch: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
-  erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu focal main"
+  erlang: "deb https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-erlang/deb/ubuntu focal main"
   grafana: "deb https://packages.grafana.com/oss/deb stable main"
   influxdb: "deb https://repos.influxdata.com/ubuntu focal stable"
   logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
   mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu bionic main"
-  rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ focal main"
+  rabbitmq: "deb https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-server/deb/ubuntu focal main"
   td-agent: "deb http://packages.treasuredata.com/4/ubuntu/focal/ focal contrib"
 
 ubuntu-ppc64le:
   mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu focal main"
-  rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ focal main"
+  rabbitmq: "deb https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-server/deb/ubuntu focal main"

releasenotes/notes/healthcheck-curl-disable-curlrc-0f85aad47379e2a5.yaml

@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    Fixes disabling the use of the ``curlrc`` configuration file in
+    ``healthcheck_curl``. `LP#1967272 <https://launchpad.net/bugs/1967272>`__

releasenotes/notes/jinja2-pass-context-3f3febcd944e3a51.yaml

@@ -0,0 +1,4 @@
+---
+fixes:
+  - |
+    Fixes an issue seen when using Jinja2 3.1.0.

releasenotes/notes/quiet-mode-with-logs-0abafc07923945ac.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Quiet mode (enabled with ``--quiet`` argument) can be combined with
+    ``--logs-dir`` option now. Console output will be quiet as expected while
+    building output will be stored in separate log files.

releasenotes/notes/rabbitmq-erlang-cloudsmith-c837bf4a450dd802.yaml

@@ -0,0 +1,10 @@
+---
+fixes:
+  - |
+    Fixes the Debian and Ubuntu images to use rabbitmq and erlang from
+    cloudsmith so that the images are still buildable and use proper versions.
+upgrade:
+  - |
+    The Debian and Ubuntu images use rabbitmq and erlang from cloudsmith now.
+    Operators might want to mirror/proxy this new source as it provides the
+    correct set of packages unlike the previous combination.

tests/playbooks/run.yml

@@ -9,6 +9,7 @@
       DEFAULT:
         debug: true
         logs_dir: "{{ kolla_build_logs_dir }}"
+        quiet: true
         base: "{{ base_distro }}"
         install_type: "{{ install_type }}"
         template_override: /etc/kolla/template_overrides.j2