Synchronise xena with upstream

.zuul.d/base.yaml

@@ -113,7 +113,6 @@
     extra-vars:
       kolla_logs_dir: "{{ zuul_output_dir }}/logs/kolla"
       kolla_build_logs_dir: "{{ kolla_logs_dir }}/build"
-      kolla_work_dir: "{{ kolla_logs_dir }}"
       virtualenv_path: "/tmp/kolla-virtualenv"
 
 - job:

.zuul.d/debian.yaml

@@ -20,6 +20,8 @@
     experimental:
       jobs:
         - kolla-build-no-infra-wheels-debian-source
+        - kolla-ansible-debian-source-ironic:
+            files: ^docker\/(base|dnsmasq|ironic|ironic-inspector|iscsid|openstack-base)\/.*
 
 - job:
     name: kolla-build-debian-source

docker/base/Dockerfile.j2

@@ -322,8 +322,8 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom
 ] %}
 
 {% set base_apt_keys = [
-   {'name': 'erlang',   'keyid': 'F77F1EDA57EBB1CC'},
-   {'name': 'rabbitmq', 'keyid': 'F6609E60DC62814E'},
+   {'name': 'erlang',   'keyid': 'E495BB49CC4BBE5B'},
+   {'name': 'rabbitmq', 'keyid': '9F4587F226208342'},
    {'name': 'haproxy',  'keyid': 'CFFB779AADC995E4F350A060505D97A41C61B9CD'},
 ] %}
 

docker/base/apt_preferences.debian

@@ -22,3 +22,8 @@ Pin-Priority: -1000
 Package: openvswitch* python3-openvswitch ovn-*
 Pin: release n=bullseye-backports
 Pin-Priority: 1024
+
+# NOTE(hrw): Debian OpenStack Team repo has some backports we do not want
+Package: git*
+Pin: release n=bullseye
+Pin-Priority: 1024

docker/base/healthcheck_curl

@@ -5,7 +5,7 @@
 : ${HEALTHCHECK_CURL_OUTPUT:='/dev/null'}
 
 export NSS_SDB_USE_CACHE=no
-curl -g -k -q -s -S --fail -o "${HEALTHCHECK_CURL_OUTPUT}" \
+curl -q -g -k -s -S --fail -o "${HEALTHCHECK_CURL_OUTPUT}" \
     --max-time "${HEALTHCHECK_CURL_MAX_TIME}" \
     --user-agent "${HEALTHCHECK_CURL_USER_AGENT}" \
     --write-out "${HEALTHCHECK_CURL_WRITE_OUT}" \

docker/base/set_configs.py

@@ -120,6 +120,7 @@ def _set_properties(self, source, dest):
             self._set_properties_from_conf(dest)
 
     def _set_properties_from_file(self, source, dest):
+        LOG.info('Copying permissions from %s onto %s', source, dest)
         shutil.copystat(source, dest)
         stat = os.stat(source)
         os.chown(dest, stat.st_uid, stat.st_gid)

docker/bifrost/bifrost-base/Dockerfile.j2

@@ -40,8 +40,13 @@ RUN apt-get update && \
 {%- else %}
 RUN echo " " && \
 {%- endif %}
-    bash -c './scripts/env-setup.sh && \
+    bash -c 'TOX_CONSTRAINTS_FILE=/requirements/upper-constraints.txt ./scripts/env-setup.sh && \
     ansible-playbook -vvvv -i /bifrost/playbooks/inventory/target /bifrost/playbooks/install.yaml \
+{%- if base_distro == 'centos' %}
+{# NOTE(mmalchuk): since DIB drop Python3.6 support we need to build DIB from #}
+{# the latest supported stable tag until we move to CentOS9 #}
+    -e dib_git_branch="3.21.1" \
+{%- endif %}
     -e @/tmp/build_arg.yml && \
 {%- if base_package_type == 'deb' %}
     apt-get clean && rm -rf /var/lib/apt/lists/*'

docker/cloudkitty/cloudkitty-api/Dockerfile.j2

@@ -13,8 +13,6 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
         {% set cloudkitty_api_packages = [
                 'openstack-cloudkitty-api'
         ] %}
-{{ macros.install_packages(cloudkitty_api_packages | customizable("packages")) }}
-
     {% elif base_package_type == 'deb' %}
 
         {% set cloudkitty_api_packages = [
@@ -23,12 +21,12 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 
     {% endif %}
 {% elif install_type == 'source' %}
-
         {% set cloudkitty_api_packages = [
         ] %}
-{{ macros.install_packages(cloudkitty_api_packages | customizable("packages")) }}
 {% endif %}
 
+{{ macros.install_packages(cloudkitty_api_packages | customizable("packages")) }}
+
 COPY extend_start.sh /usr/local/bin/kolla_cloudkitty_extend_start
 RUN chmod 755 /usr/local/bin/kolla_cloudkitty_extend_start
 

docker/elasticsearch/elasticsearch/Dockerfile.j2

@@ -12,8 +12,12 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 {{ macros.enable_extra_repos(['elasticsearch']) }}
 
 {% if base_package_type == 'rpm' %}
+
+# NOTE(hrw): post-install script of elasticsearch fails when trying to
+# install elasticsearch and java together.
+{{ macros.install_packages(['java-11-openjdk-headless']) }}
+
     {% set elasticsearch_packages = [
-        'java-11-openjdk-headless',
         'elasticsearch-oss',
     ] %}
 

docker/macros.j2

@@ -34,7 +34,7 @@
 
 {% macro install_pip(packages, constraints = true) %}
 {%- if packages is sequence and packages|length > 0 -%}
-    python3 -m pip --no-cache-dir install --upgrade{{ ' ' }}
+    SETUPTOOLS_USE_DISTUTILS=stdlib python3 -m pip --no-cache-dir install --upgrade{{ ' ' }}
     {%- if constraints %}-c /requirements/upper-constraints.txt {% endif -%}
     {{ packages | join(' ') }}
 {%- else -%}

docker/manila/manila-share/Dockerfile.j2

@@ -22,6 +22,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
         {% set manila_share_packages = [
             'manila-share',
             'ceph-common',
+            'glusterfs-client',
             'python3-cephfs',
             'python3-rados',
             'python3-rbd',
@@ -42,6 +43,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
     {% elif base_package_type == 'deb' %}
         {% set manila_share_packages = [
             'ceph-common',
+            'glusterfs-client',
             'python3-cephfs',
             'python3-rados',
             'python3-rbd',

docker/masakari/masakari-monitors/Dockerfile.j2

@@ -20,10 +20,12 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
     {% elif base_package_type == 'deb' %}
 
         {% set masakari_base_packages = [
+            'libsasl2-modules-gssapi-mit',
             'masakari-host-monitor',
             'masakari-instance-monitor',
             'masakari-introspective-instance-monitor',
             'masakari-process-monitor',
+            'sasl2-bin',
         ] %}
 
     {% endif %}
@@ -35,6 +37,8 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
     {% if base_package_type == 'rpm' %}
 
         {% set masakari_monitors_packages = [
+            'cyrus-sasl-md5',
+            'cyrus-sasl-scram',
             'python3-libvirt',
             'pacemaker-cli',
             'tcpdump',
@@ -43,8 +47,10 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
     {% elif base_package_type == 'deb' %}
 
         {% set masakari_monitors_packages = [
+            'libsasl2-modules-gssapi-mit',
             'python3-libvirt',
             'pacemaker-cli-utils',
+            'sasl2-bin',
             'tcpdump',
         ] %}
 

docker/nova/nova-compute/Dockerfile.j2

@@ -14,6 +14,8 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 
         {% set nova_compute_packages = [
             'ceph-common',
+            'cyrus-sasl-md5',
+            'cyrus-sasl-scram',
             'device-mapper-multipath',
             'e2fsprogs',
             'genisoimage',
@@ -50,6 +52,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
             'e2fsprogs',
             'genisoimage',
             'ironic-common',
+            'libsasl2-modules-gssapi-mit',
             'multipath-tools',
             'nfs-common',
             'nova-compute',
@@ -65,6 +68,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
             'python3-rados',
             'python3-rbd',
             'python3-rtslib-fb',
+            'sasl2-bin',
             'sysfsutils',
             'targetcli-fb',
             'xfsprogs'
@@ -88,6 +92,8 @@ RUN rm -f /etc/nova/nova-compute.conf
 
         {% set nova_compute_packages = [
             'ceph-common',
+            'cyrus-sasl-md5',
+            'cyrus-sasl-scram',
             'device-mapper-multipath',
             'dosfstools',
             'e2fsprogs',
@@ -127,6 +133,7 @@ RUN rm -f /etc/nova/nova-compute.conf
             'e2fsprogs',
             'genisoimage',
             'libosinfo-bin',
+            'libsasl2-modules-gssapi-mit',
             'multipath-tools',
             'nfs-common',
             'nvme-cli',
@@ -140,6 +147,7 @@ RUN rm -f /etc/nova/nova-compute.conf
             'python3-rbd',
             'python3-rtslib-fb',
             'qemu-utils',
+            'sasl2-bin',
             'sysfsutils',
             'targetcli-fb',
             'xfsprogs'

docker/nova/nova-libvirt/Dockerfile.j2

@@ -15,6 +15,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 
     {% set nova_libvirt_packages = [
         'ceph-common',
+        'cyrus-sasl-md5',
         'cyrus-sasl-scram',
         'libguestfs',
         'libvirt-client',
@@ -41,6 +42,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
     {% set nova_libvirt_packages = [
         'ceph-common',
         'ebtables',
+        'libsasl2-modules-gssapi-mit',
         'libvirt-clients',
         'libvirt-daemon-system',
         'openvswitch-switch',
@@ -51,6 +53,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
         'qemu-block-extra',
         'qemu-system',
         'qemu-utils',
+        'sasl2-bin',
         'trousers'
     ] %}
 

docker/prometheus/prometheus-libvirt-exporter/Dockerfile.j2

@@ -0,0 +1,47 @@
+FROM {{ namespace }}/{{ infra_image_prefix }}prometheus-base:{{ tag }}
+{% block labels %}
+LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build_date }}"
+{% endblock %}
+
+{% import "macros.j2" as macros with context %}
+
+{% block prometheus_libvirt_exporter_header %}{% endblock %}
+
+{% if base_package_type == 'rpm' %}
+    {% set prometheus_libvirt_exporter_packages = [
+        'git',
+        'go',
+        'libvirt-devel',
+    ] %}
+{% elif base_package_type == 'deb' %}
+    {% set prometheus_libvirt_exporter_packages = [
+        'build-essential',
+        'git',
+        'golang-go',
+        'libvirt-dev',
+        'pkg-config',
+    ] %}
+{% endif %}
+
+{{ macros.install_packages(prometheus_libvirt_exporter_packages | customizable("packages")) }}
+
+{% block prometheus_libvirt_exporter_version %}
+ARG prometheus_libvirt_exporter_version=2.3.0
+ARG prometheus_libvirt_exporter_path=github.com/AlexZzz/libvirt-exporter
+{% endblock %}
+
+{% block prometheus_libvirt_exporter_install %}
+ENV GOPATH=/build
+RUN go mod init libvirt-exporter \
+    && cd ${GOPATH} \
+    && go get -v ${prometheus_libvirt_exporter_path}@${prometheus_libvirt_exporter_version} \
+    && go build ${prometheus_libvirt_exporter_path} \
+    && mv /build/libvirt-exporter /opt \
+    && rm -rf /build
+
+{% endblock %}
+
+{% block prometheus_libvirt_exporter_footer %}{% endblock %}
+{% block footer %}{% endblock %}
+
+USER root

kolla/common/config.py

@@ -249,7 +249,9 @@
     cfg.BoolOpt('summary', default=True,
                 help='Show summary at the end of build'),
     cfg.BoolOpt('infra-rename', default=False,
-                help='Rename infrastructure images to infra')
+                help='Rename infrastructure images to infra'),
+    cfg.StrOpt('repos-yaml', default='',
+               help='Path to alternative repos.yaml file'),
 ]
 
 _BASE_OPTS = [

kolla/common/utils.py

@@ -21,21 +21,33 @@ def make_a_logger(conf=None, image_name=None):
         log = logging.getLogger(".".join([__name__, image_name]))
     else:
         log = logging.getLogger(__name__)
+
+    if conf is not None and conf.debug:
+        loglevel = logging.DEBUG
+    else:
+        loglevel = logging.INFO
+
     if not log.handlers:
-        if conf is None or not conf.logs_dir or not image_name:
-            handler = logging.StreamHandler(sys.stderr)
-            log.propagate = False
+        stream_handler = logging.StreamHandler(sys.stderr)
+        stream_handler.setFormatter(logging.Formatter(logging.BASIC_FORMAT))
+        # NOTE(hrw): quiet mode matters only on console
+        if conf is not None and conf.quiet:
+            stream_handler.setLevel(logging.CRITICAL)
         else:
+            stream_handler.setLevel(loglevel)
+        log.addHandler(stream_handler)
+        log.propagate = False
+
+        if conf is not None and conf.logs_dir and image_name:
             filename = os.path.join(conf.logs_dir, "%s.log" % image_name)
             handler = logging.FileHandler(filename, delay=True)
-        handler.setFormatter(logging.Formatter(logging.BASIC_FORMAT))
-        log.addHandler(handler)
-    if conf is not None and conf.debug:
-        log.setLevel(logging.DEBUG)
-    elif conf is not None and conf.quiet and image_name:
-        log.setLevel(logging.CRITICAL)
-    else:
-        log.setLevel(logging.INFO)
+            # NOTE(hrw): logfile will be INFO or DEBUG
+            handler.setLevel(loglevel)
+            handler.setFormatter(logging.Formatter(logging.BASIC_FORMAT))
+            log.addHandler(handler)
+
+    # NOTE(hrw): needs to be high, handlers have own levels
+    log.setLevel(logging.DEBUG)
     return log