diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2 index 188f6d2aab..caed6ef56b 100644 --- a/docker/base/Dockerfile.j2 +++ b/docker/base/Dockerfile.j2 @@ -351,8 +351,8 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom {# 901F9177AB97ACBE -- Treasure Data, Inc (Treasure Agent Official Signing key) #} {# A20F259AEB9C94BB -- Sensuapp (Freight) #} {# F1656F24C74CD1D8 -- MariaDB Signing Key #} - {# F77F1EDA57EBB1CC -- Launchpad RabbitMQ Erlang PPA key #} - {# F6609E60DC62814E -- PackageCloud RabbitMQ repository key #} + {# E495BB49CC4BBE5B -- Cloudsmith RabbitMQ Erlang repository key #} + {# 9F4587F226208342 -- Cloudsmith RabbitMQ repository key #} {% set base_apt_keys = [ '391A9AA2147192839E9DB0315EDB1B62EC4926EA', '46095ACC8548582C1A2699A9D27D666CD88E42B4', @@ -362,8 +362,8 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom '901F9177AB97ACBE', 'A20F259AEB9C94BB', 'F1656F24C74CD1D8', - 'F77F1EDA57EBB1CC', - 'F6609E60DC62814E', + 'E495BB49CC4BBE5B', + '9F4587F226208342', ] %} {% set base_remote_apt_keys = [ 'https://packages.grafana.com/gpg.key', @@ -373,8 +373,8 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom {% set base_apt_keys = [ '46095ACC8548582C1A2699A9D27D666CD88E42B4', 'F1656F24C74CD1D8', - 'F77F1EDA57EBB1CC', - 'F6609E60DC62814E', + 'E495BB49CC4BBE5B', + '9F4587F226208342', ] %} {% set base_remote_apt_keys = [ 'https://download.docker.com/linux/debian/gpg', diff --git a/docker/base/healthcheck_curl b/docker/base/healthcheck_curl index 2ef8538bac..6df232fe89 100755 --- a/docker/base/healthcheck_curl +++ b/docker/base/healthcheck_curl @@ -5,7 +5,7 @@ : ${HEALTHCHECK_CURL_OUTPUT:='/dev/null'} export NSS_SDB_USE_CACHE=no -curl -g -k -q -s -S --fail -o "${HEALTHCHECK_CURL_OUTPUT}" \ +curl -q -g -k -s -S --fail -o "${HEALTHCHECK_CURL_OUTPUT}" \ --max-time "${HEALTHCHECK_CURL_MAX_TIME}" \ --user-agent "${HEALTHCHECK_CURL_USER_AGENT}" \ --write-out "${HEALTHCHECK_CURL_WRITE_OUT}" \ diff --git a/docker/elasticsearch/elasticsearch/Dockerfile.j2 b/docker/elasticsearch/elasticsearch/Dockerfile.j2 index d09a0d8175..5633ce6d32 100644 --- a/docker/elasticsearch/elasticsearch/Dockerfile.j2 +++ b/docker/elasticsearch/elasticsearch/Dockerfile.j2 @@ -12,8 +12,12 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {{ macros.enable_extra_repos(['elasticsearch']) }} {% if base_package_type == 'rpm' %} + +# NOTE(hrw): post-install script of elasticsearch fails when trying to +# install elasticsearch and java together. +{{ macros.install_packages(['java-11-openjdk-headless']) }} + {% set elasticsearch_packages = [ - 'java-11-openjdk-headless', 'elasticsearch-oss', ] %} diff --git a/docker/macros.j2 b/docker/macros.j2 index 7076e422c9..10765ad99d 100644 --- a/docker/macros.j2 +++ b/docker/macros.j2 @@ -34,7 +34,7 @@ {% macro install_pip(packages, constraints = true) %} {%- if packages is sequence and packages|length > 0 -%} - python{{ distro_python_version }} -m pip --no-cache-dir install --upgrade{{ ' ' }} + SETUPTOOLS_USE_DISTUTILS=stdlib python{{ distro_python_version }} -m pip --no-cache-dir install --upgrade{{ ' ' }} {%- if constraints %}-c /requirements/upper-constraints.txt {% endif -%} {{ packages | join(' ') }} {%- else -%} diff --git a/docker/masakari/masakari-monitors/Dockerfile.j2 b/docker/masakari/masakari-monitors/Dockerfile.j2 index 50fb62aa94..b337ef0065 100644 --- a/docker/masakari/masakari-monitors/Dockerfile.j2 +++ b/docker/masakari/masakari-monitors/Dockerfile.j2 @@ -19,7 +19,9 @@ RUN echo '{{ install_type }} not yet available for {{ base_distro }}' \ {% if base_package_type == 'rpm' %} {% set masakari_monitors_packages = [ - 'libvirt-devel', + 'cyrus-sasl-md5', + 'cyrus-sasl-scram', + 'python3-libvirt', 'pacemaker-cli', 'tcpdump', ] %} @@ -27,8 +29,10 @@ RUN echo '{{ install_type }} not yet available for {{ base_distro }}' \ {% elif base_package_type == 'deb' %} {% set masakari_monitors_packages = [ - 'libvirt-dev', + 'libsasl2-modules-gssapi-mit', + 'python3-libvirt', 'pacemaker-cli-utils', + 'sasl2-bin', 'tcpdump', ] %} @@ -42,8 +46,10 @@ ADD masakari-monitors-archive /masakari-monitors-source '/masakari-monitors' ] %} +# NOTE(hrw): distros may provide other version of libvirt RUN ln -s masakari-monitors-source/* masakari-monitors \ - {% if distro_package_manager == 'dnf' %}&& sed -i -e 's/libvirt-python===.*/libvirt-python===6.10.0/' /requirements/upper-constraints.txt {% endif %}\ + && sed -i -e "/^libvirt-python/d" /requirements/upper-constraints.txt \ + && sed -i -e "/^libvirt-python/d" /masakari-monitors/requirements.txt \ && {{ macros.install_pip(masakari_monitors_pip_packages | customizable("pip_packages")) }} \ && mkdir -p /etc/masakari-monitors \ && chown -R masakari: /etc/masakari-monitors diff --git a/kolla/common/utils.py b/kolla/common/utils.py index 9cd8ecca9d..0729127284 100644 --- a/kolla/common/utils.py +++ b/kolla/common/utils.py @@ -21,21 +21,33 @@ def make_a_logger(conf=None, image_name=None): log = logging.getLogger(".".join([__name__, image_name])) else: log = logging.getLogger(__name__) + + if conf is not None and conf.debug: + loglevel = logging.DEBUG + else: + loglevel = logging.INFO + if not log.handlers: - if conf is None or not conf.logs_dir or not image_name: - handler = logging.StreamHandler(sys.stderr) - log.propagate = False + stream_handler = logging.StreamHandler(sys.stderr) + stream_handler.setFormatter(logging.Formatter(logging.BASIC_FORMAT)) + # NOTE(hrw): quiet mode matters only on console + if conf is not None and conf.quiet: + stream_handler.setLevel(logging.CRITICAL) else: + stream_handler.setLevel(loglevel) + log.addHandler(stream_handler) + log.propagate = False + + if conf is not None and conf.logs_dir and image_name: filename = os.path.join(conf.logs_dir, "%s.log" % image_name) handler = logging.FileHandler(filename, delay=True) - handler.setFormatter(logging.Formatter(logging.BASIC_FORMAT)) - log.addHandler(handler) - if conf is not None and conf.debug: - log.setLevel(logging.DEBUG) - elif conf is not None and conf.quiet and image_name: - log.setLevel(logging.CRITICAL) - else: - log.setLevel(logging.INFO) + # NOTE(hrw): logfile will be INFO or DEBUG + handler.setLevel(loglevel) + handler.setFormatter(logging.Formatter(logging.BASIC_FORMAT)) + log.addHandler(handler) + + # NOTE(hrw): needs to be high, handlers have own levels + log.setLevel(logging.DEBUG) return log diff --git a/kolla/image/build.py b/kolla/image/build.py index f558655595..efb11f9639 100755 --- a/kolla/image/build.py +++ b/kolla/image/build.py @@ -405,6 +405,15 @@ def followups(self): def process_source(self, image, source): dest_archive = os.path.join(image.path, source['name'] + '-archive') + # NOTE(mgoddard): Change ownership of files to root:root. This + # avoids an issue introduced by the fix for git CVE-2022-24765, + # which breaks PBR when the source checkout is not owned by the + # user installing it. LP#1969096 + def reset_userinfo(tarinfo): + tarinfo.uid = tarinfo.gid = 0 + tarinfo.uname = tarinfo.gname = "root" + return tarinfo + if source.get('type') == 'url': self.logger.debug("Getting archive from %s", source['source']) try: @@ -458,7 +467,8 @@ def process_source(self, image, source): if os.path.isdir(source['source']): with tarfile.open(dest_archive, 'w') as tar: tar.add(source['source'], - arcname=os.path.basename(source['source'])) + arcname=os.path.basename(source['source']), + filter=reset_userinfo) else: shutil.copyfile(source['source'], dest_archive) diff --git a/kolla/template/filters.py b/kolla/template/filters.py index 49708ee3fc..f2fbc6e145 100644 --- a/kolla/template/filters.py +++ b/kolla/template/filters.py @@ -12,11 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -from jinja2 import contextfilter +# NOTE: jinja2 3.1.0 dropped contextfilter in favour of pass_context. +try: + from jinja2 import pass_context +except ImportError: + from jinja2 import contextfilter as pass_context + from jinja2 import Undefined -@contextfilter +@pass_context def customizable(context, val_list, call_type): # NOTE(mgoddard): Don't try to customise undefined values. There are cases # where this might happen, for example using a generic template overrides diff --git a/kolla/template/methods.py b/kolla/template/methods.py index 19fcf4e22f..ba187158df 100644 --- a/kolla/template/methods.py +++ b/kolla/template/methods.py @@ -15,7 +15,11 @@ import os import yaml -from jinja2 import contextfunction +# NOTE: jinja2 3.1.0 dropped contextfunction in favour of pass_context. +try: + from jinja2 import pass_context +except ImportError: + from jinja2 import contextfunction as pass_context def debian_package_install(packages, clean_package_cache=True): @@ -71,7 +75,7 @@ def debian_package_install(packages, clean_package_cache=True): return ' && '.join(cmds) -@contextfunction +@pass_context def handle_repos(context, reponames, mode): """NOTE(hrw): we need to handle CentOS, Debian and Ubuntu with one macro. diff --git a/kolla/template/repos.yaml b/kolla/template/repos.yaml index c51aee40c9..fa7c0a7d50 100644 --- a/kolla/template/repos.yaml +++ b/kolla/template/repos.yaml @@ -47,53 +47,51 @@ rhel: opstools: "centos-opstools" rabbitmq: "centos-rabbitmq-38" -# NOTE(mnasiadka): Erlang repo - Debian Buster/Bullseye needs to use bionic as per RabbitMQ docs debian: - elasticsearch: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" - erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu bionic main" + erlang: "deb https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-erlang/deb/debian buster main" grafana: "deb https://packages.grafana.com/oss/deb stable main" influxdb: "deb https://repos.influxdata.com/debian buster stable" logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" kibana: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-7.x/apt stable main" mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/debian buster main" - rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/debian/ buster main" + rabbitmq: "deb https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-server/deb/debian buster main" td-agent: "deb http://packages.treasuredata.com/4/debian/buster buster contrib" debian-aarch64: elasticsearch: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" - erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu bionic main" + erlang: "deb https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-erlang/deb/debian buster main" grafana: "deb https://packages.grafana.com/oss/deb stable main" influxdb: "deb https://repos.influxdata.com/debian buster stable" logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" kibana: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-7.x/apt stable main" libvirt: "deb https://obs.linaro.org/repos/home:/marcin.juszkiewicz/debian-buster ./" mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/debian buster main" - rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/debian/ buster main" + rabbitmq: "deb https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-server/deb/debian buster main" td-agent: "deb http://packages.treasuredata.com/4/debian/buster buster contrib" ubuntu: elasticsearch: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" - erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu focal main" + erlang: "deb https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-erlang/deb/ubuntu focal main" grafana: "deb https://packages.grafana.com/oss/deb stable main" influxdb: "deb https://repos.influxdata.com/ubuntu focal stable" logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" kibana: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu focal main" qdrouterd: "deb http://ppa.launchpad.net/qpid/released/ubuntu/ focal main" - rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ focal main" + rabbitmq: "deb https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-server/deb/ubuntu focal main" td-agent: "deb http://packages.treasuredata.com/4/ubuntu/focal/ focal contrib" ubuntu-aarch64: elasticsearch: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" - erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu focal main" + erlang: "deb https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-erlang/deb/ubuntu focal main" grafana: "deb https://packages.grafana.com/oss/deb stable main" influxdb: "deb https://repos.influxdata.com/ubuntu focal stable" logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu bionic main" - rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ focal main" + rabbitmq: "deb https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-server/deb/ubuntu focal main" td-agent: "deb http://packages.treasuredata.com/4/ubuntu/focal/ focal contrib" ubuntu-ppc64le: mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu focal main" - rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ focal main" + rabbitmq: "deb https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-server/deb/ubuntu focal main" diff --git a/releasenotes/notes/healthcheck-curl-disable-curlrc-0f85aad47379e2a5.yaml b/releasenotes/notes/healthcheck-curl-disable-curlrc-0f85aad47379e2a5.yaml new file mode 100644 index 0000000000..c1c00d6ce5 --- /dev/null +++ b/releasenotes/notes/healthcheck-curl-disable-curlrc-0f85aad47379e2a5.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes disabling the use of the ``curlrc`` configuration file in + ``healthcheck_curl``. `LP#1967272 `__ diff --git a/releasenotes/notes/jinja2-pass-context-3f3febcd944e3a51.yaml b/releasenotes/notes/jinja2-pass-context-3f3febcd944e3a51.yaml new file mode 100644 index 0000000000..3a7ecc729c --- /dev/null +++ b/releasenotes/notes/jinja2-pass-context-3f3febcd944e3a51.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Fixes an issue seen when using Jinja2 3.1.0. diff --git a/releasenotes/notes/quiet-mode-with-logs-0abafc07923945ac.yaml b/releasenotes/notes/quiet-mode-with-logs-0abafc07923945ac.yaml new file mode 100644 index 0000000000..1b8e67855a --- /dev/null +++ b/releasenotes/notes/quiet-mode-with-logs-0abafc07923945ac.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Quiet mode (enabled with ``--quiet`` argument) can be combined with + ``--logs-dir`` option now. Console output will be quiet as expected while + building output will be stored in separate log files. diff --git a/releasenotes/notes/rabbitmq-erlang-cloudsmith-c837bf4a450dd802.yaml b/releasenotes/notes/rabbitmq-erlang-cloudsmith-c837bf4a450dd802.yaml new file mode 100644 index 0000000000..b87bf97632 --- /dev/null +++ b/releasenotes/notes/rabbitmq-erlang-cloudsmith-c837bf4a450dd802.yaml @@ -0,0 +1,10 @@ +--- +fixes: + - | + Fixes the Debian and Ubuntu images to use rabbitmq and erlang from + cloudsmith so that the images are still buildable and use proper versions. +upgrade: + - | + The Debian and Ubuntu images use rabbitmq and erlang from cloudsmith now. + Operators might want to mirror/proxy this new source as it provides the + correct set of packages unlike the previous combination. diff --git a/tests/playbooks/run.yml b/tests/playbooks/run.yml index 6e319d30f7..ca5df44b1b 100644 --- a/tests/playbooks/run.yml +++ b/tests/playbooks/run.yml @@ -9,6 +9,7 @@ DEFAULT: debug: true logs_dir: "{{ kolla_build_logs_dir }}" + quiet: true base: "{{ base_distro }}" install_type: "{{ install_type }}" template_override: /etc/kolla/template_overrides.j2