From 61f0b12008e3e284dc0f2a2e2eb9fd082c477ce0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Piliszek?= Date: Mon, 28 Jun 2021 06:57:10 +0000 Subject: [PATCH 1/9] [CI] Do not set ansible_python_interpreter for Zuul Zuul 4.6.0 does not allow to set ansible_python_interpreter. [1] Instead, with the current Zuul and Ansible, this should be automatically set to the proper python. This patch is required to restore the jobs which are ignored otherwise. [2] [3] Additionally, this change avoids the use of Ansible's pip module because it tries to use setuptools from the ansible_python_interpreter first even if another executable is set. [1] http://lists.openstack.org/pipermail/openstack-discuss/2021-June/023291.html [2] http://lists.openstack.org/pipermail/openstack-discuss/2021-June/023326.html [3] http://lists.openstack.org/pipermail/openstack-discuss/2021-June/023321.html Change-Id: Ib74eecd46617af51e23a3ccad664767b1bf6e04f --- .zuul.d/base.yaml | 1 - tests/playbooks/pre.yml | 9 ++++----- tests/playbooks/publish.yml | 9 +++------ 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/.zuul.d/base.yaml b/.zuul.d/base.yaml index 785ce7999e..7af5c36195 100644 --- a/.zuul.d/base.yaml +++ b/.zuul.d/base.yaml @@ -110,7 +110,6 @@ - ^tox\.ini$ vars: publisher: false - ansible_python_interpreter: python3 extra-vars: kolla_logs_dir: "{{ zuul_output_dir }}/logs/kolla" kolla_build_logs_dir: "{{ kolla_logs_dir }}/build" diff --git a/tests/playbooks/pre.yml b/tests/playbooks/pre.yml index 2c289ef0b4..30eb8c9fc7 100644 --- a/tests/playbooks/pre.yml +++ b/tests/playbooks/pre.yml @@ -38,12 +38,11 @@ when: ansible_os_family == "Debian" + - name: Create virtualenv + command: python3 -m virtualenv {{ virtualenv_path }} --python python3 + - name: Install kolla - pip: - name: . - chdir: "{{ zuul.project.src_dir }}" - virtualenv: "{{ virtualenv_path }}" - virtualenv_python: python3 + command: "{{ virtualenv_path }}/bin/python -m pip install {{ zuul.project.src_dir }}" - name: Configure Docker repo for Debian/Ubuntu block: diff --git a/tests/playbooks/publish.yml b/tests/playbooks/publish.yml index 6c688b6641..33a497b094 100644 --- a/tests/playbooks/publish.yml +++ b/tests/playbooks/publish.yml @@ -1,12 +1,9 @@ --- - hosts: all + vars: + # NOTE(yoctozepto): We need Docker SDK, the best source is Kolla venv. + ansible_python_interpreter: "{{ virtualenv_path }}/bin/python" tasks: - - name: Ensure docker python SDK is installed - become: true - pip: - name: "docker" - state: present - - name: List all containers docker_host_info: images: yes From 9fde574c6ae3f137172def7fd94d8baa57371b7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Nasiadka?= Date: Thu, 1 Jul 2021 09:19:01 +0200 Subject: [PATCH 2/9] CI: Fix epel url override epel-release now installs epel*.repos with following baseurl: baseurl=https://download.example/pub/epel/$releasever/Everything/$basearch Change-Id: I19f57d258021ff660df6b6b95e8205b69f3bea85 (cherry picked from commit acc8cb9903aa2a97342d1ca76bc9e274361194c1) --- tests/templates/template_overrides.j2 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/templates/template_overrides.j2 b/tests/templates/template_overrides.j2 index 1bf9d72310..7fdf25fb81 100644 --- a/tests/templates/template_overrides.j2 +++ b/tests/templates/template_overrides.j2 @@ -45,8 +45,8 @@ RUN sed -i \ /etc/yum.repos.d/CentOS-*.repo \ && sed -i \ -e "s|^\(metalink.*\)|#\1|" \ - -e "s|^#baseurl=http://download.fedoraproject.org/pub|baseurl=http://{{ nodepool_mirror_host }}|" \ - -e "s|^#baseurl=https://download.fedoraproject.org/pub|baseurl=http://{{ nodepool_mirror_host }}|" \ + -e "s|^#baseurl=http://download.example/pub|baseurl=http://{{ nodepool_mirror_host }}|" \ + -e "s|^#baseurl=https://download.example/pub|baseurl=http://{{ nodepool_mirror_host }}|" \ /etc/yum.repos.d/epel*.repo {%- raw -%} {% endblock %} @@ -97,7 +97,7 @@ RUN sed -i \ /etc/yum.repos.d/CentOS-*.repo \ && sed -i \ -e "s|^#\(metalink.*\)|\1|" \ - -e "s|^baseurl=http://{{ nodepool_mirror_host }}|baseurl=http://download.fedoraproject.org/pub|" \ + -e "s|^baseurl=http://{{ nodepool_mirror_host }}|#baseurl=http://download.example/pub|" \ /etc/yum.repos.d/epel*.repo {% raw %} {% elif base_package_type == 'deb' %} From b9eac37a6fe71307b78727a8d0bf810698457f26 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 13 May 2021 09:06:19 +0100 Subject: [PATCH 3/9] gnocchi: add python3-rados to gnocchi-base source image The binary images include python3-rados as a dependency, but source images do not. This change fixes that. Change-Id: I2c8cdfcd25856ecdcfd9f302965187b3b62376ad Closes-Bug: #1927756 (cherry picked from commit c29f818696c74fd32b929a763695d244e112a33c) --- docker/gnocchi/gnocchi-base/Dockerfile.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/gnocchi/gnocchi-base/Dockerfile.j2 b/docker/gnocchi/gnocchi-base/Dockerfile.j2 index 8c16047b78..a54ca94c2a 100644 --- a/docker/gnocchi/gnocchi-base/Dockerfile.j2 +++ b/docker/gnocchi/gnocchi-base/Dockerfile.j2 @@ -50,6 +50,7 @@ RUN {{ macros.install_pip(gnocchi_base_pip_packages | customizable("pip_packages {% set gnocchi_base_packages = [ 'librados2-devel', + 'python3-rados', ] %} {{ macros.install_packages(gnocchi_base_packages | customizable("packages")) }} @@ -58,6 +59,7 @@ RUN mkdir -p /var/www/cgi-bin/gnocchi {% set gnocchi_base_packages = [ 'librados-dev', + 'python3-rados', ] %} {{ macros.install_packages(gnocchi_base_packages | customizable("packages")) }} From cbc404a7265402355d1ae754aecac6fae0201013 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABl=20THEROND=20=28Fl1nt=29?= Date: Tue, 6 Jul 2021 12:04:17 +0200 Subject: [PATCH 4/9] Fix prometheus-elasticsearch-exporter container. * Replace current URL with the proper target. Change-Id: I0bb8b2429906e698858f6b699e4c4d8bacc2dd74 Closes-Bug: #1934763 (cherry picked from commit 1f8ac21622295fa9f858aaf7308160a92fe69240) --- .../prometheus/prometheus-elasticsearch-exporter/Dockerfile.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/prometheus/prometheus-elasticsearch-exporter/Dockerfile.j2 b/docker/prometheus/prometheus-elasticsearch-exporter/Dockerfile.j2 index e035496599..dab3eb4c6c 100644 --- a/docker/prometheus/prometheus-elasticsearch-exporter/Dockerfile.j2 +++ b/docker/prometheus/prometheus-elasticsearch-exporter/Dockerfile.j2 @@ -10,7 +10,7 @@ ENV elasticsearch_exporter_version=1.1.0 {% endblock %} {% block prometheus_elasticsearch_exporter_install %} -RUN curl -sSL -o /tmp/elasticsearch_exporter.tar.gz https://github.com/justwatchcom/elasticsearch_exporter/releases/download/v${elasticsearch_exporter_version}/elasticsearch_exporter-${elasticsearch_exporter_version}.linux-{{debian_arch}}.tar.gz \ +RUN curl -sSL -o /tmp/elasticsearch_exporter.tar.gz https://github.com/prometheus-community/elasticsearch_exporter/releases/download/v${elasticsearch_exporter_version}/elasticsearch_exporter-${elasticsearch_exporter_version}.linux-{{debian_arch}}.tar.gz \ && tar xvf /tmp/elasticsearch_exporter.tar.gz -C /opt/ \ && rm -f /tmp/elasticsearch_exporter.tar.gz \ && ln -s /opt/elasticsearch_exporter* /opt/elasticsearch_exporter From be0373c92859070b24665b5d4146c87bec9ccee5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABl=20THEROND=20=28Fl1nt=29?= Date: Fri, 23 Jul 2021 16:04:52 +0200 Subject: [PATCH 5/9] Fix missing templating block for kolla-toolbox. Change-Id: I4ce8244a3fb034493a75f4096de2640e11b48931 Closes-Bug: #1931867 (cherry picked from commit fb218ff9bf3803188991ae0f1a8f66ffa5176872) --- docker/kolla-toolbox/Dockerfile.j2 | 2 ++ .../add_kolla-toolbox_templating_block-3d284d886816e95a.yml | 5 +++++ 2 files changed, 7 insertions(+) create mode 100644 releasenotes/notes/add_kolla-toolbox_templating_block-3d284d886816e95a.yml diff --git a/docker/kolla-toolbox/Dockerfile.j2 b/docker/kolla-toolbox/Dockerfile.j2 index 23cbd1a8ea..118dcc20db 100644 --- a/docker/kolla-toolbox/Dockerfile.j2 +++ b/docker/kolla-toolbox/Dockerfile.j2 @@ -72,10 +72,12 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build 'virtualenv' ] %} +{% block kolla_toolbox_upper_constraints %} RUN mkdir -p /requirements \ && curl -sSL -o /requirements/upper-constraints.txt https://releases.openstack.org/constraints/upper/{{ openstack_release }} \ && {{ macros.install_pip(kolla_toolbox_pip_virtualenv_packages | customizable("pip_virtualenv_packages")) }} \ && virtualenv --system-site-packages {{ virtualenv_path }} +{% endblock %} ENV PATH {{ virtualenv_path }}/bin:$PATH diff --git a/releasenotes/notes/add_kolla-toolbox_templating_block-3d284d886816e95a.yml b/releasenotes/notes/add_kolla-toolbox_templating_block-3d284d886816e95a.yml new file mode 100644 index 0000000000..4fd78af68e --- /dev/null +++ b/releasenotes/notes/add_kolla-toolbox_templating_block-3d284d886816e95a.yml @@ -0,0 +1,5 @@ +--- +features: + - | + Add templating block for kolla-toolbox image helping to customize + upper-constraints URL used at build time for offline build scenario. From 1aba4e72a26165f4d2092430baef1350614ec479 Mon Sep 17 00:00:00 2001 From: Maksim Malchuk Date: Tue, 22 Jun 2021 00:57:16 +0300 Subject: [PATCH 6/9] Add boto3 as s3 dependency for Glance container boto3 library is needed [1] for glance_store and didn't installed yet since [2] adds s3 support back. this change adds both binary and source parts to install boto3 library. 1. https://github.com/openstack/glance_store/blob/04e5ead7c000211a4c10104ed2bb65c9df7681ae/setup.cfg#L76 2. I203134837319080ead96da69048baf90086d2117 Closes-Bug: #1884259 Change-Id: I199185e24cedd2e282c53460a24aeffc83478a12 Signed-off-by: Maksim Malchuk (cherry picked from commit 53b391d3612a322ad13e1ec65e2ed9ba7e94a3c1) --- docker/glance/glance-base/Dockerfile.j2 | 4 +++- releasenotes/notes/bug-1884259-23bdaa6c1c038a81.yaml | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/bug-1884259-23bdaa6c1c038a81.yaml diff --git a/docker/glance/glance-base/Dockerfile.j2 b/docker/glance/glance-base/Dockerfile.j2 index 5a691473fc..dbc19dcbc6 100644 --- a/docker/glance/glance-base/Dockerfile.j2 +++ b/docker/glance/glance-base/Dockerfile.j2 @@ -15,6 +15,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% if base_package_type == 'rpm' %} {% set glance_base_packages = [ 'openstack-glance', + 'python3-boto3', 'python3-oslo-vmware', 'python3-rados', 'python3-rbd' @@ -22,6 +23,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% elif base_package_type == 'deb' %} {% set glance_base_packages = [ 'glance', + 'python3-boto3', 'python3-os-brick', 'python3-oslo.vmware', 'python3-rados', @@ -55,7 +57,7 @@ ADD glance-base-archive /glance-base-source {% set glance_base_pip_packages = [ '/glance', - 'glance_store[cinder,vmware,swift]' + 'glance_store[cinder,vmware,swift,s3]' ] %} # add missing rootwrap config present in glance_store repo diff --git a/releasenotes/notes/bug-1884259-23bdaa6c1c038a81.yaml b/releasenotes/notes/bug-1884259-23bdaa6c1c038a81.yaml new file mode 100644 index 0000000000..2d7f2a2c06 --- /dev/null +++ b/releasenotes/notes/bug-1884259-23bdaa6c1c038a81.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes missing boto3 library required by glance_store. + `LP#1884259 `__ From 865e3f1bb848f3dbf64bb2e52dd51bc3209e9719 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABl=20THEROND=20=28Fl1nt=29?= Date: Thu, 17 Jun 2021 16:42:42 +0200 Subject: [PATCH 7/9] Improve offline build scenario. * Use distribution based dumb-init package. Change-Id: I75927b6016bdb7577fbff354632bdab6407f359c Related-Bug: #19311980 (cherry picked from commit 5940d1758fabf47d15bac5c08d0afa57fbb35964) --- docker/base/Dockerfile.j2 | 15 ++++----------- .../improve_offline_support-e7b2384fb7390184.yaml | 5 +++++ 2 files changed, 9 insertions(+), 11 deletions(-) create mode 100644 releasenotes/notes/improve_offline_support-e7b2384fb7390184.yaml diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2 index 90d2089a4e..e603e4bdd4 100644 --- a/docker/base/Dockerfile.j2 +++ b/docker/base/Dockerfile.j2 @@ -248,6 +248,7 @@ RUN {{ macros.install_packages(base_centos_yum_repo_packages | customizable("cen {% set base_centos_packages = [ 'ca-certificates', 'curl', + 'dumb-init', 'findutils', 'iproute', 'iscsi-initiator-utils', @@ -319,6 +320,7 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom {% set base_apt_packages = [ 'apt-utils', 'curl', + 'dumb-init', 'gawk', 'iproute2', 'kmod', @@ -330,8 +332,8 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom 'python3-pip', 'socat', 'sudo', - 'tgt'] -%} + 'tgt' +] %} {% if base_distro == 'ubuntu' %} {# 391A9AA2147192839E9DB0315EDB1B62EC4926EA -- Canonical Cloud Archive Signing Key #} @@ -416,16 +418,7 @@ COPY httpd_setup.sh /usr/local/bin/kolla_httpd_setup COPY sudoers /etc/sudoers {% if use_dumb_init %} - -{% block dumb_init_installation %} - -RUN curl -sSL https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_{{debian_arch}} -o /usr/local/bin/dumb-init \ - && chmod +x /usr/local/bin/dumb-init - -{% endblock %} - ENTRYPOINT ["dumb-init", "--single-child", "--"] - {% endif %} {% if docker_healthchecks %} diff --git a/releasenotes/notes/improve_offline_support-e7b2384fb7390184.yaml b/releasenotes/notes/improve_offline_support-e7b2384fb7390184.yaml new file mode 100644 index 0000000000..ba262ae164 --- /dev/null +++ b/releasenotes/notes/improve_offline_support-e7b2384fb7390184.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Improve the way offline scenario are supported: + * Switching dumb-init installation to distribution provided packages. From 2877c2c40e571fda90ce1352a633b8b95cf9003e Mon Sep 17 00:00:00 2001 From: Marcin Juszkiewicz Date: Thu, 28 Jan 2021 10:06:00 +0100 Subject: [PATCH 8/9] Add support for CentOS 8 Stream Change-Id: Ibb23c9d0caf115fcbba7bd322ecebe8d80b6821a (cherry picked from commit be544f4e66a6d0dbaa0c26e4565ccfe6a2548838) --- .zuul.d/base.yaml | 12 +++ .zuul.d/centos.yaml | 81 +++++++++++++++++++ doc/source/admin/image-building.rst | 8 ++ doc/source/support_matrix.rst | 17 ++-- docker/base/Dockerfile.j2 | 6 +- docker/monasca/monasca-grafana/Dockerfile.j2 | 1 + .../centos-8-stream-b5b45ccee94f7cf5.yaml | 5 ++ tests/playbooks/run.yml | 17 +++- 8 files changed, 136 insertions(+), 11 deletions(-) create mode 100644 releasenotes/notes/centos-8-stream-b5b45ccee94f7cf5.yaml diff --git a/.zuul.d/base.yaml b/.zuul.d/base.yaml index 785ce7999e..dcf29f5e1b 100644 --- a/.zuul.d/base.yaml +++ b/.zuul.d/base.yaml @@ -57,6 +57,12 @@ - name: primary label: centos-8 +- nodeset: + name: kolla-centos8-stream + nodes: + - name: primary + label: centos-8-stream + - nodeset: name: kolla-ubuntu-focal nodes: @@ -75,6 +81,12 @@ - name: primary label: centos-8-arm64 +- nodeset: + name: kolla-centos8-stream-aarch64 + nodes: + - name: primary + label: centos-8-stream-arm64 + - nodeset: name: kolla-debian-buster-aarch64 nodes: diff --git a/.zuul.d/centos.yaml b/.zuul.d/centos.yaml index a04d6f085b..beb5283a91 100644 --- a/.zuul.d/centos.yaml +++ b/.zuul.d/centos.yaml @@ -4,30 +4,40 @@ jobs: - kolla-build-centos8-binary - kolla-build-centos8-source + - kolla-build-centos8s-binary + - kolla-build-centos8s-source - kolla-ansible-centos8-source - kolla-ansible-centos8-binary - kolla-ansible-centos8-source-upgrade check-arm64: jobs: - kolla-build-centos8-source-aarch64 + - kolla-build-centos8s-source-aarch64 gate: queue: kolla jobs: - kolla-build-centos8-source + - kolla-build-centos8s-source - kolla-ansible-centos8-source - kolla-ansible-centos8-source-upgrade periodic: jobs: - kolla-publish-centos8-source-quay - kolla-publish-centos8-binary-quay + - kolla-publish-centos8s-source-quay + - kolla-publish-centos8s-binary-quay periodic-weekly: jobs: - kolla-publish-centos8-source-dockerhub - kolla-publish-centos8-binary-dockerhub + - kolla-publish-centos8s-source-dockerhub + - kolla-publish-centos8s-binary-dockerhub release: jobs: - kolla-publish-centos8-source-dockerhub - kolla-publish-centos8-binary-dockerhub + - kolla-publish-centos8s-source-dockerhub + - kolla-publish-centos8s-binary-dockerhub experimental: jobs: - kolla-build-no-infra-wheels-centos8-source @@ -66,6 +76,17 @@ base_distro: centos install_type: binary +- job: + name: kolla-build-centos8s-binary + parent: kolla-base + nodeset: kolla-centos8-stream + voting: false + vars: + base_distro: centos + base_image: quay.io/centos/centos + base_tag: stream8 + install_type: binary + - job: name: kolla-publish-centos8-binary-dockerhub parent: kolla-build-centos8-binary @@ -77,6 +98,17 @@ secrets: - kolla_dockerhub_creds +- job: + name: kolla-publish-centos8s-binary-dockerhub + parent: kolla-build-centos8s-binary + post-run: tests/playbooks/publish.yml + vars: + publisher: true + kolla_registry: dockerhub + kolla_namespace: kolla + secrets: + - kolla_dockerhub_creds + - job: name: kolla-publish-centos8-binary-quay parent: kolla-build-centos8-binary @@ -88,6 +120,17 @@ secrets: - kolla_quay_io_creds +- job: + name: kolla-publish-centos8s-binary-quay + parent: kolla-build-centos8s-binary + post-run: tests/playbooks/publish.yml + vars: + publisher: true + kolla_registry: quay.io + kolla_namespace: openstack.kolla + secrets: + - kolla_quay_io_creds + - job: name: kolla-build-centos8-source parent: kolla-base @@ -96,12 +139,28 @@ base_distro: centos install_type: source +- job: + name: kolla-build-centos8s-source + parent: kolla-base + nodeset: kolla-centos8-stream + vars: + base_distro: centos + base_image: quay.io/centos/centos + base_tag: stream8 + install_type: source + - job: name: kolla-build-centos8-source-aarch64 parent: kolla-build-centos8-source nodeset: kolla-centos8-aarch64 voting: false +- job: + name: kolla-build-centos8s-source-aarch64 + parent: kolla-build-centos8s-source + nodeset: kolla-centos8-stream-aarch64 + voting: false + - job: name: kolla-publish-centos8-source-dockerhub parent: kolla-build-centos8-source @@ -113,6 +172,17 @@ secrets: - kolla_dockerhub_creds +- job: + name: kolla-publish-centos8s-source-dockerhub + parent: kolla-build-centos8s-source + post-run: tests/playbooks/publish.yml + vars: + publisher: true + kolla_registry: dockerhub + kolla_namespace: kolla + secrets: + - kolla_dockerhub_creds + - job: name: kolla-publish-centos8-source-quay parent: kolla-build-centos8-source @@ -124,6 +194,17 @@ secrets: - kolla_quay_io_creds +- job: + name: kolla-publish-centos8s-source-quay + parent: kolla-build-centos8s-source + post-run: tests/playbooks/publish.yml + vars: + publisher: true + kolla_registry: quay.io + kolla_namespace: openstack.kolla + secrets: + - kolla_quay_io_creds + - job: name: kolla-build-no-infra-wheels-centos8-source parent: kolla-build-no-infra-wheels-base diff --git a/doc/source/admin/image-building.rst b/doc/source/admin/image-building.rst index 069f67e2d6..1e51e135ed 100644 --- a/doc/source/admin/image-building.rst +++ b/doc/source/admin/image-building.rst @@ -78,6 +78,14 @@ See the :ref:`support matrix ` for information on supported base image distribution versions and supported images on each distribution. +The Victoria release supports both CentOS 8 Linux and CentOS 8 Stream base +container images. The default is to use CentOS 8 Linux. CentOS 8 Stream +may be used as follows: + +.. code-block:: console + + kolla-build --base centos --base-image quay.io/centos/centos --base-tag stream8 + It is possible to build only a subset of images by specifying them on the command line: diff --git a/doc/source/support_matrix.rst b/doc/source/support_matrix.rst index 9c1d13af24..87a6cfa1bb 100644 --- a/doc/source/support_matrix.rst +++ b/doc/source/support_matrix.rst @@ -19,14 +19,15 @@ The following base container images are supported: CentOS 7 is no longer supported as a base container image. The Train release supports both CentOS 7 and 8 images, and provides a route for migration. -============= ============ ================ -Distribution Default base Default base tag -============= ============ ================ -CentOS 8 centos 8 -Debian Buster debian 10 -RHEL 8 rhel 8 -Ubuntu Focal ubuntu 20.04 -============= ============ ================ +=============== ============ =============================== ================ +Distribution Default base Default base image Default base tag +=============== ============ =============================== ================ +CentOS 8 Linux centos centos 8 +CentOS 8 Stream centos quay.io/centos/centos stream8 +Debian Buster debian debian 10 +RHEL 8 rhel rhel 8 +Ubuntu Focal ubuntu ubuntu 20.04 +=============== ============ =============================== ================ The remainder of this document outlines which images are supported on which of these distribution. diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2 index 90d2089a4e..35dd1f0956 100644 --- a/docker/base/Dockerfile.j2 +++ b/docker/base/Dockerfile.j2 @@ -146,7 +146,10 @@ RUN if [[ -e /etc/yum.repos.d/rabbitmq_rabbitmq-server.repo ]]; then \ 'glibc-all-langpacks' ] %} -RUN {{ macros.install_packages(base_centos_language_packages | customizable("centos_language_packages"), chain=True, clean=False) }} \ +# NOTE(hrw): this macro file drops all languages other than C.UTF-8 so horizon fails +# https://bugzilla.redhat.com/show_bug.cgi?id=1729770 +RUN rm -f /etc/rpm/macros.image-language-conf \ + && {{ macros.install_packages(base_centos_language_packages | customizable("centos_language_packages"), chain=True, clean=False) }} \ && {{ macros.rpm_security_update(clean_package_cache) }} {% endblock %} @@ -249,6 +252,7 @@ RUN {{ macros.install_packages(base_centos_yum_repo_packages | customizable("cen 'ca-certificates', 'curl', 'findutils', + 'hostname', 'iproute', 'iscsi-initiator-utils', 'lvm2', diff --git a/docker/monasca/monasca-grafana/Dockerfile.j2 b/docker/monasca/monasca-grafana/Dockerfile.j2 index 7c10d02e88..0adefcaad3 100644 --- a/docker/monasca/monasca-grafana/Dockerfile.j2 +++ b/docker/monasca/monasca-grafana/Dockerfile.j2 @@ -62,6 +62,7 @@ RUN gem install fpm \ && tar --strip 1 -xvf /tmp/monasca-grafana.tgz -C ${monasca_grafana_build_path} \ && rm -f /tmp/monasca-grafana.tgz \ && cd ${monasca_grafana_build_path} \ + && export GO111MODULE=auto \ && go run build.go setup \ && npm config set prefix /usr/local -g \ && npm install npm@6.x -g \ diff --git a/releasenotes/notes/centos-8-stream-b5b45ccee94f7cf5.yaml b/releasenotes/notes/centos-8-stream-b5b45ccee94f7cf5.yaml new file mode 100644 index 0000000000..7cd352ed5e --- /dev/null +++ b/releasenotes/notes/centos-8-stream-b5b45ccee94f7cf5.yaml @@ -0,0 +1,5 @@ +--- +other: + - | + CentOS images are now buildable using CentOS 8 Stream + as a base. diff --git a/tests/playbooks/run.yml b/tests/playbooks/run.yml index 47cdf74b44..2035f427cf 100644 --- a/tests/playbooks/run.yml +++ b/tests/playbooks/run.yml @@ -3,7 +3,8 @@ vars_files: - ../vars/zuul.yml vars: - tag_suffix: "{{ '-aarch64' if ansible_architecture == 'aarch64' else '' }}" + arch_suffix: "{{ '-aarch64' if ansible_architecture == 'aarch64' else '' }}" + tag_suffix: "{{ '-centos8s' if ansible_distribution | lower == 'centos' and base_tag == 'stream8' else '' }}" kolla_build_config: DEFAULT: debug: true @@ -22,12 +23,24 @@ mode: 0777 become: true + - name: Add base_image and base_tag config + vars: + kolla_base_image_config: + DEFAULT: + base_image: "{{ base_image }}" + base_tag: "{{ base_tag }}" + set_fact: + kolla_build_config: "{{ kolla_build_config | combine(kolla_base_image_config, recursive=True) }}" + when: + - base_tag is defined + - base_image is defined + - name: Add publisher config vars: kolla_publisher_config: DEFAULT: namespace: "{{ kolla_namespace }}" - tag: "{{ (zuul.tag if zuul.pipeline == 'release' else zuul.branch | basename) ~ tag_suffix }}" + tag: "{{ (zuul.tag if zuul.pipeline == 'release' else zuul.branch | basename) ~ arch_suffix ~ tag_suffix }}" set_fact: kolla_build_config: "{{ kolla_build_config | combine(kolla_publisher_config, recursive=True) }}" when: From 3978b25cfcb0f871ed6dde718f3b8d76f0d3f0b5 Mon Sep 17 00:00:00 2001 From: Scott Shambarger Date: Sun, 23 May 2021 20:34:46 -0700 Subject: [PATCH 9/9] monasca-thresh: Allow topology check and removal in storm Patch adds a script in the monasca-thresh image that can be used to check if a topology exists in Storm, and optionally kill it. This is part of a bug in kolla-ansible where topologies were not submitted to Storm, but run locally. This patch includes a topology check script enabled by KOLLA_BOOTSTRAP which will exit kolla_start if the topology exists, and optionally enables topology removal (to allow replacement) enabled by TOPOLOGY_REPLACE. Topology names and various timeouts may be customized. If the new env variables are not set, existing behavior is unchanged. Partial-Bug: #1808805 Change-Id: If8f0730031435dda4235b7f2d2c23e5f5f767f87 --- docker/monasca/monasca-thresh/Dockerfile.j2 | 8 +- docker/monasca/monasca-thresh/extend_start.sh | 6 ++ .../monasca-thresh/topology_bootstrap.sh | 90 +++++++++++++++++++ .../notes/bug-1808805-e63af01591f03506.yaml | 8 ++ 4 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 docker/monasca/monasca-thresh/topology_bootstrap.sh create mode 100644 releasenotes/notes/bug-1808805-e63af01591f03506.yaml diff --git a/docker/monasca/monasca-thresh/Dockerfile.j2 b/docker/monasca/monasca-thresh/Dockerfile.j2 index 554fd31f0b..acf5a99421 100644 --- a/docker/monasca/monasca-thresh/Dockerfile.j2 +++ b/docker/monasca/monasca-thresh/Dockerfile.j2 @@ -66,8 +66,14 @@ RUN cd /monasca-common-source/java \ # Overwrite the script inherited from Storm COPY extend_start.sh /usr/local/bin/kolla_extend_start + +# Add bootstrap script +COPY topology_bootstrap.sh /usr/local/bin/topology_bootstrap + RUN touch /usr/local/bin/kolla_monasca_extend_start \ - && chmod 755 /usr/local/bin/kolla_extend_start /usr/local/bin/kolla_monasca_extend_start + && chmod 755 /usr/local/bin/kolla_extend_start \ + /usr/local/bin/kolla_monasca_extend_start \ + /usr/local/bin/topology_bootstrap {% block monasca_thresh_footer %}{% endblock %} {% block footer %}{% endblock %} diff --git a/docker/monasca/monasca-thresh/extend_start.sh b/docker/monasca/monasca-thresh/extend_start.sh index 6c4dda35d5..62451a8e16 100644 --- a/docker/monasca/monasca-thresh/extend_start.sh +++ b/docker/monasca/monasca-thresh/extend_start.sh @@ -42,3 +42,9 @@ if [[ $(ls -Ab ${MONASCA_WORKER_DIR}) != "" ]]; then fi . /usr/local/bin/kolla_monasca_extend_start + +# Bootstrap and exit if KOLLA_BOOTSTRAP variable is set. This catches all cases +# of the KOLLA_BOOTSTRAP variable being set, including empty. +if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then + . /usr/local/bin/topology_bootstrap +fi diff --git a/docker/monasca/monasca-thresh/topology_bootstrap.sh b/docker/monasca/monasca-thresh/topology_bootstrap.sh new file mode 100644 index 0000000000..923d745e59 --- /dev/null +++ b/docker/monasca/monasca-thresh/topology_bootstrap.sh @@ -0,0 +1,90 @@ +#!/bin/sh + +# This script should be sourced by kolla_extend_start when bootstrapping +# +# Optional env(): +# TOPOLOGY_NAME("monasca-thresh") - topology name to check +# TOPOLOGY_KILL_TIMEOUT(5) - secs to wait for topology kill +# STORM_WAIT_RETRIES(24) - retries to check for storm +# STORM_WAIT_TIMEOUT(20) - secs to wait for storm list +# STORM_WAIT_DELAY(5) - secs between storm list attempts + +# - If topology exists, then: +# a) if TOPOLOGY_REPLACE is set, the existing topology is killed +# and script falls through (topology may be added) +# b) otherwise script exits with 0 (topology already exists) +# - If topology doesn't exist, script falls through (topology may be added) +# - If storm cannot be reached, or kill fails, script exits with 1 + +TOPOLOGY_NAME=${TOPOLOGY_NAME:-monasca-thresh} +TOPOLOGY_KILL_TIMEOUT=${TOPOLOGY_KILL_TIMEOUT:-5} + +# defaults from monasca-thresh +STORM_WAIT_RETRIES=${STORM_WAIT_RETRIES:-24} +STORM_WAIT_TIMEOUT=${STORM_WAIT_TIMEOUT:-20} +STORM_WAIT_DELAY=${STORM_WAIT_DELAY:-5} + +STORM="/opt/storm/bin/storm" + +echo "Waiting for storm to become available..." +success="false" +for i in $(seq "$STORM_WAIT_RETRIES"); do + if timeout "$STORM_WAIT_TIMEOUT" "$STORM" list; then + echo "Storm is available, continuing..." + success="true" + break + else + echo "Connection attempt $i of $STORM_WAIT_RETRIES failed" + sleep "$STORM_WAIT_DELAY" + fi +done + +if [ "$success" != "true" ]; then + echo "Unable to connect to Storm! Exiting..." + sleep 1 + exit 1 +fi + +locate_topology() { # + echo "Searching for topology $1 in the storm" + topologies=$("$STORM" list | awk '/-----/,0{if (!/-----/)print $1}') + found="false" + for topology in $topologies; do + if [ "$topology" = "$1" ]; then + echo "Found storm topology with name: $topology" + found="true" + break + fi + done +} + +# search for existing topology +locate_topology "$TOPOLOGY_NAME" + +if [ "$found" = "true" ]; then + + if [[ ! "${!TOPOLOGY_REPLACE[@]}" ]]; then + echo "Topology $TOPOLOGY_NAME found, submission not necessary" + exit 0 + fi + + echo "Topology replacement requested, killing old one..." + "$STORM" kill "$TOPOLOGY_NAME" -w "$TOPOLOGY_KILL_TIMEOUT" + + echo "Wait $TOPOLOGY_KILL_TIMEOUT secs for topology to reap its artifacts..." + sleep "$TOPOLOGY_KILL_TIMEOUT" + + for i in $(seq "$STORM_WAIT_RETRIES"); do + locate_topology "$TOPOLOGY_NAME" + [ "$found" != "true" ] && break + echo "... wait some more..." + sleep "$STORM_WAIT_DELAY" + done + if [ "$found" = "true" ]; then + echo "Unable to kill existing topology, giving up..." + exit 1 + fi + echo "Topology successfully killed, continuing..." +else + echo "Topology not found, continuing..." +fi diff --git a/releasenotes/notes/bug-1808805-e63af01591f03506.yaml b/releasenotes/notes/bug-1808805-e63af01591f03506.yaml new file mode 100644 index 0000000000..60597c1d01 --- /dev/null +++ b/releasenotes/notes/bug-1808805-e63af01591f03506.yaml @@ -0,0 +1,8 @@ +--- +fixes: + - | + Adds an option to the monasca-thresh container which checks + if the topology is currently submitted (KOLLA_BOOTSTRAP), with + an option to kill it (TOPOLOGY_REPLACE). Topology names + and various timeouts may be customized. + `LP#1808805 `__