From 4cab35b1a81055c3696e5fa42d794d6fc96076e6 Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Tue, 12 Nov 2024 10:19:54 +0000 Subject: [PATCH 1/7] Change Bifrost Python upper-constraints source Users can override the upper-constraints.txt by setting the source of openstack-base in kolla-build.conf. However, bifrost-base Dockerfile so far doesn't pass overridden upper-constraints file to Bifrost install playbook resulting the playbook not acknowledging it and just use upstream one. This change sets the source of Bifrost upper-constraints to use upper-constraints of openstack-base for consistency among all Kolla images With this, change the environment variable UPPER_CONSTRAINTS_FILE is only affecting kolla-toolbox image. Therefore, moved UPPER_CONSTRATINS_FILE variable example to the ``kolla_toolbox_pip_conf`` block of kolla-toolbox Dockerfile. Additionally, updated 'Python packages build options' section of documentation to represent latest method of changing Python package related options (including this change). Change-Id: I0a9c782ca1177ef84e4323cb1e5bfac0fb3f383c --- doc/source/admin/image-building.rst | 68 +++++++++++++++++-- docker/base/Dockerfile.j2 | 1 - docker/bifrost/bifrost-base/Dockerfile.j2 | 1 + docker/kolla-toolbox/Dockerfile.j2 | 6 +- ...r-constraints-source-e93cb72e88823d56.yaml | 14 ++++ 5 files changed, 83 insertions(+), 7 deletions(-) create mode 100644 releasenotes/notes/change-bifrost-upper-constraints-source-e93cb72e88823d56.yaml diff --git a/doc/source/admin/image-building.rst b/doc/source/admin/image-building.rst index aad3c33a3e..02abb19b93 100644 --- a/doc/source/admin/image-building.rst +++ b/doc/source/admin/image-building.rst @@ -320,11 +320,69 @@ Python packages build options The block ``base_pip_conf`` in the ``base`` Dockerfile can be used to provide the PyPI build customisation options via the standard environment variables -like ``PIP_INDEX_URL``, ``PIP_TRUSTED_HOST``, etc. Also here can be provided -the standard environment variable ``UPPER_CONSTRAINTS_FILE`` used for building -the ``bifrost_deploy`` container when PyPI upper-constraints needs to be -overridden. Also this variable would be used in the ``kolla-toolbox`` if -provided instead of the defaults. +like ``PIP_INDEX_URL``, ``PIP_TRUSTED_HOST``, etc. + +To override PYPI upper-constraints of all OpenStack images, you can +define the source location of openstack-base. in ``kolla-build.conf``. + +Upstream repository of `openstack-base (requirements) `__ +has a source of +`upper constraints file `__. + +Make a fork or clone the repository then customise ``upper-constraints.txt`` +and define the location of ``openstack-base`` in ``kolla_build.conf``. + +.. path /etc/kolla/kolla-build.conf +.. code-block:: ini + + # These examples use upstream openstack-base as a demonstration + # To use custom openstack-base, make changes accordingly + + # Using git source + [openstack-base] + type = git + location = https://opendev.org/openstack/requirements + reference = master + + # Using URL source + [openstack-base] + type = url + location = https://tarballs.opendev.org/openstack/requirements/requirements-master.tar.gz + + # Using local source + [openstack-base] + type = local + location = /home/kolla/src/requirements + +To remove or change the version of specific Python packages in +``openstack-base`` upper-constraints, you can use the block +``openstack_base_override_upper_constraints`` in your template file, +for example, ``template-overrides.j2``: + +.. code-block:: jinja + + {% block openstack_base_override_upper_constraints %} + RUN {{ macros.upper_constraints_version_change("sqlparse", "0.4.4", "0.5.0") }} + RUN {{ macros.upper_constraints_remove("reno") }} + {% endblock %} + +``kolla-toolbox`` image needs different approach as it does not uses +``openstack-base`` as a base image. +A variable ``UPPER_CONSTRAINTS_FILE`` is set in the +Dockerfile of ``kolla-toolbox``. +To change variable, add the following contents to the +``kolla_toolbox_pip_conf`` block in your template file, for example, +``template-overrides.j2``: + +.. code-block:: jinja + + {% block kolla_toolbox_pip_conf %} + ENV UPPER_CONSTRAINTS_FILE=https://releases.openstack.org/constraints/upper/master + {% endblock %} + +.. note:: + + ``UPPER_CONSTRAINTS_FILE`` must be a valid URL to the file Plugin functionality -------------------- diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2 index bb61c83078..c7ec3de1f0 100644 --- a/docker/base/Dockerfile.j2 +++ b/docker/base/Dockerfile.j2 @@ -379,7 +379,6 @@ RUN touch /usr/local/bin/kolla_extend_start \ # the variables like PIP_INDEX_URL, PIP_EXTRA_INDEX_URL, PIP_TRUSTED_HOST etc. should be defined here. # ENV PIP_INDEX_URL=https://pypi.python.org/simple # ENV PIP_TRUSTED_HOST=pypi.python.org -# ENV UPPER_CONSTRAINTS_FILE=https://releases.openstack.org/constraints/upper/{{ openstack_release }} {% endblock %} {% block base_footer %}{% endblock %} diff --git a/docker/bifrost/bifrost-base/Dockerfile.j2 b/docker/bifrost/bifrost-base/Dockerfile.j2 index 3d0fe726f3..8aa8f54aec 100644 --- a/docker/bifrost/bifrost-base/Dockerfile.j2 +++ b/docker/bifrost/bifrost-base/Dockerfile.j2 @@ -45,6 +45,7 @@ RUN echo " " && \ $VENV/bin/ansible-galaxy collection install -r /bifrost/ansible-collections-requirements.yml && \ $VENV/bin/ansible-playbook -vvvv -i /bifrost/playbooks/inventory/target \ /bifrost/playbooks/install.yaml \ + -e upper_constraints_file="/requirements/upper-constraints.txt" \ -e git_branch={{ openstack_branch_slashed }} \ -e ipa_upstream_release={{ openstack_branch }} \ -e enable_inspector=true \ diff --git a/docker/kolla-toolbox/Dockerfile.j2 b/docker/kolla-toolbox/Dockerfile.j2 index 7a60674af3..29c3ab640e 100644 --- a/docker/kolla-toolbox/Dockerfile.j2 +++ b/docker/kolla-toolbox/Dockerfile.j2 @@ -78,9 +78,13 @@ RUN cd /usr/bin && \ ln -s python3.12 python3 {% endif %} +{% block kolla_toolbox_pip_conf %} +ENV UPPER_CONSTRAINTS_FILE=https://releases.openstack.org/constraints/upper/{{ openstack_release }} +{% endblock %} + {% block kolla_toolbox_upper_constraints %} RUN mkdir -p /requirements \ - && curl -o /requirements/upper-constraints.txt ${UPPER_CONSTRAINTS_FILE:-https://releases.openstack.org/constraints/upper/{{ openstack_release }}} \ + && curl -o /requirements/upper-constraints.txt $UPPER_CONSTRAINTS_FILE \ && {{ macros.upper_constraints_remove("openstacksdk") }} \ && python3 -m venv --system-site-packages {{ venv_path }} \ && KOLLA_DISTRO_PYTHON_VERSION=$(/usr/bin/python3 -c "import sys; print('{}.{}'.format(sys.version_info.major, sys.version_info.minor))") \ diff --git a/releasenotes/notes/change-bifrost-upper-constraints-source-e93cb72e88823d56.yaml b/releasenotes/notes/change-bifrost-upper-constraints-source-e93cb72e88823d56.yaml new file mode 100644 index 0000000000..0b3b233540 --- /dev/null +++ b/releasenotes/notes/change-bifrost-upper-constraints-source-e93cb72e88823d56.yaml @@ -0,0 +1,14 @@ +--- +features: + - | + Updated the contents of Python package options in documentation to reflect + latest customisation methods. +upgrade: + - | + The use of ``UPPER_CONSTRAINTS_FILE`` environment variable was moved from + ``base_pip_conf`` block in base Dockerfile to ``kolla_toolbox_pip_conf`` + block in kolla-toolbox Dockerfile as the environment variable now only + affects Kolla-toolbox build. + Bifrost python upper-constraints now follows openstack-base's + python upper-constraints instead of ``UPPER_CONSTRAINTS_FILE`` environment + variable. From e1fff6252581ee221c1972e671abd80f24393fae Mon Sep 17 00:00:00 2001 From: Grzegorz Koper Date: Thu, 20 Mar 2025 10:23:16 +0100 Subject: [PATCH 2/7] kolla-toolbox: Bump ansible-core to 2.18 Change-Id: I0bdef4252d4f2850d2b1a40feac3c373e5c453e7 --- docker/kolla-toolbox/Dockerfile.j2 | 2 +- releasenotes/notes/ansible-core-2.18-582103463aba6e8a.yaml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/ansible-core-2.18-582103463aba6e8a.yaml diff --git a/docker/kolla-toolbox/Dockerfile.j2 b/docker/kolla-toolbox/Dockerfile.j2 index e9b38fc808..d0695d4d5b 100644 --- a/docker/kolla-toolbox/Dockerfile.j2 +++ b/docker/kolla-toolbox/Dockerfile.j2 @@ -91,7 +91,7 @@ RUN mkdir -p /requirements \ ENV PATH {{ venv_path }}/bin:$PATH {% set kolla_toolbox_pip_packages = [ - 'ansible-core==2.17.*', + 'ansible-core==2.18.*', 'cmd2', 'influxdb', 'openstacksdk', diff --git a/releasenotes/notes/ansible-core-2.18-582103463aba6e8a.yaml b/releasenotes/notes/ansible-core-2.18-582103463aba6e8a.yaml new file mode 100644 index 0000000000..89cd983a23 --- /dev/null +++ b/releasenotes/notes/ansible-core-2.18-582103463aba6e8a.yaml @@ -0,0 +1,4 @@ +--- +upgrade: + - | + Kolla toolbox is now using ``ansible-core 2.18`` From 47a6ad020d7a52b14106938e03e27d65280999cb Mon Sep 17 00:00:00 2001 From: Piotr Milewski Date: Wed, 12 Feb 2025 13:53:14 +0100 Subject: [PATCH 3/7] Opensearch and opensearch-dashboard image size reduction When building an image, the RUN command grants permissions which causes the previous layer to be copied. During the test, the image was reduced from 2.84GB to 1.57GB Closes-Bug: #2104007 Change-Id: I7d2367d4c277287f4db2885f5720a35d934f94b7 --- docker/opensearch/opensearch-dashboards/Dockerfile.j2 | 5 ++--- docker/opensearch/opensearch/Dockerfile.j2 | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/docker/opensearch/opensearch-dashboards/Dockerfile.j2 b/docker/opensearch/opensearch-dashboards/Dockerfile.j2 index 4459ec3e62..4edf1bf631 100644 --- a/docker/opensearch/opensearch-dashboards/Dockerfile.j2 +++ b/docker/opensearch/opensearch-dashboards/Dockerfile.j2 @@ -12,9 +12,8 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% set opensearch_dashboards_packages = ['opensearch-dashboards'] %} -{{ macros.install_packages(opensearch_dashboards_packages | customizable("packages")) }} - -RUN chown -R opensearch-dashboards:opensearch-dashboards /usr/share/opensearch-dashboards +{{ macros.install_packages(opensearch_dashboards_packages | customizable("packages")) }} \ + && chown -R opensearch-dashboards:opensearch-dashboards /usr/share/opensearch-dashboards COPY extend_start.sh /usr/local/bin/kolla_extend_start RUN chmod 755 /usr/local/bin/kolla_extend_start diff --git a/docker/opensearch/opensearch/Dockerfile.j2 b/docker/opensearch/opensearch/Dockerfile.j2 index 32d928a21b..600a5a27a2 100644 --- a/docker/opensearch/opensearch/Dockerfile.j2 +++ b/docker/opensearch/opensearch/Dockerfile.j2 @@ -16,9 +16,8 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build ENV OPENSEARCH_INITIAL_ADMIN_PASSWORD="Kolla24OpenSearch" -{{ macros.install_packages(opensearch_packages | customizable("packages")) }} - -RUN chown -R opensearch:opensearch /usr/share/opensearch +{{ macros.install_packages(opensearch_packages | customizable("packages")) }} \ + && chown -R opensearch:opensearch /usr/share/opensearch COPY extend_start.sh /usr/local/bin/kolla_extend_start RUN chmod 755 /usr/local/bin/kolla_extend_start From 057e7ad604b1c529e63b52ca2ac00614860c7f9f Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Tue, 25 Mar 2025 17:28:33 +0100 Subject: [PATCH 4/7] CI: Sort alphabetically zuul templates Change-Id: I0c001c77bae17eace8c9ca9d94828d04101752fe --- .zuul.d/project.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.zuul.d/project.yaml b/.zuul.d/project.yaml index c35dacc86d..a93a1ef017 100644 --- a/.zuul.d/project.yaml +++ b/.zuul.d/project.yaml @@ -1,13 +1,13 @@ --- - project: templates: + - check-requirements - openstack-python3-jobs - openstack-python3-jobs-arm64 - openstack-cover-jobs - - check-requirements + - periodic-stable-jobs - publish-openstack-docs-pti - release-notes-jobs-python3 - - periodic-stable-jobs check: jobs: - kolla-tox-genconfig From a09ba2027cbc9ab540d62a93c09def0626f5e541 Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Tue, 25 Mar 2025 18:48:03 +0100 Subject: [PATCH 5/7] Deprecate bifrost It has long been sort of an abomination of the container world, let's use this deprecation notice as a motivation to work out a kolla-ansible mechanism utilising regular ironic container images to deploy standalone ironic. Change-Id: I6b4e6edc110eebb524396138d24b873cb4ea6726 --- releasenotes/notes/deprecate-bifrost-dd93b6d8eb04cac6.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 releasenotes/notes/deprecate-bifrost-dd93b6d8eb04cac6.yaml diff --git a/releasenotes/notes/deprecate-bifrost-dd93b6d8eb04cac6.yaml b/releasenotes/notes/deprecate-bifrost-dd93b6d8eb04cac6.yaml new file mode 100644 index 0000000000..358147f4ed --- /dev/null +++ b/releasenotes/notes/deprecate-bifrost-dd93b6d8eb04cac6.yaml @@ -0,0 +1,6 @@ +--- +deprecations: + - | + Building ``bifrost-deploy`` container image has been deprecated, in favor + of a new ironic standalone deployment using ``ironic`` container images + that will be implemented in ``kolla-ansible`` in the next cycles. From 0733a13d7be896dac4004212f9eef48a845b26c2 Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Mon, 17 Mar 2025 11:04:05 +0100 Subject: [PATCH 6/7] Switch to 2025.1 sources Change-Id: If0cd6444e5d0c898d199a98204588f8ffa4351b9 --- kolla/common/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kolla/common/config.py b/kolla/common/config.py index 14bfc00ded..678b0db539 100644 --- a/kolla/common/config.py +++ b/kolla/common/config.py @@ -37,7 +37,7 @@ 'rocky': 'Rocky Linux 9.* (Blue Onyx)', 'ubuntu': 'Ubuntu 24.04.* LTS', } -OPENSTACK_RELEASE = 'master' +OPENSTACK_RELEASE = '2025.1' # This is noarch repository so we will use it on all architectures DELOREAN_DEPS = "https://trunk.rdoproject.org/centos9-epoxy/" \ From 6da05fc6d59cb0015f557e291aaa6fe4e836e55b Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Sun, 17 Mar 2024 07:28:19 +0100 Subject: [PATCH 7/7] neutron: Add docker clients for ML2/OVS improvement Docker does not support running a container in a given network namespace (netns) so we need a client installed in the Neutron container images. Related-Bug: #1891469 Change-Id: I5349449183dbb7f4ec2042a21526076fb42591b3 --- docker/base/Dockerfile.j2 | 2 ++ docker/base/docker-ce.repo | 6 ++++++ docker/neutron/neutron-base/Dockerfile.j2 | 4 +++- kolla/template/repos.yaml | 24 +++++++++++++++++++++++ 4 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 docker/base/docker-ce.repo diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2 index c680f458cb..426ec0eaa2 100644 --- a/docker/base/Dockerfile.j2 +++ b/docker/base/Dockerfile.j2 @@ -54,6 +54,7 @@ COPY dnf.conf /etc/dnf/dnf.conf #### BEGIN REPO ENABLEMENT {% set base_yum_repo_files = [ + 'docker-ce.repo', 'grafana.repo', 'influxdb.repo', 'mariadb.repo', @@ -302,6 +303,7 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom {# NOTE(hrw): type field defaults to 'asc' which is used for single keys #} {% set base_remote_apt_keys = [ + {'name': 'docker-ce', 'url': 'https://download.docker.com/linux/debian/gpg'}, {'name': 'grafana', 'url': 'https://rpm.grafana.com/gpg.key'}, {'name': 'influxdb', 'url': 'https://repos.influxdata.com/influxdata-archive_compat.key'}, {'name': 'mariadb', 'url': 'https://downloads.mariadb.com/MariaDB/mariadb-keyring-2019.gpg', 'type': 'gpg'}, diff --git a/docker/base/docker-ce.repo b/docker/base/docker-ce.repo new file mode 100644 index 0000000000..d976c58526 --- /dev/null +++ b/docker/base/docker-ce.repo @@ -0,0 +1,6 @@ +[docker-ce] +name=Docker CE Stable - $basearch +baseurl=https://download.docker.com/linux/centos/$releasever/$basearch/stable +enabled=1 +gpgcheck=1 +gpgkey=https://download.docker.com/linux/centos/gpg diff --git a/docker/neutron/neutron-base/Dockerfile.j2 b/docker/neutron/neutron-base/Dockerfile.j2 index 3d0b748693..6e3eb2ed54 100644 --- a/docker/neutron/neutron-base/Dockerfile.j2 +++ b/docker/neutron/neutron-base/Dockerfile.j2 @@ -7,7 +7,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% import "macros.j2" as macros with context %} -{{ macros.enable_extra_repos(['openvswitch']) }} +{{ macros.enable_extra_repos(['docker-ce', 'openvswitch']) }} {{ macros.configure_user(name='neutron') }} {% if base_package_type == 'rpm' %} @@ -16,6 +16,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build 'conntrack-tools', 'dnsmasq', 'dnsmasq-utils', + 'docker-ce-cli', 'haproxy', 'iproute-tc', 'ipset', @@ -35,6 +36,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build 'conntrack', 'dnsmasq', 'dnsmasq-utils', + 'docker-ce-cli', 'haproxy', 'iproute2', 'ipset', diff --git a/kolla/template/repos.yaml b/kolla/template/repos.yaml index 927e5c4058..86d4508f27 100644 --- a/kolla/template/repos.yaml +++ b/kolla/template/repos.yaml @@ -2,6 +2,7 @@ centos: ceph: "centos-ceph-reef" crb: "crb" + docker-ce: "docker-ce" epel: "epel" erlang: "rabbitmq_rabbitmq-erlang" extras: "extras" @@ -20,6 +21,7 @@ centos: centos-aarch64: ceph: "centos-ceph-reef" crb: "crb" + docker-ce: "docker-ce" epel: "epel" erlang: "copr-hrw-erlang-for-rabbitmq" extras: "extras" @@ -39,6 +41,11 @@ centos-aarch64: # https://www.rabbitmq.com/install-debian.html#apt-cloudsmith debian: + docker-ce: + url: "https://download.docker.com/linux/debian" + suite: "bookworm" + component: "stable" + gpg_key: "docker-ce.asc" erlang: url: "https://ppa.launchpadcontent.net/rabbitmq/rabbitmq-erlang/ubuntu" suite: "jammy" @@ -86,6 +93,11 @@ debian: gpg_key: "rabbitmq.gpg" debian-aarch64: + docker-ce: + url: "https://download.docker.com/linux/debian" + suite: "bookworm" + component: "stable" + gpg_key: "docker-ce.asc" erlang: url: "https://ppa.launchpadcontent.net/rabbitmq/rabbitmq-erlang/ubuntu" suite: "jammy" @@ -138,6 +150,7 @@ debian-aarch64: rocky: ceph: "centos-ceph-reef" crb: "crb" + docker-ce: "docker-ce" epel: "epel" erlang: "rabbitmq_rabbitmq-erlang" extras: "extras" @@ -156,6 +169,7 @@ rocky: rocky-aarch64: ceph: "centos-ceph-reef" crb: "crb" + docker-ce: "docker-ce" epel: "epel" erlang: "copr-hrw-erlang-for-rabbitmq" extras: "extras" @@ -172,6 +186,11 @@ rocky-aarch64: rabbitmq: "rabbitmq_rabbitmq-server" ubuntu: + docker-ce: + url: "https://download.docker.com/linux/ubuntu" + suite: "noble" + component: "stable" + gpg_key: "docker-ce.asc" erlang: url: "https://ppa.launchpadcontent.net/rabbitmq/rabbitmq-erlang/ubuntu" suite: "jammy" @@ -219,6 +238,11 @@ ubuntu: gpg_key: "rabbitmq.gpg" ubuntu-aarch64: + docker-ce: + url: "https://download.docker.com/linux/ubuntu" + suite: "noble" + component: "stable" + gpg_key: "docker-ce.asc" erlang: url: "https://ppa.launchpadcontent.net/rabbitmq/rabbitmq-erlang/ubuntu" suite: "jammy"