From 0d3cec115326c50e4f73c40805a58a585baf9d56 Mon Sep 17 00:00:00 2001 From: Michal Arbet Date: Mon, 30 Sep 2024 17:49:26 +0200 Subject: [PATCH 1/2] Added systemd-standalone-sysusers package for Debuntu This package provides a standalone sysusers binary with minimal dependencies, designed for use in non-systemd environments. This package simply ensures that all packages that want to install systemd as dependency will have to settle for this one because the systemd package breaks systemd-standalone-sysusers, as it is an alternative. This backport is necessary to fix failing Opensearch builds while backporting Noble support to 2024.1. [1] https://packages.debian.org/bookworm/systemd-standalone-sysusers [2] https://packages.ubuntu.com/noble/systemd-standalone-sysusers Needed-by: Ie12c43dc1de122c203edc28181366457ccd67834 Change-Id: I3ba85069c265844bd2f8b5e6d2b14f32e4898cfd (cherry picked from commit 55168f909d3c95b6dfc386d3fbee25fd20a7cae8) --- docker/base/Dockerfile.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2 index 7613befcda..bd3b5011be 100644 --- a/docker/base/Dockerfile.j2 +++ b/docker/base/Dockerfile.j2 @@ -268,6 +268,7 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom {% set base_apt_packages = [ 'apt-utils', 'dumb-init', + 'systemd-standalone-sysusers', 'gawk', 'iproute2', 'kmod', From ce5b6560bc93e5533bf465596644885b2927a333 Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Fri, 2 Feb 2024 15:40:28 +0100 Subject: [PATCH 2/2] Add support for Ubuntu 24.04 LTS Add setuptools because Python3.12 dropped distutils taskflow needs to be bumped to 5.8.0 because that's the first version that supports python3.12 Needed-By: https://review.opendev.org/c/openstack/kolla-ansible/+/932539 Co-Authored-By: Jakub Darmach Change-Id: Ie12c43dc1de122c203edc28181366457ccd67834 (cherry picked from commit 5f9850c5edaa53399b10fb76b0cc1b759aa4e059) --- .zuul.d/base.yaml | 12 +++ .zuul.d/project.yaml | 4 + .zuul.d/ubuntu.yaml | 71 +++++++++++--- doc/source/support_matrix.rst | 6 ++ docker/base/Dockerfile.j2 | 14 ++- ....list.ubuntu => sources.list.ubuntu.jammy} | 0 ...ch64 => sources.list.ubuntu.jammy.aarch64} | 0 docker/base/sources.list.ubuntu.noble | 16 +++ docker/base/sources.list.ubuntu.noble.aarch64 | 15 +++ docker/bifrost/bifrost-base/Dockerfile.j2 | 11 ++- docker/cinder/cinder-base/Dockerfile.j2 | 5 + kolla/common/config.py | 2 +- kolla/image/kolla_worker.py | 5 +- kolla/template/methods.py | 17 ++-- kolla/template/repos-noble.yaml | 97 +++++++++++++++++++ .../notes/ubuntu-24-04-585855b7d4753ce3.yaml | 5 + requirements.txt | 1 + tests/playbooks/run.yml | 10 ++ 18 files changed, 262 insertions(+), 29 deletions(-) rename docker/base/{sources.list.ubuntu => sources.list.ubuntu.jammy} (100%) rename docker/base/{sources.list.ubuntu.aarch64 => sources.list.ubuntu.jammy.aarch64} (100%) create mode 100644 docker/base/sources.list.ubuntu.noble create mode 100644 docker/base/sources.list.ubuntu.noble.aarch64 create mode 100644 kolla/template/repos-noble.yaml create mode 100644 releasenotes/notes/ubuntu-24-04-585855b7d4753ce3.yaml diff --git a/.zuul.d/base.yaml b/.zuul.d/base.yaml index c44a74b311..d853cb9f22 100644 --- a/.zuul.d/base.yaml +++ b/.zuul.d/base.yaml @@ -87,6 +87,12 @@ - name: primary label: ubuntu-jammy +- nodeset: + name: kolla-ubuntu-noble + nodes: + - name: primary + label: ubuntu-noble + - nodeset: name: kolla-debian-bookworm nodes: @@ -117,6 +123,12 @@ - name: primary label: ubuntu-jammy-arm64 +- nodeset: + name: kolla-ubuntu-noble-aarch64 + nodes: + - name: primary + label: ubuntu-noble-arm64 + - job: name: kolla-base parent: base diff --git a/.zuul.d/project.yaml b/.zuul.d/project.yaml index c35dacc86d..7618af0c68 100644 --- a/.zuul.d/project.yaml +++ b/.zuul.d/project.yaml @@ -11,6 +11,10 @@ check: jobs: - kolla-tox-genconfig + - openstack-tox-py312 + check-arm64: + jobs: + - openstack-tox-py312-arm64 gate: jobs: - kolla-tox-genconfig diff --git a/.zuul.d/ubuntu.yaml b/.zuul.d/ubuntu.yaml index a69eb104bd..be1328c45c 100644 --- a/.zuul.d/ubuntu.yaml +++ b/.zuul.d/ubuntu.yaml @@ -2,8 +2,10 @@ - project: check: jobs: - - kolla-build-ubuntu - - kolla-build-ubuntu-podman + - kolla-build-ubuntu-jammy + - kolla-build-ubuntu-noble + - kolla-build-ubuntu-jammy-podman + - kolla-build-ubuntu-noble-podman - kolla-ansible-ubuntu - kolla-ansible-ubuntu-upgrade # Test rabbitmq and mariadb in multinode ceph jobs. @@ -33,22 +35,25 @@ files: ^docker/bifrost/ gate: jobs: - - kolla-build-ubuntu - - kolla-build-ubuntu-podman + - kolla-build-ubuntu-jammy + - kolla-build-ubuntu-noble + - kolla-build-ubuntu-jammy-podman + - kolla-build-ubuntu-noble-podman - kolla-ansible-ubuntu - kolla-ansible-ubuntu-upgrade periodic: jobs: - - kolla-publish-ubuntu-quay + - kolla-publish-ubuntu-jammy-quay + - kolla-publish-ubuntu-noble-quay periodic-weekly: jobs: - - kolla-publish-ubuntu-dockerhub + - kolla-publish-ubuntu-jammy-dockerhub experimental: jobs: - kolla-build-no-infra-wheels-ubuntu - job: - name: kolla-build-ubuntu + name: kolla-build-ubuntu-jammy parent: kolla-base nodeset: kolla-ubuntu-jammy vars: @@ -56,7 +61,16 @@ base_distro_version: jammy - job: - name: kolla-build-ubuntu-podman + name: kolla-build-ubuntu-noble + parent: kolla-base + nodeset: kolla-ubuntu-noble + vars: + base_distro: ubuntu + base_distro_version: noble + base_distro_tag: '24.04' + +- job: + name: kolla-build-ubuntu-jammy-podman parent: kolla-base-podman nodeset: kolla-ubuntu-jammy vars: @@ -64,14 +78,29 @@ base_distro_version: jammy - job: - name: kolla-build-ubuntu-aarch64 - parent: kolla-build-ubuntu + name: kolla-build-ubuntu-noble-podman + parent: kolla-base-podman + nodeset: kolla-ubuntu-noble + vars: + base_distro: ubuntu + base_distro_version: noble + base_distro_tag: '24.04' + +- job: + name: kolla-build-ubuntu-jammy-aarch64 + parent: kolla-build-ubuntu-jammy nodeset: kolla-ubuntu-jammy-aarch64 voting: false - job: - name: kolla-publish-ubuntu-dockerhub - parent: kolla-build-ubuntu + name: kolla-build-ubuntu-noble-aarch64 + parent: kolla-build-ubuntu-noble + nodeset: kolla-ubuntu-noble-aarch64 + voting: false + +- job: + name: kolla-publish-ubuntu-jammy-dockerhub + parent: kolla-build-ubuntu-jammy post-run: tests/playbooks/publish.yml vars: publisher: true @@ -81,8 +110,20 @@ - kolla_dockerhub_credentials - job: - name: kolla-publish-ubuntu-quay - parent: kolla-build-ubuntu + name: kolla-publish-ubuntu-jammy-quay + parent: kolla-build-ubuntu-jammy + post-run: tests/playbooks/publish.yml + vars: + publisher: true + kolla_registry: quay.io + kolla_namespace: openstack.kolla + secrets: + - kolla_quay_io_creds + - kolla_quay_io_api + +- job: + name: kolla-publish-ubuntu-noble-quay + parent: kolla-build-ubuntu-noble post-run: tests/playbooks/publish.yml vars: publisher: true @@ -95,6 +136,6 @@ - job: name: kolla-build-no-infra-wheels-ubuntu parent: kolla-build-no-infra-wheels-base - nodeset: kolla-ubuntu-jammy + nodeset: kolla-ubuntu-noble vars: base_distro: ubuntu diff --git a/doc/source/support_matrix.rst b/doc/source/support_matrix.rst index b2dc5ff3e5..c5638fbcbe 100644 --- a/doc/source/support_matrix.rst +++ b/doc/source/support_matrix.rst @@ -20,8 +20,14 @@ Distribution Default base Default base tag Rocky Linux quay.io/rockylinux/rockylinux 9 Debian Bullseye debian bullseye Ubuntu Jammy ubuntu 22.04 +Ubuntu Noble ubuntu 24.04 ================== =============================== ================ +.. note:: + In order to build Ubuntu Noble based images ``base_tag`` needs to be set + to ``24.04`` (if using a local image from your own registry - the tag needs + to at least start with 24.04, e.g. '24.04-my-version') + The remainder of this document outlines which images are supported on which of these distribution. diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2 index bd3b5011be..ea33be3000 100644 --- a/docker/base/Dockerfile.j2 +++ b/docker/base/Dockerfile.j2 @@ -237,11 +237,21 @@ RUN cat /tmp/kolla_bashrc >> /etc/bash.bashrc \ -e s/#*LAST_SYSTEM_GID=999/LAST_SYSTEM_GID=59999/g /etc/adduser.conf {% block base_ubuntu_package_sources_list %} -{% if base_distro == 'debian' or ( base_distro == 'ubuntu' and base_arch == 'x86_64' ) %} +{% if base_distro == 'debian' %} RUN rm -f /etc/apt/sources.list.d/debian.sources COPY sources.list.{{ base_distro }} /etc/apt/sources.list +{% elif ( base_distro == 'ubuntu' and base_arch == 'x86_64' ) %} +{% if base_distro_tag.startswith('22.04') %} +COPY sources.list.{{ base_distro }}.jammy /etc/apt/sources.list +{% elif base_distro_tag.startswith('24.04') %} +COPY sources.list.{{ base_distro }}.noble /etc/apt/sources.list +{% endif %} {% else %} -COPY sources.list.{{ base_distro }}.{{ base_arch }} /etc/apt/sources.list +{% if base_distro_tag.startswith('22.04') %} +COPY sources.list.{{ base_distro }}.jammy.{{ base_arch }} /etc/apt/sources.list +{% elif base_distro_tag.startswith('24.04') %} +COPY sources.list.{{ base_distro }}.noble.{{ base_arch }} /etc/apt/sources.list +{% endif %} {% endif %} COPY sources.list /etc/apt/sources.list.d/kolla-custom.list {% endblock %} diff --git a/docker/base/sources.list.ubuntu b/docker/base/sources.list.ubuntu.jammy similarity index 100% rename from docker/base/sources.list.ubuntu rename to docker/base/sources.list.ubuntu.jammy diff --git a/docker/base/sources.list.ubuntu.aarch64 b/docker/base/sources.list.ubuntu.jammy.aarch64 similarity index 100% rename from docker/base/sources.list.ubuntu.aarch64 rename to docker/base/sources.list.ubuntu.jammy.aarch64 diff --git a/docker/base/sources.list.ubuntu.noble b/docker/base/sources.list.ubuntu.noble new file mode 100644 index 0000000000..8f6baf853d --- /dev/null +++ b/docker/base/sources.list.ubuntu.noble @@ -0,0 +1,16 @@ +# For non-x86 architectures we use sources.list.ubuntu. + +# Default repos +deb mirror://mirrors.ubuntu.com/mirrors.txt noble main universe +deb mirror://mirrors.ubuntu.com/mirrors.txt noble-updates main universe +deb mirror://mirrors.ubuntu.com/mirrors.txt noble-security main universe + +# Backports have a lower priority and must be explicitly installed to be used +deb http://archive.ubuntu.com/ubuntu/ noble-backports main universe + +# NOTE: In Caracal we don't need to add the repo for the updated packages like qemu, +# libvirt, and openvswitch. + +# NOTE(hrw): extra repositories are added into image when they are needed as +# separate files in /etc/apt/sources.list.d/ directory. For that purpose they +# are defined in kolla/template/repos.yaml file. diff --git a/docker/base/sources.list.ubuntu.noble.aarch64 b/docker/base/sources.list.ubuntu.noble.aarch64 new file mode 100644 index 0000000000..81a5c7a4f8 --- /dev/null +++ b/docker/base/sources.list.ubuntu.noble.aarch64 @@ -0,0 +1,15 @@ +# Default repos +deb http://ports.ubuntu.com/ noble main universe +deb http://ports.ubuntu.com/ noble-updates main universe +deb http://ports.ubuntu.com/ noble-security main universe + +# Backports have a lower priority and must be explicitly installed to be used +deb http://ports.ubuntu.com/ noble-backports main universe + +# We need to add the repo for the updated packages they provide. The main ones +# are qemu, libvirt, and openvswitch. +deb http://ubuntu-cloud.archive.canonical.com/ubuntu noble-updates/dalmatian main + +# NOTE(hrw): extra repositories are added into image when they are needed as +# separate files in /etc/apt/sources.list.d/ directory. For that purpose they +# are defined in kolla/template/repos.yaml file. diff --git a/docker/bifrost/bifrost-base/Dockerfile.j2 b/docker/bifrost/bifrost-base/Dockerfile.j2 index bfe31dbbfc..b6267b670d 100644 --- a/docker/bifrost/bifrost-base/Dockerfile.j2 +++ b/docker/bifrost/bifrost-base/Dockerfile.j2 @@ -37,11 +37,18 @@ ENV ANSIBLE_GATHER_TIMEOUT=30 {% block bifrost_ansible_install %} {%- if base_package_type == 'deb' %} RUN apt-get --error-on=any update && \ + {%- if base_distro_tag.startswith('24.04') %} + bash -c 'export VENV=/var/lib/kolla/venv && \ + {# NOTE(darmach): Bumped to ansible-core 2.16 to match Python 3.12 #} + $VENV/bin/pip install "ansible>=9,<10" && \ + {%- else %} + bash -c '$VENV/bin/pip install "ansible>=6,<7" && \ + {%- endif %} {%- else %} RUN echo " " && \ +bash -c 'export VENV=/var/lib/kolla/venv && \ +$VENV/bin/pip install "ansible>=6,<7" && \ {%- endif %} - bash -c 'export VENV=/var/lib/kolla/venv && \ - $VENV/bin/pip install "ansible>=6,<7" && \ $VENV/bin/ansible-galaxy collection install -r /bifrost/ansible-collections-requirements.yml && \ ansible-playbook -vvvv -i /bifrost/playbooks/inventory/target \ /bifrost/playbooks/install.yaml \ diff --git a/docker/cinder/cinder-base/Dockerfile.j2 b/docker/cinder/cinder-base/Dockerfile.j2 index 4446dcf0e0..ab6d605d01 100644 --- a/docker/cinder/cinder-base/Dockerfile.j2 +++ b/docker/cinder/cinder-base/Dockerfile.j2 @@ -29,6 +29,11 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build 'qemu-utils' ] %} {% endif %} + +{% if base_distro_tag.startswith('24.04') %} +RUN {{ macros.upper_constraints_version_change("taskflow", "5.6.0", "5.8.0") }} +{% endif %} + {{ macros.install_packages(cinder_base_packages | customizable("packages")) }} {% set cinder_base_pip_packages = [ diff --git a/kolla/common/config.py b/kolla/common/config.py index 0c72aa19cc..4b80fed829 100644 --- a/kolla/common/config.py +++ b/kolla/common/config.py @@ -35,7 +35,7 @@ 'centos': 'CentOS Stream 9', 'debian': 'Debian GNU/Linux 12 (bookworm)', 'rocky': 'Rocky Linux 9.* (Blue Onyx)', - 'ubuntu': 'Ubuntu 22.04', + 'ubuntu': 'Ubuntu 2{2,4}.04', } OPENSTACK_RELEASE = '2024.1' diff --git a/kolla/image/kolla_worker.py b/kolla/image/kolla_worker.py index b959ec7540..fe245fd090 100644 --- a/kolla/image/kolla_worker.py +++ b/kolla/image/kolla_worker.py @@ -122,7 +122,10 @@ def __init__(self, conf): self.distro_package_manager = 'apt' self.base_package_type = 'deb' elif self.base in ['ubuntu']: - self.conf.distro_python_version = "3.10" + if self.base_tag.startswith('24.04'): + self.conf.distro_python_version = "3.12" + else: + self.conf.distro_python_version = "3.10" self.distro_package_manager = 'apt' self.base_package_type = 'deb' else: diff --git a/kolla/template/methods.py b/kolla/template/methods.py index 1a59267029..f190845b68 100644 --- a/kolla/template/methods.py +++ b/kolla/template/methods.py @@ -89,20 +89,21 @@ def handle_repos(context, reponames, mode): if not isinstance(reponames, list): raise TypeError("First argument should be a list of repositories") - if context.get('repos_yaml'): - repofile = context.get('repos_yaml') - else: - repofile = os.path.dirname(os.path.realpath(__file__)) + '/repos.yaml' + base_package_type = context.get('base_package_type') + base_distro = context.get('base_distro') + base_distro_tag = context.get('base_distro_tag') + base_arch = context.get('base_arch') + + repofile = context.get('repos_yaml') or ( + os.path.dirname(os.path.realpath(__file__)) + + ('/repos-noble.yaml' if base_distro_tag == '24.04' else '/repos.yaml') + ) with open(repofile, 'r') as repos_file: repo_data = {} for name, params in yaml.safe_load(repos_file).items(): repo_data[name] = params - base_package_type = context.get('base_package_type') - base_distro = context.get('base_distro') - base_arch = context.get('base_arch') - commands = '' try: diff --git a/kolla/template/repos-noble.yaml b/kolla/template/repos-noble.yaml new file mode 100644 index 0000000000..e297421588 --- /dev/null +++ b/kolla/template/repos-noble.yaml @@ -0,0 +1,97 @@ +--- +ubuntu: + erlang: + url: "https://ppa.launchpadcontent.net/rabbitmq/rabbitmq-erlang/ubuntu" + suite: "noble" + component: "main" + gpg_key: "erlang-ppa.gpg" + fluentd: + url: "https://packages.treasuredata.com/lts/5/ubuntu/noble/" + suite: "noble" + component: "contrib" + gpg_key: "treasuredata.asc" + grafana: + url: "https://apt.grafana.com" + suite: "stable" + component: "main" + gpg_key: "grafana.asc" + influxdb: + url: "https://repos.influxdata.com/ubuntu" + # TODO(mnasiadka): Switch to noble when available + suite: "jammy" + component: "stable" + gpg_key: "influxdb.asc" + mariadb: + url: "https://dlm.mariadb.com/repo/mariadb-server/10.11/repo/ubuntu" + suite: "noble" + component: "main" + gpg_key: "mariadb.gpg" + opensearch: + url: "https://artifacts.opensearch.org/releases/bundle/opensearch/2.x/apt/" + suite: "stable" + component: "main" + gpg_key: "opensearch.asc" + opensearch-dashboards: + url: "https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/2.x/apt/" + suite: "stable" + component: "main" + gpg_key: "opensearch.asc" + proxysql: + url: "https://repo.proxysql.com/ProxySQL/proxysql-2.6.x/noble/" + suite: "./" + component: "" + gpg_key: "proxysql.asc" + rabbitmq: + url: "https://ppa1.rabbitmq.com/rabbitmq/rabbitmq-server/deb/ubuntu" + suite: "noble" + component: "main" + gpg_key: "rabbitmq.gpg" + +ubuntu-aarch64: + erlang: + url: "https://ppa.launchpadcontent.net/rabbitmq/rabbitmq-erlang/ubuntu" + suite: "noble" + component: "main" + gpg_key: "erlang-ppa.gpg" + fluentd: + url: "https://packages.treasuredata.com/lts/5/ubuntu/noble/" + suite: "noble" + component: "contrib" + gpg_key: "treasuredata.asc" + grafana: + url: "https://apt.grafana.com" + suite: "stable" + component: "main" + gpg_key: "grafana.asc" + influxdb: + url: "https://repos.influxdata.com/ubuntu" + # TODO(mnasiadka): Switch to noble when available + suite: "jammy" + component: "stable" + gpg_key: "influxdb.asc" + mariadb: + url: "https://dlm.mariadb.com/repo/mariadb-server/10.11/repo/ubuntu" + suite: "noble" + component: "main" + gpg_key: "mariadb.gpg" + opensearch: + url: "https://artifacts.opensearch.org/releases/bundle/opensearch/2.x/apt/" + suite: "stable" + component: "main" + gpg_key: "opensearch.asc" + opensearch-dashboards: + url: "https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/2.x/apt/" + suite: "stable" + component: "main" + gpg_key: "opensearch.asc" + proxysql: + url: "https://repo.proxysql.com/ProxySQL/proxysql-2.6.x/noble/" + suite: "./" + component: "" + gpg_key: "proxysql.asc" + rabbitmq: + url: "https://ppa1.rabbitmq.com/rabbitmq/rabbitmq-server/deb/ubuntu" + suite: "noble" + component: "main" + arch: "amd64" + gpg_key: "rabbitmq.gpg" diff --git a/releasenotes/notes/ubuntu-24-04-585855b7d4753ce3.yaml b/releasenotes/notes/ubuntu-24-04-585855b7d4753ce3.yaml new file mode 100644 index 0000000000..c6f6c0a966 --- /dev/null +++ b/releasenotes/notes/ubuntu-24-04-585855b7d4753ce3.yaml @@ -0,0 +1,5 @@ +--- +upgrade: + - | + Adds support for building Ubuntu Noble 24.04 images when specifying + ``base_tag = 24.04``. diff --git a/requirements.txt b/requirements.txt index f471843cf3..f96132cc73 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ pbr!=2.1.0,>=2.0.0 # Apache-2.0 Jinja2>=3.0.1 # BSD License (3 clause) GitPython>=1.0.1 # BSD License (3 clause) oslo.config>=5.1.0 # Apache-2.0 +setuptools>=64.0.0 # PSF/ZPL; python_version >= '3.12' diff --git a/tests/playbooks/run.yml b/tests/playbooks/run.yml index 0547f0c811..4c430535ee 100644 --- a/tests/playbooks/run.yml +++ b/tests/playbooks/run.yml @@ -23,6 +23,16 @@ mode: 0777 become: true + - name: Add base_tag config + vars: + kolla_base_tag_config: + DEFAULT: + base_tag: "{{ base_distro_tag }}" + set_fact: + kolla_build_config: "{{ kolla_build_config | combine(kolla_base_tag_config, recursive=True) }}" + when: + - base_distro_tag is defined + - name: Use ubuntu/debian base image from mirror vars: kolla_mirror_config: