From 4a7af468b6c24322fd89b97933b1fee84e7b8d85 Mon Sep 17 00:00:00 2001 From: FelipeAFV Date: Mon, 25 Nov 2024 15:26:25 -0300 Subject: [PATCH 1/7] Adds missing nvme package in cinder-backup The package moved to the cinder-base image since it already added to the cinder-volume in I84c7633765749a5e9ced983505ef290c32154ccc. Closes-Bug: #2089587 Change-Id: I06e61ab388d9ecfa78e5d1fe1ae9cb8602d1cd36 Signed-off-by: FelipeAFV Signed-off-by: Maksim Malchuk --- docker/cinder/cinder-base/Dockerfile.j2 | 4 +++- docker/cinder/cinder-volume/Dockerfile.j2 | 2 -- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/cinder/cinder-base/Dockerfile.j2 b/docker/cinder/cinder-base/Dockerfile.j2 index d7ba7612ac..f3cc00cc99 100644 --- a/docker/cinder/cinder-base/Dockerfile.j2 +++ b/docker/cinder/cinder-base/Dockerfile.j2 @@ -16,13 +16,15 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build 'ceph-common', 'cryptsetup', 'lvm2', + 'nvme-cli', 'qemu-img' ] %} {% elif base_package_type == 'deb' %} {% set cinder_base_packages = [ 'ceph-common', - 'lvm2', 'cryptsetup', + 'lvm2', + 'nvme-cli', 'python3-cephfs', 'python3-rados', 'python3-rbd', diff --git a/docker/cinder/cinder-volume/Dockerfile.j2 b/docker/cinder/cinder-volume/Dockerfile.j2 index 5b36b6503e..e55b05e440 100644 --- a/docker/cinder/cinder-volume/Dockerfile.j2 +++ b/docker/cinder/cinder-volume/Dockerfile.j2 @@ -12,7 +12,6 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build 'lsscsi', 'device-mapper-multipath', 'nfs-utils', - 'nvme-cli', 'nvmetcli', 'sysfsutils', 'targetcli' @@ -22,7 +21,6 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build 'lsscsi', 'multipath-tools', 'nfs-common', - 'nvme-cli', 'sysfsutils', 'targetcli-fb', 'thin-provisioning-tools', From f82b9e3b184a4a8e475af790342a5026ac5705bd Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Wed, 11 Sep 2024 13:07:05 +0200 Subject: [PATCH 2/7] Add ovn-sb-db-relay image with newer ovn-ctl We need newer ovn-ctl with [1] - for now download that from ovn-org/ovn@master - once 25.03 is branched we should get that the normal way. [1]: https://github.com/ovn-org/ovn/commit/2ecba75432cf90ffb5d6a3a53b8aed521ecf0f44 Change-Id: I429f00ed4bdcd24409d14453ab53c5fa11fdb00b --- docker/ovn/ovn-sb-db-relay/Dockerfile.j2 | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 docker/ovn/ovn-sb-db-relay/Dockerfile.j2 diff --git a/docker/ovn/ovn-sb-db-relay/Dockerfile.j2 b/docker/ovn/ovn-sb-db-relay/Dockerfile.j2 new file mode 100644 index 0000000000..4beebaa4fa --- /dev/null +++ b/docker/ovn/ovn-sb-db-relay/Dockerfile.j2 @@ -0,0 +1,15 @@ +FROM {{ namespace }}/{{ image_prefix }}ovn-sb-db-server:{{ tag }} +{% block labels %} +LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build_date }}" +{% endblock %} + +{% block ovn_sb_db_server_header %}{% endblock %} + +{% block ovn_sb_db_relay_ovn_ctl %} +{# TODO(mnasiadka): Switch to 25.03 branch when available, ideally rpm/deb packages will be + available. #} +RUN curl -o /usr/share/ovn/scripts/ovn-ctl https://raw.githubusercontent.com/ovn-org/ovn/refs/heads/main/utilities/ovn-ctl +{% endblock %} + +{% block ovn_sb_db_relay_footer %}{% endblock %} +{% block footer %}{% endblock %} From 96d6a94278b921d63d2207b91a171a644db57c4d Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Mon, 27 May 2024 21:32:20 +0200 Subject: [PATCH 3/7] CI: Move pre tasks into roles Change-Id: If5aa7371d14041d6e0efca866341be2894fd6f87 --- .zuul.d/base.yaml | 2 + roles/kolla-build-deps/tasks/main.yml | 74 ++++++++++ tests/playbooks/pre.yml | 196 +------------------------- 3 files changed, 77 insertions(+), 195 deletions(-) create mode 100644 roles/kolla-build-deps/tasks/main.yml diff --git a/.zuul.d/base.yaml b/.zuul.d/base.yaml index 98c54f74cf..25f571ade6 100644 --- a/.zuul.d/base.yaml +++ b/.zuul.d/base.yaml @@ -126,6 +126,8 @@ run: tests/playbooks/run.yml post-run: tests/playbooks/post.yml attempts: 5 + roles: + - zuul: openstack/ansible-collection-kolla irrelevant-files: - ^.*\.rst$ - ^doc/.* diff --git a/roles/kolla-build-deps/tasks/main.yml b/roles/kolla-build-deps/tasks/main.yml new file mode 100644 index 0000000000..2863cc0604 --- /dev/null +++ b/roles/kolla-build-deps/tasks/main.yml @@ -0,0 +1,74 @@ +--- +- name: Create dir for kolla logs + ansible.builtin.file: + path: "{{ kolla_logs_dir }}" + state: directory + +- name: Dump host info to logs + ansible.builtin.command: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tools/dump_info.sh" + args: + chdir: "{{ kolla_logs_dir }}" + +- name: Create dir for kolla build logs + ansible.builtin.file: + path: "{{ kolla_build_logs_dir }}" + state: directory + +- name: Install Python3 pip and setuptools + ansible.builtin.package: + name: + - python3-pip + - python3-setuptools + become: true + +# NOTE(hrw): On RedHat systems venv is part of python3-libs +- name: Install Python3 venv on Debian systems + ansible.builtin.package: + name: + - python3-venv + become: true + when: ansible_facts.os_family == "Debian" + +- name: Install dbus-python dependencies on Debian systems + ansible.builtin.package: + name: + - python3-dev + - libdbus-glib-1-dev + become: true + when: ansible_facts.os_family == "Debian" + +- name: Install dbus-python dependencies on EL systems + ansible.builtin.package: + name: + - python3-devel + - dbus-glib-devel + - dbus-daemon + become: true + when: ansible_facts.os_family == "RedHat" + +- name: Install kolla in a venv + ansible.builtin.pip: + name: + - "pip" + - "." + state: latest + virtualenv: "{{ virtualenv_path }}" + virtualenv_command: "python3.{{ ansible_facts.python.version.minor }} -m venv" + chdir: "{{ zuul.project.src_dir }}" + +- name: Configure container engine + ansible.builtin.include_role: + name: "{{ container_engine }}" + +- name: Install podman/docker_sdk in a venv + vars: + virtualenv: "{{ virtualenv_path }}" + docker_sdk_virtualenv_owner: "{{ lookup('env', 'USER') }}" + ansible.builtin.include_role: + name: "{{ container_engine }}_sdk" + +- name: Ensure container engine socket is world-writable + ansible.builtin.file: + path: "{{ '/run/docker.sock' if container_engine == 'docker' else '/run/podman/podman.sock' }}" + mode: 0666 + become: true diff --git a/tests/playbooks/pre.yml b/tests/playbooks/pre.yml index 6d59c40497..7ebb660319 100644 --- a/tests/playbooks/pre.yml +++ b/tests/playbooks/pre.yml @@ -4,198 +4,4 @@ - ../vars/zuul.yml roles: - configure-ephemeral - tasks: - - name: Create dir for kolla logs - file: - path: "{{ kolla_logs_dir }}" - state: directory - - - name: Dump host info to logs - command: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tools/dump_info.sh" - args: - chdir: "{{ kolla_logs_dir }}" - - - name: Create dir for kolla build logs - file: - path: "{{ kolla_build_logs_dir }}" - state: directory - - - name: Install Python3 pip and setuptools - package: - name: - - python3-pip - - python3-setuptools - become: true - - # NOTE(hrw): On RedHat systems venv is part of python3-libs - - name: Install Python3 venv on Debian systems - package: - name: - - python3-venv - become: true - when: - ansible_os_family == "Debian" - - - name: Create virtualenv - command: python3 -m venv {{ virtualenv_path }} - - - name: Install kolla - command: "{{ virtualenv_path }}/bin/python -m pip install {{ zuul.project.src_dir }}" - - - name: Install docker python library - command: "{{ virtualenv_path }}/bin/python -m pip install docker" - when: container_engine == "docker" - - - name: Install podman python library - command: "{{ virtualenv_path }}/bin/python -m pip install podman rich" - when: container_engine == "podman" - - - name: Configure Docker repo for Debian/Ubuntu - block: - - name: Add key for Docker APT repository - apt_key: - url: "{{ nodepool_docker_proxy }}/{{ ansible_distribution | lower }}/gpg" - state: present - - - name: Add Docker APT repository - apt_repository: - repo: "deb {{ nodepool_docker_proxy }}/{{ ansible_distribution | lower }} {{ ansible_distribution_release }} stable" - state: present - - - name: Ensure AppArmor is installed - package: - name: apparmor - state: present - - when: - - ansible_os_family == "Debian" - - container_engine == "docker" - become: true - - - name: Configure Docker repo for CentOS - block: - - name: Add Docker repository - yum_repository: - name: docker - description: Docker - baseurl: "{{ nodepool_docker_proxy }}/centos/9/$basearch/stable" - enabled: yes - gpgcheck: yes - gpgkey: "{{ nodepool_docker_proxy }}/centos/gpg" - # module_hotfixes: True # enabled below (dnf, not yum, feature) - - - name: Enable module_hotfixes in Docker YUM repository - lineinfile: - path: /etc/yum.repos.d/docker.repo - line: 'module_hotfixes=True' - when: - - ansible_os_family == "RedHat" - - container_engine == "docker" - become: true - - - name: Configure docker - block: - - name: Ensure /etc/docker exists - become: true - file: - path: /etc/docker - state: directory - mode: 0755 - - - name: Configure registry-mirror in daemon.json - become: true - copy: - dest: /etc/docker/daemon.json - content: | - { - "registry-mirrors": [ - "http://{{ zuul_site_mirror_fqdn }}:8082/" - ] - } - - # NOTE(yoctozepto): We configure Docker before installing it because Debuntu starts services - # during installation. - - name: Install Docker - package: - name: docker-ce - become: true - - - name: Ensure Docker service is started - service: - name: docker - state: started - become: true - - - name: Ensure Docker socket is world-writable - file: - path: /run/docker.sock - mode: 0666 - become: true - when: - - container_engine == "docker" - - - name: Configure podman - block: - - name: Ensure /etc/containers exists - become: true - file: - path: /etc/containers - state: directory - mode: 0755 - - - name: Configure registries.conf - become: true - copy: - dest: /etc/containers/registries.conf - content: | - unqualified-search-registries = ['docker.io'] - - [[registry]] - prefix = "docker.io" - location = "docker.io" - - [[registry.mirror]] - prefix = "docker.io" - location = "{{ zuul_site_mirror_fqdn }}:8082" - - - name: Install Podman - package: - name: podman - become: true - - # NOTE(jangutter): It appears that the default mount option - # in the shipped `/etc/containers/storage.conf` for EL9 based distros - # (Rocky, CentOS Stream, ...) has severe performance implications for - # Kolla builds. This is because enabling `metacopy=on` disables `Native - # Overlay Diff` This can be removed if the config is dropped from those - # distros, or the underlying incompatibility can somehow be addressed. - # Debian based distros do not ship `storage.conf` and seem - # to be unaffected. - - name: Remove metacopy, enable native overlay diff - ini_file: - path: /etc/containers/storage.conf - section: storage.options.overlay - option: mountopt - value: '"nodev"' - become: true - when: ansible_os_family == "RedHat" - - - name: Ensure Podman service is started - service: - name: podman - state: started - become: true - - - name: Ensure Podman socket is world-writable - file: - path: /run/podman/podman.sock - mode: 0666 - become: true - when: - - container_engine == "podman" - - - name: Run multiarch/qemu-user-static image to support cross-arch build - command: - cmd: "{{ container_engine }} run --rm --privileged multiarch/qemu-user-static --reset -p yes" - become: true - when: ansible_facts.architecture != (base_arch | default('x86_64')) + - kolla-build-deps From cfa266a3a2fd2f29ba48ecb93018197af5510649 Mon Sep 17 00:00:00 2001 From: Andriy Kurilin Date: Tue, 17 Sep 2024 13:49:32 +0200 Subject: [PATCH 4/7] Allow to use configure_user macro for external templates Commit 79a234239516b94b1b79c7ddafcc4fbe4e3ba155 introduced an ability to build 'in-house'/'not-built-in' projects with Kolla. This commits extends this feature with an ability to use configure_user macro for 'not-built-in' users. The implementation is done the same way as for 'sources', i.e. engine after registers new section structure as soon as it finds unknown '-user' section. In addition, the documentation and CI are extended to cover ``--docker-dir`` option. Change-Id: I690d9f3829083f2493bf286a1c45764b9699219b --- doc/source/admin/image-building.rst | 92 +++++++++++++++++++ docker/macros.j2 | 3 + kolla/image/kolla_worker.py | 13 ++- kolla/template/methods.py | 6 ++ ...-to-custom-templates-61c143326a35c7ed.yaml | 9 ++ tests/playbooks/run.yml | 17 ++++ tests/templates/docker/releaser/Dockerfile.j2 | 25 +++++ .../templates/docker/releaser/extend_start.sh | 11 +++ 8 files changed, 175 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/bring-configure_user-macro-to-custom-templates-61c143326a35c7ed.yaml create mode 100644 tests/templates/docker/releaser/Dockerfile.j2 create mode 100644 tests/templates/docker/releaser/extend_start.sh diff --git a/doc/source/admin/image-building.rst b/doc/source/admin/image-building.rst index 3f7de30e7e..d3639e0e69 100644 --- a/doc/source/admin/image-building.rst +++ b/doc/source/admin/image-building.rst @@ -525,6 +525,98 @@ The template becomes now: RUN cp /additions/jenkins/jenkins.json /jenkins.json {% endblock %} +Custom docker templates +----------------------- + +In order to unify the process of managing OpenStack-related projects, Kolla +provides a way of building images for external 'non-built-in' projects. + +If the template for a 'non-built-in' project meets Kolla template standards, +an operator can provide a root directory with a template via the +``--docker-dir`` CLI option (can be specified multiple times). + +All Kolla's jinja2 macros should be available the same as for built-in +projects with some notes: + +- The ``configure_user`` macro. As the 'non-built-in' user is unknown to Kolla, + there are no default values for user ID and group ID to use. + To use this macro, an operator should specify "non-default" user details + with ``-user`` configuration section and include info + for ``uid`` and ``gid`` at least. + +Let's look into how an operator can build an image for an in-house project +with Kolla using `openstack/releases `_ +project. + +First, create a ``Dockerfile.j2`` template for the project. + +.. path /home/kolla/custom-kolla-docker-templates/releaser/Dockerfile.j2 +.. code-block:: jinja + + FROM {{ namespace }}/{{ image_prefix }}openstack-base:{{ tag }} + + {% block labels %} + LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build_date }}" + {% endblock %} + + {% block releaser_header %}{% endblock %} + + {% import "macros.j2" as macros with context %} + + {{ macros.configure_user(name='releaser') }} + + RUN ln -s releaser-source/* /releaser \ + && {{ macros.install_pip(['/releaser-source'] | customizable("pip_packages")) }} \ + && mkdir -p /etc/releaser \ + && chown -R releaser: /etc/releaser \ + && chmod 750 /etc/sudoers.d \ + && touch /usr/local/bin/kolla_releaser_extend_start \ + && chmod 644 /usr/local/bin/kolla_extend_start /usr/local/bin/kolla_releaser_extend_start + + {% block footer %}{% endblock %} + +Suggested directory structure: + +.. code-block:: console + + custom-kolla-docker-templates + |__ releaser + |__ Dockerfile.j2 + +Then, modify Kolla's configuration so the engine can download sources and +configure users. + +.. path /etc/kolla/kolla-build.conf +.. code-block:: ini + + [releaser] + type = git + location = https://opendev.org/openstack/releases + reference = master + + [releaser-user] + uid = 53001 + gid = 53001 + +Last pre-check before building a new image - ensure that the new template +is visible for Kolla: + +.. code-block:: console + + $ kolla-build --list-images --docker-dir custom-kolla-docker-templates "^releaser$" + 1 : base + 2 : releaser + 3 : openstack-base + +And finally, build the ``releaser`` image, passing the ``--docker-dir`` +argument: + +.. code-block:: console + + kolla-build --docker-dir custom-kolla-docker-templates "^releaser$" + +Can I use the ``--template-override`` option for custom templates? Yes! + Custom repos ------------ diff --git a/docker/macros.j2 b/docker/macros.j2 index 48c56208c1..34d67335c7 100644 --- a/docker/macros.j2 +++ b/docker/macros.j2 @@ -43,6 +43,9 @@ {% endmacro %} {% macro configure_user(name, groups=None, shell=None, homedir=None) %} +{%- if name not in users %} +{{ raise_error("Failed to find configuration for '" + name + "' user. Try specifying '" + name + "-user' config section.") }} +{%- endif %} {% set user=users[name] %} {%- if not homedir %} {% set homedir='/var/lib/' + name %} diff --git a/kolla/image/kolla_worker.py b/kolla/image/kolla_worker.py index 798c8eaa7d..3a78a7f35b 100644 --- a/kolla/image/kolla_worker.py +++ b/kolla/image/kolla_worker.py @@ -271,6 +271,7 @@ def _get_methods(self): return { 'debian_package_install': jinja_methods.debian_package_install, 'handle_repos': jinja_methods.handle_repos, + 'raise_error': jinja_methods.raise_error, } def get_users(self): @@ -280,7 +281,17 @@ def get_users(self): for section in all_sections: match = re.search('^.*-user$', section) if match: - user = self.conf[match.group(0)] + cfg_group_name = match.group(0) + + if cfg_group_name not in self.conf._groups: + self.conf.register_opts( + common_config.get_user_opts( + None, None, + # cut `-user` suffix + group=cfg_group_name[:-5]), + group=cfg_group_name + ) + user = self.conf[cfg_group_name] ret[match.group(0)[:-5]] = { 'uid': user.uid, 'gid': user.gid, diff --git a/kolla/template/methods.py b/kolla/template/methods.py index 6747504fa1..e9edbfeb01 100644 --- a/kolla/template/methods.py +++ b/kolla/template/methods.py @@ -11,6 +11,8 @@ # limitations under the License. import os +import typing as t + import yaml from jinja2 import pass_context @@ -150,3 +152,7 @@ def handle_repos(context, reponames, mode): commands = "RUN %s" % commands return commands + + +def raise_error(msg: str) -> t.NoReturn: + raise Exception(msg) diff --git a/releasenotes/notes/bring-configure_user-macro-to-custom-templates-61c143326a35c7ed.yaml b/releasenotes/notes/bring-configure_user-macro-to-custom-templates-61c143326a35c7ed.yaml new file mode 100644 index 0000000000..ca7c6559ed --- /dev/null +++ b/releasenotes/notes/bring-configure_user-macro-to-custom-templates-61c143326a35c7ed.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + Extends the support of externally-managed projects provided by the + ``--docker-dir`` option with an ability to use ``configure_user`` + jinja2 macros like Kolla built-in projects. + The operator should specify "non-default" user details with + ``-user`` configuration section and include info for + ``uid`` and ``gid`` at least. diff --git a/tests/playbooks/run.yml b/tests/playbooks/run.yml index f21de9f330..238b2f7bd6 100644 --- a/tests/playbooks/run.yml +++ b/tests/playbooks/run.yml @@ -44,6 +44,23 @@ when: - publisher + - name: Add external docker dir config + vars: + kolla_build_external_docker_config: + DEFAULT: + docker_dir: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tests/templates/docker" + releaser: + type: git + reference: master + location: "https://opendev.org/openstack/releases.git" + releaser-user: + uid: 56000 + gid: 56000 + set_fact: + kolla_build_config: "{{ kolla_build_config | combine(kolla_build_external_docker_config, recursive=True) }}" + when: + - not publisher + - import_role: name: kolla-build-config diff --git a/tests/templates/docker/releaser/Dockerfile.j2 b/tests/templates/docker/releaser/Dockerfile.j2 new file mode 100644 index 0000000000..665302534f --- /dev/null +++ b/tests/templates/docker/releaser/Dockerfile.j2 @@ -0,0 +1,25 @@ +FROM {{ namespace }}/{{ image_prefix }}openstack-base:{{ tag }} + +{% block labels %} +LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build_date }}" +{% endblock %} + +{% block releaser_header %}{% endblock %} + +{% import "macros.j2" as macros with context %} + +{{ macros.configure_user(name='releaser') }} + +COPY extend_start.sh /usr/local/bin/kolla_extend_start + +ADD releaser-archive /releaser-source + +RUN ln -s releaser-source/* /releaser \ + && {{ macros.install_pip(['/releaser'] | customizable("pip_packages")) }} \ + && mkdir -p /etc/releaser \ + && chown -R releaser: /etc/releaser \ + && chmod 750 /etc/sudoers.d \ + && touch /usr/local/bin/kolla_releaser_extend_start \ + && chmod 644 /usr/local/bin/kolla_extend_start /usr/local/bin/kolla_releaser_extend_start + +{% block footer %}{% endblock %} diff --git a/tests/templates/docker/releaser/extend_start.sh b/tests/templates/docker/releaser/extend_start.sh new file mode 100644 index 0000000000..5d5c3b0411 --- /dev/null +++ b/tests/templates/docker/releaser/extend_start.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +if [[ ! -d "/var/log/kolla/releaser" ]]; then + mkdir -p /var/log/kolla/releaser +fi + +if [[ $(stat -c %a /var/log/kolla/releaser) != "755" ]]; then + chmod 755 /var/log/kolla/releaser +fi + +. /usr/local/bin/kolla_releaser_extend_start From 405f95019354b9be45a750e246127bd9c68d2508 Mon Sep 17 00:00:00 2001 From: Michal Arbet Date: Wed, 19 Feb 2025 23:22:07 +0100 Subject: [PATCH 5/7] Remove Git remote origin after cloning sources When kolla is cloning a Git repository during the build process, the `.git/config` file retains the remote origin URL, which may expose sensitive credentials if authentication tokens are embedded in the URL. This fix ensures that the remote is removed after cloning to prevent unintended exposure of credentials. Closes-Bug: #2098904 Change-Id: Iec21fc2363e03133ec77326da7392bc25b40132a --- kolla/image/tasks.py | 1 + releasenotes/notes/bug-2098904-4c5670049a7e1a66.yaml | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 releasenotes/notes/bug-2098904-4c5670049a7e1a66.yaml diff --git a/kolla/image/tasks.py b/kolla/image/tasks.py index a36ba5ceac..28af3d52d0 100644 --- a/kolla/image/tasks.py +++ b/kolla/image/tasks.py @@ -232,6 +232,7 @@ def reset_userinfo(tarinfo): git.Git().clone(source['source'], clone_dir) git.Git(clone_dir).checkout(source['reference']) reference_sha = git.Git(clone_dir).rev_parse('HEAD') + git.Git(clone_dir).remote("remove", "origin") self.logger.debug("Git checkout by reference %s (%s)", source['reference'], reference_sha) except Exception as e: diff --git a/releasenotes/notes/bug-2098904-4c5670049a7e1a66.yaml b/releasenotes/notes/bug-2098904-4c5670049a7e1a66.yaml new file mode 100644 index 0000000000..7d8e3dafb8 --- /dev/null +++ b/releasenotes/notes/bug-2098904-4c5670049a7e1a66.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Removes Git remote URLs after cloning to prevent credential exposure. + `LP#2098904 `__ From af632db5d138253c36a06a6bcd7d6c9b6fa85edc Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Thu, 12 Dec 2024 12:47:55 +0100 Subject: [PATCH 6/7] CI: Use ansible-lint for CI roles and playbooks Removing dbus-python deps because now a-c-k handles that. Depends-On: https://review.opendev.org/c/openstack/ansible-collection-kolla/+/942179 Change-Id: I97a97da73a7ccb27789e979656867e4adfe8a633 --- .ansible-lint | 12 +++++++++ .yamllint | 10 -------- roles/configure-ephemeral/meta/main.yml | 4 +++ roles/configure-ephemeral/tasks/main.yml | 14 +++++------ roles/kolla-build-config/tasks/main.yml | 5 +++- roles/kolla-build-deps/tasks/main.yml | 26 ++++++-------------- roles/kolla-build/tasks/main.yml | 5 ++++ test-requirements.txt | 1 + test-requirements.yml | 5 ++++ tests/playbooks/post.yml | 16 ++++++++---- tests/playbooks/pre.yml | 3 ++- tests/playbooks/publish.yml | 31 +++++++++++++----------- tests/playbooks/run.yml | 19 +++++++++------ tests/vars/zuul.yml | 6 ++++- tox.ini | 4 +-- 15 files changed, 94 insertions(+), 67 deletions(-) create mode 100644 .ansible-lint delete mode 100644 .yamllint create mode 100644 roles/configure-ephemeral/meta/main.yml create mode 100644 test-requirements.yml diff --git a/.ansible-lint b/.ansible-lint new file mode 100644 index 0000000000..a335f18003 --- /dev/null +++ b/.ansible-lint @@ -0,0 +1,12 @@ +--- +exclude_paths: + - .cache/ # implicit unless exclude_paths is defined in config + - .zuul.d/ +offline: true +parseable: true +profile: basic +skip_list: + - package-latest + - role-name +strict: true +use_default_rules: true diff --git a/.yamllint b/.yamllint deleted file mode 100644 index bfc9e04c14..0000000000 --- a/.yamllint +++ /dev/null @@ -1,10 +0,0 @@ ---- -extends: default -ignore: | - .tox/ - -rules: - line-length: disable - truthy: disable - braces: - max-spaces-inside: 1 diff --git a/roles/configure-ephemeral/meta/main.yml b/roles/configure-ephemeral/meta/main.yml new file mode 100644 index 0000000000..a6b0fd849e --- /dev/null +++ b/roles/configure-ephemeral/meta/main.yml @@ -0,0 +1,4 @@ +--- +collections: + - ansible.posix + - community.general diff --git a/roles/configure-ephemeral/tasks/main.yml b/roles/configure-ephemeral/tasks/main.yml index 7d6402c301..d8733a8f93 100644 --- a/roles/configure-ephemeral/tasks/main.yml +++ b/roles/configure-ephemeral/tasks/main.yml @@ -3,7 +3,7 @@ # unallocated ephemeral device attached at /dev/xvde - name: Set ephemeral device if /dev/xvde exists when: ansible_devices["xvde"] is defined - set_fact: + ansible.builtin.set_fact: ephemeral_device: "/dev/xvde" # On other providers, we have a device called "ephemeral0". @@ -11,15 +11,15 @@ when: ephemeral_device is undefined block: - name: Get ephemeral0 device node - command: /sbin/blkid -L ephemeral0 + ansible.builtin.command: /sbin/blkid -L ephemeral0 register: ephemeral0 # rc !=0 is expected - failed_when: False - changed_when: False + failed_when: false + changed_when: false - name: Set ephemeral device if LABEL exists when: "ephemeral0.rc == 0" - set_fact: + ansible.builtin.set_fact: ephemeral_device: "{{ ephemeral0.stdout }}" - name: Configure additional disk (if available) @@ -53,13 +53,13 @@ fstype: ext4 dev: "{{ ephemeral_device }}" - - name: "Ensure {{ configure_ephemeral_mountpoint }} mountpoint is created" + - name: "Ensure mountpoint {{ configure_ephemeral_mountpoint }}" ansible.builtin.file: path: "{{ configure_ephemeral_mountpoint }}" owner: root group: root state: directory - mode: 0755 + mode: "0755" - name: Mount additional filesystem ansible.posix.mount: diff --git a/roles/kolla-build-config/tasks/main.yml b/roles/kolla-build-config/tasks/main.yml index f4d7f3eec4..4460ceb670 100644 --- a/roles/kolla-build-config/tasks/main.yml +++ b/roles/kolla-build-config/tasks/main.yml @@ -1,5 +1,8 @@ --- - name: Ensure kolla-build.conf exists - template: + ansible.builtin.template: src: kolla-build.conf.j2 dest: "{{ kolla_build_config_path }}" + mode: "0660" + owner: zuul + group: zuul diff --git a/roles/kolla-build-deps/tasks/main.yml b/roles/kolla-build-deps/tasks/main.yml index 2863cc0604..ebb5f00fa9 100644 --- a/roles/kolla-build-deps/tasks/main.yml +++ b/roles/kolla-build-deps/tasks/main.yml @@ -3,16 +3,23 @@ ansible.builtin.file: path: "{{ kolla_logs_dir }}" state: directory + mode: "0755" + owner: zuul + group: zuul - name: Dump host info to logs ansible.builtin.command: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tools/dump_info.sh" args: chdir: "{{ kolla_logs_dir }}" + changed_when: true - name: Create dir for kolla build logs ansible.builtin.file: path: "{{ kolla_build_logs_dir }}" state: directory + mode: "0755" + owner: zuul + group: zuul - name: Install Python3 pip and setuptools ansible.builtin.package: @@ -29,23 +36,6 @@ become: true when: ansible_facts.os_family == "Debian" -- name: Install dbus-python dependencies on Debian systems - ansible.builtin.package: - name: - - python3-dev - - libdbus-glib-1-dev - become: true - when: ansible_facts.os_family == "Debian" - -- name: Install dbus-python dependencies on EL systems - ansible.builtin.package: - name: - - python3-devel - - dbus-glib-devel - - dbus-daemon - become: true - when: ansible_facts.os_family == "RedHat" - - name: Install kolla in a venv ansible.builtin.pip: name: @@ -70,5 +60,5 @@ - name: Ensure container engine socket is world-writable ansible.builtin.file: path: "{{ '/run/docker.sock' if container_engine == 'docker' else '/run/podman/podman.sock' }}" - mode: 0666 + mode: "0666" become: true diff --git a/roles/kolla-build/tasks/main.yml b/roles/kolla-build/tasks/main.yml index ec09e4f79a..e5529f381d 100644 --- a/roles/kolla-build/tasks/main.yml +++ b/roles/kolla-build/tasks/main.yml @@ -3,6 +3,9 @@ ansible.builtin.template: src: "{{ kolla_build_template_overrides_path }}" dest: /etc/kolla/template_overrides.j2 + mode: "0660" + owner: zuul + group: zuul when: kolla_build_template_overrides_path | length > 0 - name: Run kolla-build to template out dockerfiles @@ -10,9 +13,11 @@ cmd: >- {{ kolla_build_venv_path }}/bin/kolla-build --template-only --work-dir {{ kolla_build_logs_dir }}/work_dir + changed_when: true - name: Run kolla-build vars: platform: "{{ '--platform linux/arm64' if base_arch == 'aarch64' and base_arch != ansible_facts.architecture }}" ansible.builtin.command: cmd: "{{ kolla_build_venv_path }}/bin/kolla-build {{ platform }}" + changed_when: true diff --git a/test-requirements.txt b/test-requirements.txt index ad35b87960..e9fa093f77 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,3 +1,4 @@ +ansible-lint<25 # MIT bandit!=1.6.0,>=1.1.0 # Apache-2.0 bashate>=0.5.1 # Apache-2.0 coverage!=4.4,>=4.0 # Apache-2.0 diff --git a/test-requirements.yml b/test-requirements.yml new file mode 100644 index 0000000000..51f1743f6f --- /dev/null +++ b/test-requirements.yml @@ -0,0 +1,5 @@ +--- +collections: + - ansible.posix + - community.docker + - community.general diff --git a/tests/playbooks/post.yml b/tests/playbooks/post.yml index 7245b7ba8d..0306638103 100644 --- a/tests/playbooks/post.yml +++ b/tests/playbooks/post.yml @@ -1,13 +1,16 @@ --- -- hosts: all +- name: Zuul post + hosts: all tasks: - name: Dump host info to logs - command: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tools/dump_info.sh" + ansible.builtin.command: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tools/dump_info.sh" args: chdir: "{{ kolla_logs_dir }}" + changed_when: true - name: Collect various info to logs - shell: + changed_when: true + ansible.builtin.shell: cmd: | set +o errexit set -o xtrace @@ -63,7 +66,9 @@ ps -eo user,pid,ppid,lwp,%cpu,%mem,size,rss,cmd > ps.txt # docker related information - (sudo {{ container_engine }} info && sudo {{ container_engine }} system df && sudo {{ container_engine }} system df -v) > {{ container_engine }}-info.txt + (sudo {{ container_engine }} info && \ + sudo {{ container_engine }} system df && \ + sudo {{ container_engine }} system df -v) > {{ container_engine }}-info.txt sudo cp -r /etc/kolla kolla_configs @@ -71,6 +76,7 @@ sudo chown -R {{ ansible_user_id }}: . # Parse build logs and extract pkg install info - {{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tests/files/process_build_logs.py -l build -b {{ base_distro }} > packages-info.txt + {{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tests/files/process_build_logs.py \ + -l build -b {{ base_distro }} > packages-info.txt executable: /bin/bash chdir: "{{ kolla_logs_dir }}" diff --git a/tests/playbooks/pre.yml b/tests/playbooks/pre.yml index 7ebb660319..7733709678 100644 --- a/tests/playbooks/pre.yml +++ b/tests/playbooks/pre.yml @@ -1,5 +1,6 @@ --- -- hosts: all +- name: Zuul pre + hosts: all vars_files: - ../vars/zuul.yml roles: diff --git a/tests/playbooks/publish.yml b/tests/playbooks/publish.yml index 76352d299d..fd9124cd57 100644 --- a/tests/playbooks/publish.yml +++ b/tests/playbooks/publish.yml @@ -1,44 +1,48 @@ --- -- hosts: all +- name: Zuul publish + hosts: all vars: # NOTE(yoctozepto): We need Docker SDK, the best source is Kolla venv. ansible_python_interpreter: "{{ virtualenv_path }}/bin/python" tasks: - name: List all containers - docker_host_info: - images: yes + community.docker.docker_host_info: + images: true images_filters: reference: "{{ kolla_namespace }}/*" register: docker_host_info - - block: + - name: Publish to Dockerhub + when: kolla_registry == 'dockerhub' + block: - name: Login to Dockerhub - docker_login: + community.docker.docker_login: username: "{{ kolla_dockerhub_credentials.username | trim }}" password: "{{ kolla_dockerhub_credentials.password | trim }}" - name: Push built container images - docker_image: + community.docker.docker_image: name: "{{ item.RepoTags.0 }}" - push: yes + push: true source: local loop: "{{ docker_host_info.images }}" register: push_status until: push_status.failed is false retries: 5 - when: kolla_registry == 'dockerhub' - - block: + - name: Publish to quay.io + when: kolla_registry == 'quay.io' + block: - name: Login to quay.io - docker_login: + community.docker.docker_login: registry: quay.io username: "{{ kolla_quay_io_creds.username | trim }}" password: "{{ kolla_quay_io_creds.password | trim }}" - name: Push built container images - docker_image: + community.docker.docker_image: name: "{{ item.RepoTags.0 }}" - push: yes + push: true repository: "quay.io/{{ item.RepoTags.0 }}" source: local loop: "{{ docker_host_info.images }}" @@ -47,7 +51,7 @@ retries: 5 - name: Ensure repository visibility is public - uri: + ansible.builtin.uri: url: "https://quay.io/api/v1/repository/{{ item.RepoTags.0 }}/changevisibility" method: POST headers: @@ -55,4 +59,3 @@ body: '{"visibility": "public"}' body_format: json loop: "{{ docker_host_info.images }}" - when: kolla_registry == 'quay.io' diff --git a/tests/playbooks/run.yml b/tests/playbooks/run.yml index 238b2f7bd6..c946343a53 100644 --- a/tests/playbooks/run.yml +++ b/tests/playbooks/run.yml @@ -1,5 +1,6 @@ --- -- hosts: all +- name: Zuul run + hosts: all vars_files: - ../vars/zuul.yml vars: @@ -18,10 +19,10 @@ network_mode: host tasks: - name: Ensure /etc/kolla exists - file: + ansible.builtin.file: path: /etc/kolla state: directory - mode: 0777 + mode: "0777" become: true - name: Use ubuntu/debian base image from mirror @@ -29,7 +30,7 @@ kolla_mirror_config: DEFAULT: base_image: "quay.io/openstack.kolla/{{ base_distro }}" - set_fact: + ansible.builtin.set_fact: kolla_build_config: "{{ kolla_build_config | combine(kolla_mirror_config, recursive=True) }}" when: base_distro in ['debian', 'ubuntu'] @@ -39,7 +40,7 @@ DEFAULT: namespace: "{{ kolla_namespace }}" tag: "{{ zuul.branch | basename }}-{{ base_distro }}-{{ base_distro_version }}{{ tag_suffix }}" - set_fact: + ansible.builtin.set_fact: kolla_build_config: "{{ kolla_build_config | combine(kolla_publisher_config, recursive=True) }}" when: - publisher @@ -56,13 +57,15 @@ releaser-user: uid: 56000 gid: 56000 - set_fact: + ansible.builtin.set_fact: kolla_build_config: "{{ kolla_build_config | combine(kolla_build_external_docker_config, recursive=True) }}" when: - not publisher - - import_role: + - name: Import kolla-build-config role + import_role: name: kolla-build-config - - import_role: + - name: Import kolla-build role + import_role: name: kolla-build diff --git a/tests/vars/zuul.yml b/tests/vars/zuul.yml index 1c0c99e3db..e49f9d3560 100644 --- a/tests/vars/zuul.yml +++ b/tests/vars/zuul.yml @@ -7,4 +7,8 @@ nodepool_cbs_centos_proxy: "http://{{ zuul_site_mirror_fqdn }}:8080/cbs.centos" nodepool_docker_proxy: "http://{{ zuul_site_mirror_fqdn }}:8080/docker" # NOTE(hrw): wheel cache goes over 80/443 not on 8080 -nodepool_wheel_mirror: "https://{{ zuul_site_mirror_fqdn }}/wheel/{{ ansible_distribution | lower }}-{{ (ansible_os_family == 'Debian') | ternary(ansible_distribution_version, ansible_distribution_major_version) }}-{{ ansible_architecture | lower }}" +nodepool_wheel_mirror: >- + https://{{ zuul_site_mirror_fqdn }}/wheel/{{ ansible_distribution | lower }}- + {{ (ansible_os_family == 'Debian') | + ternary(ansible_distribution_version, ansible_distribution_major_version) }}- + {{ ansible_architecture | lower }} diff --git a/tox.ini b/tox.ini index 923d4d821d..e57c38d0fb 100644 --- a/tox.ini +++ b/tox.ini @@ -50,7 +50,6 @@ commands = deps = {[testenv]deps} codespell - yamllint allowlist_externals = bash commands = bash {toxinidir}/tools/run-bashate.sh @@ -58,8 +57,9 @@ commands = bash {toxinidir}/tools/validate-all-dockerfiles.sh python {toxinidir}/tools/validate-all-file.py bandit -r docker kolla tests tools - yamllint -s . codespell -I {toxinidir}/.codespell-ignore + ansible-galaxy collection install -r test-requirements.yml + ansible-lint [testenv:bandit] commands = bandit -r docker kolla tests tools From 498b793c869384f6564fc0e2d422200321974745 Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Fri, 21 Feb 2025 11:59:07 +0100 Subject: [PATCH 7/7] CI: Add rocky support to process_build_logs.py Change-Id: I57e05a235359547a4477dc869e92047818dc5c15 --- tests/files/process_build_logs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/files/process_build_logs.py b/tests/files/process_build_logs.py index 22b7182273..0db5228d9e 100755 --- a/tests/files/process_build_logs.py +++ b/tests/files/process_build_logs.py @@ -30,7 +30,7 @@ required=True) args = vars(parser.parse_args()) -if args['base'] not in ['centos']: +if args['base'] not in ['rocky', 'centos']: print("Non rpm-based distros are not yet supported.") sys.exit()