diff --git a/.ansible-lint b/.ansible-lint new file mode 100644 index 0000000000..a335f18003 --- /dev/null +++ b/.ansible-lint @@ -0,0 +1,12 @@ +--- +exclude_paths: + - .cache/ # implicit unless exclude_paths is defined in config + - .zuul.d/ +offline: true +parseable: true +profile: basic +skip_list: + - package-latest + - role-name +strict: true +use_default_rules: true diff --git a/.yamllint b/.yamllint deleted file mode 100644 index bfc9e04c14..0000000000 --- a/.yamllint +++ /dev/null @@ -1,10 +0,0 @@ ---- -extends: default -ignore: | - .tox/ - -rules: - line-length: disable - truthy: disable - braces: - max-spaces-inside: 1 diff --git a/.zuul.d/base.yaml b/.zuul.d/base.yaml index 98c54f74cf..25f571ade6 100644 --- a/.zuul.d/base.yaml +++ b/.zuul.d/base.yaml @@ -126,6 +126,8 @@ run: tests/playbooks/run.yml post-run: tests/playbooks/post.yml attempts: 5 + roles: + - zuul: openstack/ansible-collection-kolla irrelevant-files: - ^.*\.rst$ - ^doc/.* diff --git a/doc/source/admin/image-building.rst b/doc/source/admin/image-building.rst index 3f7de30e7e..d3639e0e69 100644 --- a/doc/source/admin/image-building.rst +++ b/doc/source/admin/image-building.rst @@ -525,6 +525,98 @@ The template becomes now: RUN cp /additions/jenkins/jenkins.json /jenkins.json {% endblock %} +Custom docker templates +----------------------- + +In order to unify the process of managing OpenStack-related projects, Kolla +provides a way of building images for external 'non-built-in' projects. + +If the template for a 'non-built-in' project meets Kolla template standards, +an operator can provide a root directory with a template via the +``--docker-dir`` CLI option (can be specified multiple times). + +All Kolla's jinja2 macros should be available the same as for built-in +projects with some notes: + +- The ``configure_user`` macro. As the 'non-built-in' user is unknown to Kolla, + there are no default values for user ID and group ID to use. + To use this macro, an operator should specify "non-default" user details + with ``-user`` configuration section and include info + for ``uid`` and ``gid`` at least. + +Let's look into how an operator can build an image for an in-house project +with Kolla using `openstack/releases `_ +project. + +First, create a ``Dockerfile.j2`` template for the project. + +.. path /home/kolla/custom-kolla-docker-templates/releaser/Dockerfile.j2 +.. code-block:: jinja + + FROM {{ namespace }}/{{ image_prefix }}openstack-base:{{ tag }} + + {% block labels %} + LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build_date }}" + {% endblock %} + + {% block releaser_header %}{% endblock %} + + {% import "macros.j2" as macros with context %} + + {{ macros.configure_user(name='releaser') }} + + RUN ln -s releaser-source/* /releaser \ + && {{ macros.install_pip(['/releaser-source'] | customizable("pip_packages")) }} \ + && mkdir -p /etc/releaser \ + && chown -R releaser: /etc/releaser \ + && chmod 750 /etc/sudoers.d \ + && touch /usr/local/bin/kolla_releaser_extend_start \ + && chmod 644 /usr/local/bin/kolla_extend_start /usr/local/bin/kolla_releaser_extend_start + + {% block footer %}{% endblock %} + +Suggested directory structure: + +.. code-block:: console + + custom-kolla-docker-templates + |__ releaser + |__ Dockerfile.j2 + +Then, modify Kolla's configuration so the engine can download sources and +configure users. + +.. path /etc/kolla/kolla-build.conf +.. code-block:: ini + + [releaser] + type = git + location = https://opendev.org/openstack/releases + reference = master + + [releaser-user] + uid = 53001 + gid = 53001 + +Last pre-check before building a new image - ensure that the new template +is visible for Kolla: + +.. code-block:: console + + $ kolla-build --list-images --docker-dir custom-kolla-docker-templates "^releaser$" + 1 : base + 2 : releaser + 3 : openstack-base + +And finally, build the ``releaser`` image, passing the ``--docker-dir`` +argument: + +.. code-block:: console + + kolla-build --docker-dir custom-kolla-docker-templates "^releaser$" + +Can I use the ``--template-override`` option for custom templates? Yes! + Custom repos ------------ diff --git a/docker/cinder/cinder-base/Dockerfile.j2 b/docker/cinder/cinder-base/Dockerfile.j2 index 0f0b208948..312f128085 100644 --- a/docker/cinder/cinder-base/Dockerfile.j2 +++ b/docker/cinder/cinder-base/Dockerfile.j2 @@ -16,13 +16,15 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build 'ceph-common', 'cryptsetup', 'lvm2', + 'nvme-cli', 'qemu-img' ] %} {% elif base_package_type == 'deb' %} {% set cinder_base_packages = [ 'ceph-common', - 'lvm2', 'cryptsetup', + 'lvm2', + 'nvme-cli', 'python3-cephfs', 'python3-rados', 'python3-rbd', diff --git a/docker/cinder/cinder-volume/Dockerfile.j2 b/docker/cinder/cinder-volume/Dockerfile.j2 index 5b36b6503e..e55b05e440 100644 --- a/docker/cinder/cinder-volume/Dockerfile.j2 +++ b/docker/cinder/cinder-volume/Dockerfile.j2 @@ -12,7 +12,6 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build 'lsscsi', 'device-mapper-multipath', 'nfs-utils', - 'nvme-cli', 'nvmetcli', 'sysfsutils', 'targetcli' @@ -22,7 +21,6 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build 'lsscsi', 'multipath-tools', 'nfs-common', - 'nvme-cli', 'sysfsutils', 'targetcli-fb', 'thin-provisioning-tools', diff --git a/docker/macros.j2 b/docker/macros.j2 index 48c56208c1..34d67335c7 100644 --- a/docker/macros.j2 +++ b/docker/macros.j2 @@ -43,6 +43,9 @@ {% endmacro %} {% macro configure_user(name, groups=None, shell=None, homedir=None) %} +{%- if name not in users %} +{{ raise_error("Failed to find configuration for '" + name + "' user. Try specifying '" + name + "-user' config section.") }} +{%- endif %} {% set user=users[name] %} {%- if not homedir %} {% set homedir='/var/lib/' + name %} diff --git a/docker/ovn/ovn-sb-db-relay/Dockerfile.j2 b/docker/ovn/ovn-sb-db-relay/Dockerfile.j2 new file mode 100644 index 0000000000..4beebaa4fa --- /dev/null +++ b/docker/ovn/ovn-sb-db-relay/Dockerfile.j2 @@ -0,0 +1,15 @@ +FROM {{ namespace }}/{{ image_prefix }}ovn-sb-db-server:{{ tag }} +{% block labels %} +LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build_date }}" +{% endblock %} + +{% block ovn_sb_db_server_header %}{% endblock %} + +{% block ovn_sb_db_relay_ovn_ctl %} +{# TODO(mnasiadka): Switch to 25.03 branch when available, ideally rpm/deb packages will be + available. #} +RUN curl -o /usr/share/ovn/scripts/ovn-ctl https://raw.githubusercontent.com/ovn-org/ovn/refs/heads/main/utilities/ovn-ctl +{% endblock %} + +{% block ovn_sb_db_relay_footer %}{% endblock %} +{% block footer %}{% endblock %} diff --git a/kolla/image/kolla_worker.py b/kolla/image/kolla_worker.py index 798c8eaa7d..3a78a7f35b 100644 --- a/kolla/image/kolla_worker.py +++ b/kolla/image/kolla_worker.py @@ -271,6 +271,7 @@ def _get_methods(self): return { 'debian_package_install': jinja_methods.debian_package_install, 'handle_repos': jinja_methods.handle_repos, + 'raise_error': jinja_methods.raise_error, } def get_users(self): @@ -280,7 +281,17 @@ def get_users(self): for section in all_sections: match = re.search('^.*-user$', section) if match: - user = self.conf[match.group(0)] + cfg_group_name = match.group(0) + + if cfg_group_name not in self.conf._groups: + self.conf.register_opts( + common_config.get_user_opts( + None, None, + # cut `-user` suffix + group=cfg_group_name[:-5]), + group=cfg_group_name + ) + user = self.conf[cfg_group_name] ret[match.group(0)[:-5]] = { 'uid': user.uid, 'gid': user.gid, diff --git a/kolla/image/tasks.py b/kolla/image/tasks.py index a36ba5ceac..28af3d52d0 100644 --- a/kolla/image/tasks.py +++ b/kolla/image/tasks.py @@ -232,6 +232,7 @@ def reset_userinfo(tarinfo): git.Git().clone(source['source'], clone_dir) git.Git(clone_dir).checkout(source['reference']) reference_sha = git.Git(clone_dir).rev_parse('HEAD') + git.Git(clone_dir).remote("remove", "origin") self.logger.debug("Git checkout by reference %s (%s)", source['reference'], reference_sha) except Exception as e: diff --git a/kolla/template/methods.py b/kolla/template/methods.py index 6747504fa1..e9edbfeb01 100644 --- a/kolla/template/methods.py +++ b/kolla/template/methods.py @@ -11,6 +11,8 @@ # limitations under the License. import os +import typing as t + import yaml from jinja2 import pass_context @@ -150,3 +152,7 @@ def handle_repos(context, reponames, mode): commands = "RUN %s" % commands return commands + + +def raise_error(msg: str) -> t.NoReturn: + raise Exception(msg) diff --git a/releasenotes/notes/bring-configure_user-macro-to-custom-templates-61c143326a35c7ed.yaml b/releasenotes/notes/bring-configure_user-macro-to-custom-templates-61c143326a35c7ed.yaml new file mode 100644 index 0000000000..ca7c6559ed --- /dev/null +++ b/releasenotes/notes/bring-configure_user-macro-to-custom-templates-61c143326a35c7ed.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + Extends the support of externally-managed projects provided by the + ``--docker-dir`` option with an ability to use ``configure_user`` + jinja2 macros like Kolla built-in projects. + The operator should specify "non-default" user details with + ``-user`` configuration section and include info for + ``uid`` and ``gid`` at least. diff --git a/releasenotes/notes/bug-2098904-4c5670049a7e1a66.yaml b/releasenotes/notes/bug-2098904-4c5670049a7e1a66.yaml new file mode 100644 index 0000000000..7d8e3dafb8 --- /dev/null +++ b/releasenotes/notes/bug-2098904-4c5670049a7e1a66.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Removes Git remote URLs after cloning to prevent credential exposure. + `LP#2098904 `__ diff --git a/roles/configure-ephemeral/meta/main.yml b/roles/configure-ephemeral/meta/main.yml new file mode 100644 index 0000000000..a6b0fd849e --- /dev/null +++ b/roles/configure-ephemeral/meta/main.yml @@ -0,0 +1,4 @@ +--- +collections: + - ansible.posix + - community.general diff --git a/roles/configure-ephemeral/tasks/main.yml b/roles/configure-ephemeral/tasks/main.yml index 7d6402c301..d8733a8f93 100644 --- a/roles/configure-ephemeral/tasks/main.yml +++ b/roles/configure-ephemeral/tasks/main.yml @@ -3,7 +3,7 @@ # unallocated ephemeral device attached at /dev/xvde - name: Set ephemeral device if /dev/xvde exists when: ansible_devices["xvde"] is defined - set_fact: + ansible.builtin.set_fact: ephemeral_device: "/dev/xvde" # On other providers, we have a device called "ephemeral0". @@ -11,15 +11,15 @@ when: ephemeral_device is undefined block: - name: Get ephemeral0 device node - command: /sbin/blkid -L ephemeral0 + ansible.builtin.command: /sbin/blkid -L ephemeral0 register: ephemeral0 # rc !=0 is expected - failed_when: False - changed_when: False + failed_when: false + changed_when: false - name: Set ephemeral device if LABEL exists when: "ephemeral0.rc == 0" - set_fact: + ansible.builtin.set_fact: ephemeral_device: "{{ ephemeral0.stdout }}" - name: Configure additional disk (if available) @@ -53,13 +53,13 @@ fstype: ext4 dev: "{{ ephemeral_device }}" - - name: "Ensure {{ configure_ephemeral_mountpoint }} mountpoint is created" + - name: "Ensure mountpoint {{ configure_ephemeral_mountpoint }}" ansible.builtin.file: path: "{{ configure_ephemeral_mountpoint }}" owner: root group: root state: directory - mode: 0755 + mode: "0755" - name: Mount additional filesystem ansible.posix.mount: diff --git a/roles/kolla-build-config/tasks/main.yml b/roles/kolla-build-config/tasks/main.yml index f4d7f3eec4..4460ceb670 100644 --- a/roles/kolla-build-config/tasks/main.yml +++ b/roles/kolla-build-config/tasks/main.yml @@ -1,5 +1,8 @@ --- - name: Ensure kolla-build.conf exists - template: + ansible.builtin.template: src: kolla-build.conf.j2 dest: "{{ kolla_build_config_path }}" + mode: "0660" + owner: zuul + group: zuul diff --git a/roles/kolla-build-deps/tasks/main.yml b/roles/kolla-build-deps/tasks/main.yml new file mode 100644 index 0000000000..ebb5f00fa9 --- /dev/null +++ b/roles/kolla-build-deps/tasks/main.yml @@ -0,0 +1,64 @@ +--- +- name: Create dir for kolla logs + ansible.builtin.file: + path: "{{ kolla_logs_dir }}" + state: directory + mode: "0755" + owner: zuul + group: zuul + +- name: Dump host info to logs + ansible.builtin.command: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tools/dump_info.sh" + args: + chdir: "{{ kolla_logs_dir }}" + changed_when: true + +- name: Create dir for kolla build logs + ansible.builtin.file: + path: "{{ kolla_build_logs_dir }}" + state: directory + mode: "0755" + owner: zuul + group: zuul + +- name: Install Python3 pip and setuptools + ansible.builtin.package: + name: + - python3-pip + - python3-setuptools + become: true + +# NOTE(hrw): On RedHat systems venv is part of python3-libs +- name: Install Python3 venv on Debian systems + ansible.builtin.package: + name: + - python3-venv + become: true + when: ansible_facts.os_family == "Debian" + +- name: Install kolla in a venv + ansible.builtin.pip: + name: + - "pip" + - "." + state: latest + virtualenv: "{{ virtualenv_path }}" + virtualenv_command: "python3.{{ ansible_facts.python.version.minor }} -m venv" + chdir: "{{ zuul.project.src_dir }}" + +- name: Configure container engine + ansible.builtin.include_role: + name: "{{ container_engine }}" + +- name: Install podman/docker_sdk in a venv + vars: + virtualenv: "{{ virtualenv_path }}" + docker_sdk_virtualenv_owner: "{{ lookup('env', 'USER') }}" + ansible.builtin.include_role: + name: "{{ container_engine }}_sdk" + +- name: Ensure container engine socket is world-writable + ansible.builtin.file: + path: "{{ '/run/docker.sock' if container_engine == 'docker' else '/run/podman/podman.sock' }}" + mode: "0666" + become: true diff --git a/roles/kolla-build/tasks/main.yml b/roles/kolla-build/tasks/main.yml index ec09e4f79a..e5529f381d 100644 --- a/roles/kolla-build/tasks/main.yml +++ b/roles/kolla-build/tasks/main.yml @@ -3,6 +3,9 @@ ansible.builtin.template: src: "{{ kolla_build_template_overrides_path }}" dest: /etc/kolla/template_overrides.j2 + mode: "0660" + owner: zuul + group: zuul when: kolla_build_template_overrides_path | length > 0 - name: Run kolla-build to template out dockerfiles @@ -10,9 +13,11 @@ cmd: >- {{ kolla_build_venv_path }}/bin/kolla-build --template-only --work-dir {{ kolla_build_logs_dir }}/work_dir + changed_when: true - name: Run kolla-build vars: platform: "{{ '--platform linux/arm64' if base_arch == 'aarch64' and base_arch != ansible_facts.architecture }}" ansible.builtin.command: cmd: "{{ kolla_build_venv_path }}/bin/kolla-build {{ platform }}" + changed_when: true diff --git a/test-requirements.txt b/test-requirements.txt index ad35b87960..e9fa093f77 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,3 +1,4 @@ +ansible-lint<25 # MIT bandit!=1.6.0,>=1.1.0 # Apache-2.0 bashate>=0.5.1 # Apache-2.0 coverage!=4.4,>=4.0 # Apache-2.0 diff --git a/test-requirements.yml b/test-requirements.yml new file mode 100644 index 0000000000..51f1743f6f --- /dev/null +++ b/test-requirements.yml @@ -0,0 +1,5 @@ +--- +collections: + - ansible.posix + - community.docker + - community.general diff --git a/tests/files/process_build_logs.py b/tests/files/process_build_logs.py index 22b7182273..0db5228d9e 100755 --- a/tests/files/process_build_logs.py +++ b/tests/files/process_build_logs.py @@ -30,7 +30,7 @@ required=True) args = vars(parser.parse_args()) -if args['base'] not in ['centos']: +if args['base'] not in ['rocky', 'centos']: print("Non rpm-based distros are not yet supported.") sys.exit() diff --git a/tests/playbooks/post.yml b/tests/playbooks/post.yml index 7245b7ba8d..0306638103 100644 --- a/tests/playbooks/post.yml +++ b/tests/playbooks/post.yml @@ -1,13 +1,16 @@ --- -- hosts: all +- name: Zuul post + hosts: all tasks: - name: Dump host info to logs - command: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tools/dump_info.sh" + ansible.builtin.command: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tools/dump_info.sh" args: chdir: "{{ kolla_logs_dir }}" + changed_when: true - name: Collect various info to logs - shell: + changed_when: true + ansible.builtin.shell: cmd: | set +o errexit set -o xtrace @@ -63,7 +66,9 @@ ps -eo user,pid,ppid,lwp,%cpu,%mem,size,rss,cmd > ps.txt # docker related information - (sudo {{ container_engine }} info && sudo {{ container_engine }} system df && sudo {{ container_engine }} system df -v) > {{ container_engine }}-info.txt + (sudo {{ container_engine }} info && \ + sudo {{ container_engine }} system df && \ + sudo {{ container_engine }} system df -v) > {{ container_engine }}-info.txt sudo cp -r /etc/kolla kolla_configs @@ -71,6 +76,7 @@ sudo chown -R {{ ansible_user_id }}: . # Parse build logs and extract pkg install info - {{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tests/files/process_build_logs.py -l build -b {{ base_distro }} > packages-info.txt + {{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tests/files/process_build_logs.py \ + -l build -b {{ base_distro }} > packages-info.txt executable: /bin/bash chdir: "{{ kolla_logs_dir }}" diff --git a/tests/playbooks/pre.yml b/tests/playbooks/pre.yml index 6d59c40497..7733709678 100644 --- a/tests/playbooks/pre.yml +++ b/tests/playbooks/pre.yml @@ -1,201 +1,8 @@ --- -- hosts: all +- name: Zuul pre + hosts: all vars_files: - ../vars/zuul.yml roles: - configure-ephemeral - tasks: - - name: Create dir for kolla logs - file: - path: "{{ kolla_logs_dir }}" - state: directory - - - name: Dump host info to logs - command: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tools/dump_info.sh" - args: - chdir: "{{ kolla_logs_dir }}" - - - name: Create dir for kolla build logs - file: - path: "{{ kolla_build_logs_dir }}" - state: directory - - - name: Install Python3 pip and setuptools - package: - name: - - python3-pip - - python3-setuptools - become: true - - # NOTE(hrw): On RedHat systems venv is part of python3-libs - - name: Install Python3 venv on Debian systems - package: - name: - - python3-venv - become: true - when: - ansible_os_family == "Debian" - - - name: Create virtualenv - command: python3 -m venv {{ virtualenv_path }} - - - name: Install kolla - command: "{{ virtualenv_path }}/bin/python -m pip install {{ zuul.project.src_dir }}" - - - name: Install docker python library - command: "{{ virtualenv_path }}/bin/python -m pip install docker" - when: container_engine == "docker" - - - name: Install podman python library - command: "{{ virtualenv_path }}/bin/python -m pip install podman rich" - when: container_engine == "podman" - - - name: Configure Docker repo for Debian/Ubuntu - block: - - name: Add key for Docker APT repository - apt_key: - url: "{{ nodepool_docker_proxy }}/{{ ansible_distribution | lower }}/gpg" - state: present - - - name: Add Docker APT repository - apt_repository: - repo: "deb {{ nodepool_docker_proxy }}/{{ ansible_distribution | lower }} {{ ansible_distribution_release }} stable" - state: present - - - name: Ensure AppArmor is installed - package: - name: apparmor - state: present - - when: - - ansible_os_family == "Debian" - - container_engine == "docker" - become: true - - - name: Configure Docker repo for CentOS - block: - - name: Add Docker repository - yum_repository: - name: docker - description: Docker - baseurl: "{{ nodepool_docker_proxy }}/centos/9/$basearch/stable" - enabled: yes - gpgcheck: yes - gpgkey: "{{ nodepool_docker_proxy }}/centos/gpg" - # module_hotfixes: True # enabled below (dnf, not yum, feature) - - - name: Enable module_hotfixes in Docker YUM repository - lineinfile: - path: /etc/yum.repos.d/docker.repo - line: 'module_hotfixes=True' - when: - - ansible_os_family == "RedHat" - - container_engine == "docker" - become: true - - - name: Configure docker - block: - - name: Ensure /etc/docker exists - become: true - file: - path: /etc/docker - state: directory - mode: 0755 - - - name: Configure registry-mirror in daemon.json - become: true - copy: - dest: /etc/docker/daemon.json - content: | - { - "registry-mirrors": [ - "http://{{ zuul_site_mirror_fqdn }}:8082/" - ] - } - - # NOTE(yoctozepto): We configure Docker before installing it because Debuntu starts services - # during installation. - - name: Install Docker - package: - name: docker-ce - become: true - - - name: Ensure Docker service is started - service: - name: docker - state: started - become: true - - - name: Ensure Docker socket is world-writable - file: - path: /run/docker.sock - mode: 0666 - become: true - when: - - container_engine == "docker" - - - name: Configure podman - block: - - name: Ensure /etc/containers exists - become: true - file: - path: /etc/containers - state: directory - mode: 0755 - - - name: Configure registries.conf - become: true - copy: - dest: /etc/containers/registries.conf - content: | - unqualified-search-registries = ['docker.io'] - - [[registry]] - prefix = "docker.io" - location = "docker.io" - - [[registry.mirror]] - prefix = "docker.io" - location = "{{ zuul_site_mirror_fqdn }}:8082" - - - name: Install Podman - package: - name: podman - become: true - - # NOTE(jangutter): It appears that the default mount option - # in the shipped `/etc/containers/storage.conf` for EL9 based distros - # (Rocky, CentOS Stream, ...) has severe performance implications for - # Kolla builds. This is because enabling `metacopy=on` disables `Native - # Overlay Diff` This can be removed if the config is dropped from those - # distros, or the underlying incompatibility can somehow be addressed. - # Debian based distros do not ship `storage.conf` and seem - # to be unaffected. - - name: Remove metacopy, enable native overlay diff - ini_file: - path: /etc/containers/storage.conf - section: storage.options.overlay - option: mountopt - value: '"nodev"' - become: true - when: ansible_os_family == "RedHat" - - - name: Ensure Podman service is started - service: - name: podman - state: started - become: true - - - name: Ensure Podman socket is world-writable - file: - path: /run/podman/podman.sock - mode: 0666 - become: true - when: - - container_engine == "podman" - - - name: Run multiarch/qemu-user-static image to support cross-arch build - command: - cmd: "{{ container_engine }} run --rm --privileged multiarch/qemu-user-static --reset -p yes" - become: true - when: ansible_facts.architecture != (base_arch | default('x86_64')) + - kolla-build-deps diff --git a/tests/playbooks/publish.yml b/tests/playbooks/publish.yml index 76352d299d..fd9124cd57 100644 --- a/tests/playbooks/publish.yml +++ b/tests/playbooks/publish.yml @@ -1,44 +1,48 @@ --- -- hosts: all +- name: Zuul publish + hosts: all vars: # NOTE(yoctozepto): We need Docker SDK, the best source is Kolla venv. ansible_python_interpreter: "{{ virtualenv_path }}/bin/python" tasks: - name: List all containers - docker_host_info: - images: yes + community.docker.docker_host_info: + images: true images_filters: reference: "{{ kolla_namespace }}/*" register: docker_host_info - - block: + - name: Publish to Dockerhub + when: kolla_registry == 'dockerhub' + block: - name: Login to Dockerhub - docker_login: + community.docker.docker_login: username: "{{ kolla_dockerhub_credentials.username | trim }}" password: "{{ kolla_dockerhub_credentials.password | trim }}" - name: Push built container images - docker_image: + community.docker.docker_image: name: "{{ item.RepoTags.0 }}" - push: yes + push: true source: local loop: "{{ docker_host_info.images }}" register: push_status until: push_status.failed is false retries: 5 - when: kolla_registry == 'dockerhub' - - block: + - name: Publish to quay.io + when: kolla_registry == 'quay.io' + block: - name: Login to quay.io - docker_login: + community.docker.docker_login: registry: quay.io username: "{{ kolla_quay_io_creds.username | trim }}" password: "{{ kolla_quay_io_creds.password | trim }}" - name: Push built container images - docker_image: + community.docker.docker_image: name: "{{ item.RepoTags.0 }}" - push: yes + push: true repository: "quay.io/{{ item.RepoTags.0 }}" source: local loop: "{{ docker_host_info.images }}" @@ -47,7 +51,7 @@ retries: 5 - name: Ensure repository visibility is public - uri: + ansible.builtin.uri: url: "https://quay.io/api/v1/repository/{{ item.RepoTags.0 }}/changevisibility" method: POST headers: @@ -55,4 +59,3 @@ body: '{"visibility": "public"}' body_format: json loop: "{{ docker_host_info.images }}" - when: kolla_registry == 'quay.io' diff --git a/tests/playbooks/run.yml b/tests/playbooks/run.yml index f21de9f330..c946343a53 100644 --- a/tests/playbooks/run.yml +++ b/tests/playbooks/run.yml @@ -1,5 +1,6 @@ --- -- hosts: all +- name: Zuul run + hosts: all vars_files: - ../vars/zuul.yml vars: @@ -18,10 +19,10 @@ network_mode: host tasks: - name: Ensure /etc/kolla exists - file: + ansible.builtin.file: path: /etc/kolla state: directory - mode: 0777 + mode: "0777" become: true - name: Use ubuntu/debian base image from mirror @@ -29,7 +30,7 @@ kolla_mirror_config: DEFAULT: base_image: "quay.io/openstack.kolla/{{ base_distro }}" - set_fact: + ansible.builtin.set_fact: kolla_build_config: "{{ kolla_build_config | combine(kolla_mirror_config, recursive=True) }}" when: base_distro in ['debian', 'ubuntu'] @@ -39,13 +40,32 @@ DEFAULT: namespace: "{{ kolla_namespace }}" tag: "{{ zuul.branch | basename }}-{{ base_distro }}-{{ base_distro_version }}{{ tag_suffix }}" - set_fact: + ansible.builtin.set_fact: kolla_build_config: "{{ kolla_build_config | combine(kolla_publisher_config, recursive=True) }}" when: - publisher - - import_role: + - name: Add external docker dir config + vars: + kolla_build_external_docker_config: + DEFAULT: + docker_dir: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tests/templates/docker" + releaser: + type: git + reference: master + location: "https://opendev.org/openstack/releases.git" + releaser-user: + uid: 56000 + gid: 56000 + ansible.builtin.set_fact: + kolla_build_config: "{{ kolla_build_config | combine(kolla_build_external_docker_config, recursive=True) }}" + when: + - not publisher + + - name: Import kolla-build-config role + import_role: name: kolla-build-config - - import_role: + - name: Import kolla-build role + import_role: name: kolla-build diff --git a/tests/templates/docker/releaser/Dockerfile.j2 b/tests/templates/docker/releaser/Dockerfile.j2 new file mode 100644 index 0000000000..665302534f --- /dev/null +++ b/tests/templates/docker/releaser/Dockerfile.j2 @@ -0,0 +1,25 @@ +FROM {{ namespace }}/{{ image_prefix }}openstack-base:{{ tag }} + +{% block labels %} +LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build_date }}" +{% endblock %} + +{% block releaser_header %}{% endblock %} + +{% import "macros.j2" as macros with context %} + +{{ macros.configure_user(name='releaser') }} + +COPY extend_start.sh /usr/local/bin/kolla_extend_start + +ADD releaser-archive /releaser-source + +RUN ln -s releaser-source/* /releaser \ + && {{ macros.install_pip(['/releaser'] | customizable("pip_packages")) }} \ + && mkdir -p /etc/releaser \ + && chown -R releaser: /etc/releaser \ + && chmod 750 /etc/sudoers.d \ + && touch /usr/local/bin/kolla_releaser_extend_start \ + && chmod 644 /usr/local/bin/kolla_extend_start /usr/local/bin/kolla_releaser_extend_start + +{% block footer %}{% endblock %} diff --git a/tests/templates/docker/releaser/extend_start.sh b/tests/templates/docker/releaser/extend_start.sh new file mode 100644 index 0000000000..5d5c3b0411 --- /dev/null +++ b/tests/templates/docker/releaser/extend_start.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +if [[ ! -d "/var/log/kolla/releaser" ]]; then + mkdir -p /var/log/kolla/releaser +fi + +if [[ $(stat -c %a /var/log/kolla/releaser) != "755" ]]; then + chmod 755 /var/log/kolla/releaser +fi + +. /usr/local/bin/kolla_releaser_extend_start diff --git a/tests/vars/zuul.yml b/tests/vars/zuul.yml index 1c0c99e3db..e49f9d3560 100644 --- a/tests/vars/zuul.yml +++ b/tests/vars/zuul.yml @@ -7,4 +7,8 @@ nodepool_cbs_centos_proxy: "http://{{ zuul_site_mirror_fqdn }}:8080/cbs.centos" nodepool_docker_proxy: "http://{{ zuul_site_mirror_fqdn }}:8080/docker" # NOTE(hrw): wheel cache goes over 80/443 not on 8080 -nodepool_wheel_mirror: "https://{{ zuul_site_mirror_fqdn }}/wheel/{{ ansible_distribution | lower }}-{{ (ansible_os_family == 'Debian') | ternary(ansible_distribution_version, ansible_distribution_major_version) }}-{{ ansible_architecture | lower }}" +nodepool_wheel_mirror: >- + https://{{ zuul_site_mirror_fqdn }}/wheel/{{ ansible_distribution | lower }}- + {{ (ansible_os_family == 'Debian') | + ternary(ansible_distribution_version, ansible_distribution_major_version) }}- + {{ ansible_architecture | lower }} diff --git a/tox.ini b/tox.ini index 923d4d821d..e57c38d0fb 100644 --- a/tox.ini +++ b/tox.ini @@ -50,7 +50,6 @@ commands = deps = {[testenv]deps} codespell - yamllint allowlist_externals = bash commands = bash {toxinidir}/tools/run-bashate.sh @@ -58,8 +57,9 @@ commands = bash {toxinidir}/tools/validate-all-dockerfiles.sh python {toxinidir}/tools/validate-all-file.py bandit -r docker kolla tests tools - yamllint -s . codespell -I {toxinidir}/.codespell-ignore + ansible-galaxy collection install -r test-requirements.yml + ansible-lint [testenv:bandit] commands = bandit -r docker kolla tests tools